Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion nodelet/pkg/nodelet/nodelet.go
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ func (n *Nodelet) Run(ctx context.Context) error {

func CreateNodeletFromConfig(ctx context.Context, cfg *config.Config) (*Nodelet, error) {
log := zap.S()
phases, err := loadRolePhases(ctx, *cfg)
phases, err := loadRolePhases(ctx, cfg)
if err != nil {
// Phases could not be loaded. Cannot recover from this.
return nil, fmt.Errorf("could not load phases: %v", err)
Expand Down
2 changes: 1 addition & 1 deletion nodelet/pkg/pf9kube/pf9/pf9-kube/defaults.env
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ MAX_CERTS_RETRIES=10
source /etc/os-release
if [[ "$ID" == "centos" ]]; then
CERTS_SERIAL_FILE="/etc/pf9/kube.srl"
elif [[ "$ID" == "rhel" ]]; then
elif [[ "$ID" == "rhel" || "$ID" == "rocky" ]]; then
CERTS_SERIAL_FILE="/etc/pf9/kube.d/rootCA.srl"
fi

Expand Down
10 changes: 5 additions & 5 deletions nodelet/pkg/pf9kube/pf9/pf9-kube/master_utils.sh
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ function ensure_etcd_running()
#
# TODO
# PMK-3665: Customise ETCD in platform9 managed kubernetes cluster
#
#
# The flexibility of customizing ETCD with the help of environment variables
# needs support from DU side as well if we want it to be truly customizable
# at the time of cluster creation or at the time of cluster update.
Expand Down Expand Up @@ -152,7 +152,7 @@ function ensure_etcd_running()
#
# periodic can be used if you want to retain key value revisions from the
# last time window specified in ETCD_AUTO_COMPACTION_RETENTION env variable.
# e.g. 1h or 30m
# e.g. 1h or 30m
#
# revision can be used if you want to retains last n revisions of key values.
# You can specify the value in in ETCD_AUTO_COMPACTION_RETENTION env variable.
Expand Down Expand Up @@ -332,7 +332,7 @@ function prepare_conf_files()

# Image registries
local quay_registry="${QUAY_PRIVATE_REGISTRY:-quay.io}"
local k8s_registry="${K8S_PRIVATE_REGISTRY:-k8s.gcr.io}"
local k8s_registry="${K8S_PRIVATE_REGISTRY:-registry.k8s.io}"
local gcr_registry="${GCR_PRIVATE_REGISTRY:-gcr.io}"
local docker_registry="${DOCKER_PRIVATE_REGISTRY}"

Expand Down Expand Up @@ -877,7 +877,7 @@ function post_upgrade_monitoring_fix()
echo "Monitoring found installed, fixing ownership of objects in pf9-monitoring"
fi

# Need to create configmap because we are upgrading from an older pf9-kube version
# Need to create configmap because we are upgrading from an older pf9-kube version
# where the new owner configmap was not present
${KUBECTL_SYSTEM} create configmap monitoring-owner -n pf9-monitoring \
--dry-run -o yaml | ${KUBECTL_SYSTEM} apply -f -
Expand Down Expand Up @@ -1025,7 +1025,7 @@ function ensure_dns()
{
local coredns_template="${CONF_SRC_DIR}/networkapps/coredns.yaml"
local coredns_file="${CONF_SRC_DIR}/networkapps/coredns-applied.yaml"
local k8s_registry="${K8S_PRIVATE_REGISTRY:-k8s.gcr.io}"
local k8s_registry="${K8S_PRIVATE_REGISTRY:-registry.k8s.io}"

# Replace configuration values in calico spec with user input
sed -e "s|__DNS_IP__|${DNS_IP}|g" \
Expand Down
15 changes: 13 additions & 2 deletions nodelet/pkg/pf9kube/pf9/pf9-kube/os.sh
Original file line number Diff line number Diff line change
@@ -1,16 +1,27 @@
source /etc/os-release

if [[ "$ID" == "ubuntu" ]]; then
if [[ "$VERSION_ID" == "16.04" || "$VERSION_ID" == "18.04" || "$VERSION_ID" == "20.04" ]]; then
if [[ "$VERSION_ID" == "16.04" || "$VERSION_ID" == "18.04" || "$VERSION_ID" == "20.04" || "$VERSION_ID" == "22.04" ]]; then
source os_ubuntu.sh
export OS_FAMILY="ubuntu"
export OS_VERSION="${VERSION_ID}"
else
echo "Unknown Ubuntu version: ${VERSION_ID}"
exit 1
fi
elif [[ "$ID" == "centos" || "$ID" == "rhel" ]]; then
elif [[ "$ID" == "centos" || "$ID" == "rhel" || "$ID" == "rocky" ]]; then
source os_centos.sh
export OS_FAMILY="centos"
if [[ "$VERSION_ID" =~ 9.* ]]; then
export OS_VERSION="9.x"
elif [[ "$VERSION_ID" =~ 8.* ]]; then
export OS_VERSION="8.x"
elif [[ "$VERSION_ID" =~ 7.* ]]; then
export OS_VERSION="7.x"
else
echo "Unknown CentOS/RHEL version: ${VERSION_ID}"
exit 1
fi
else
echo "Unknown OS: ${ID}"
exit 1
Expand Down
42 changes: 31 additions & 11 deletions nodelet/pkg/pf9kube/pf9/pf9-kube/os_centos.sh
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,18 @@ DOCKER_PACKAGE="docker-ce"
DOCKER_CLI="docker-ce-cli"
CONTAINERD_PACKAGE="containerd.io"

if [[ "$VERSION_ID" =~ ^8.* ]]; then
if [[ "$VERSION_ID" =~ ^9.* ]]; then
DOCKER_PACKAGE_VERSION="3:20.10.24-3.el9"
DOCKER_CLI_VERSION="1:20.10.24-3.el9"
CONTAINERD_PACKAGE_VERSION="1.6.6-3.1.el9"
elif [[ "$VERSION_ID" =~ ^8.* ]]; then
DOCKER_PACKAGE_VERSION="3:20.10.6-3.el8"
DOCKER_CLI_VERSION="1:20.10.6-3.el8"
CONTAINERD_PACKAGE_VERSION="1.4.12-3.1.el8"
CONTAINERD_PACKAGE_VERSION="1.6.6-3.1.el8"
else
DOCKER_PACKAGE_VERSION="19.03.11-3.el7"
DOCKER_CLI_VERSION="19.03.11-3.el7"
CONTAINERD_PACKAGE_VERSION="1.4.12-3.1.el7"
CONTAINERD_PACKAGE_VERSION="1.6.6-3.1.el7"
fi


Expand Down Expand Up @@ -149,23 +153,39 @@ function install_runtime_repo()
{
# Add the repository key
rpm --import ${DOCKER_CENTOS_REPO_KEY}

# use $'' to make sure \n are interpreted correctly
local docker_repo_string=$'[dockerrepo]\nname=Docker CE Stable - \$basearch\n'

if [ ! -z $DOCKER_CENTOS_REPO_URL ]; then
# Install the repository
cat > /etc/yum.repos.d/docker.mirrors <<EOF
# Install the repository
if [ "$OS_VERSION" == "7.x" ]; then
DOCKER_REPO_UPSTREAM=https://download.docker.com/linux/centos/7/\$basearch/stable
elif [ "$OS_VERSION" == "8.x" ]; then
# use $'' to make sure \n are interpreted correctly
DOCKER_REPO_UPSTREAM=https://download.docker.com/linux/centos/8/\$basearch/stable
elif [ "$OS_VERSION" == "9.x" ]; then
DOCKER_REPO_UPSTREAM=https://download.docker.com/linux/centos/9/\$basearch/stable
else
echo "Unknown CentOS/RHEL version: ${OS_VERSION}"
exit 1
fi
cat > /etc/yum.repos.d/docker.mirrors <<EOF
$DOCKER_CENTOS_REPO_URL
https://download.docker.com/linux/centos/7/\$basearch/stable
$DOCKER_REPO_UPSTREAM
EOF
# use $'' to make sure \n are interpreted correctly
docker_repo_string+=$'mirrorlist=file:///etc/yum.repos.d/docker.mirrors\n'
else
# use the default configuration
docker_repo_string+=$'baseurl=https://download.docker.com/linux/centos/7/\$basearch/stable\nenabled=1\ngpgcheck=1\n'
if [ "$OS_VERSION" == "7.x" ]; then
docker_repo_string+=$'baseurl=https://download.docker.com/linux/centos/7/$basearch/stable\nenabled=1\ngpgcheck=1\n'
elif [ "$OS_VERSION" == "8.x" ]; then
docker_repo_string+=$'baseurl=https://download.docker.com/linux/centos/8/$basearch/stable\nenabled=1\ngpgcheck=1\n'
elif [ "$OS_VERSION" == "9.x" ]; then
docker_repo_string+=$'baseurl=https://download.docker.com/linux/centos/9/$basearch/stable\nenabled=1\ngpgcheck=1\n'
else
echo "Unknown CentOS/RHEL version: ${OS_VERSION}"
exit 1
fi
fi

# Install the repository
echo "$docker_repo_string" > /etc/yum.repos.d/docker.repo
}
Expand Down
10 changes: 5 additions & 5 deletions nodelet/pkg/pf9kube/pf9/pf9-kube/utils.sh
Original file line number Diff line number Diff line change
Expand Up @@ -546,7 +546,7 @@ function ensure_kubelet_running()
local node_name=$1
local kubeconfig="/etc/pf9/kube.d/kubeconfigs/kubelet.yaml"
local log_dir_path="/var/log/pf9/kubelet/"
local k8s_registry="${K8S_PRIVATE_REGISTRY:-k8s.gcr.io}"
local k8s_registry="${K8S_PRIVATE_REGISTRY:-registry.k8s.io}"
local pause_img="${k8s_registry}/pause:3.6"

prepare_kubelet_bootstrap_config
Expand All @@ -565,7 +565,7 @@ function ensure_kubelet_running()
--pod-infra-container-image=${pause_img} \
--dynamic-config-dir=${KUBELET_DYNAMIC_CONFIG_DIR} \
--cgroup-driver=${CONTAINERD_CGROUP}"

# container-runtime: The container runtime to use. Possible values: docker, remote
# container-runtime-endpoint: The endpoint of remote runtime service. Currently unix socket endpoint is supported on Linux
# Examples: unix:///var/run/dockershim.sock or /run/containerd/containerd.sock
Expand All @@ -580,7 +580,7 @@ function ensure_kubelet_running()

if [ "$RUNTIME" == "containerd" ]; then
local container_log_max_files=${CONTAINER_LOG_MAX_FILES:-${DOCKER_LOG_MAX_FILE}}
# Why not use DOCKER_LOG_MAX_SIZE variable?
# Why not use DOCKER_LOG_MAX_SIZE variable?
# The formatting for docker config is 10m while kubelet expects 10Mi. To avoid implement string manipulation in bash just hardcoding
# the same default as docker config for now.
local container_log_max_size=${CONTAINER_LOG_MAX_SIZE:-"10Mi"}
Expand Down Expand Up @@ -719,7 +719,7 @@ function ensure_proxy_running()
--privileged \
--volume ${kubeconfig}:${kubeconfig_in_container}"

local k8s_registry="${K8S_PRIVATE_REGISTRY:-k8s.gcr.io}"
local k8s_registry="${K8S_PRIVATE_REGISTRY:-registry.k8s.io}"
local container_name="proxy"
local container_img="${k8s_registry}/kube-proxy:$KUBERNETES_VERSION"

Expand Down Expand Up @@ -1227,7 +1227,7 @@ function make_kubeconfig()
fi
if [[ "$kube_server" == "$MASTER_IP" && "$USE_HOSTNAME" == "true" && "$CLOUD_PROVIDER_TYPE" == "local" && "$MASTER_VIP_ENABLED" == "false" ]]; then
kube_server=$HOSTNAME
fi
fi
if [ "$K8S_API_PORT" != "443" ]; then
kube_server="${kube_server}:${K8S_API_PORT}"
fi
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,12 @@ func (p *Phase) GetPhaseName() string {
func (p *Phase) runCommand(ctx context.Context, op string, cfg config.Config) ([]string, int) {
baseCmd := constants.BaseCommand
if _, ok := constants.ValidCgroupOps[op]; ok && cfg.UseCgroups {
baseCmd = constants.BaseCgroupCommand
if cfg.CgroupsV2 {
baseCmd = append(constants.BaseCgroupV2Command, fmt.Sprintf(constants.CgroupV2QuotaParam, cfg.CPULimit))
baseCmd = append(baseCmd, constants.BaseCommand...)
} else {
baseCmd = constants.BaseCgroupCommand
}
}
command := append(baseCmd, p.Filename, op)
if cfg.IsDebug() {
Expand Down
42 changes: 28 additions & 14 deletions nodelet/pkg/phases/phases.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
"regexp"
"time"

"github.com/platform9/nodelet/nodelet/pkg/utils/command"
Expand All @@ -18,7 +19,7 @@ var GetLocalCmd = command.New

// InitAndLoadRolePhases initializes and then returns an map of order -> phase
// when successful otherwise returns an error
func InitAndLoadRolePhases(ctx context.Context, cfg config.Config) ([]PhaseInterface, error) {
func InitAndLoadRolePhases(ctx context.Context, cfg *config.Config) ([]PhaseInterface, error) {
var err error
if cfg.UseCgroups {
if cfg.DisableScripts {
Expand Down Expand Up @@ -49,9 +50,20 @@ func InitAndLoadRolePhases(ctx context.Context, cfg config.Config) ([]PhaseInter
return phaseList, nil
}

func setupCgroup(ctx context.Context, cfg config.Config) error {
func setupCgroup(ctx context.Context, cfg *config.Config) error {
localCmd := GetLocalCmd()
commands := [][]string{}

_, err := localCmd.RunCommand(ctx, nil, -1, "", "grep", "-i", "Rocky", "/etc/os-release")
if err == nil {
_, out, err := localCmd.RunCommandWithStdOut(ctx, nil, -1, "", "sed", "-nE", `s/^VERSION_ID=(.+)/\1/p`, "/etc/os-release")
if err == nil && len(out) > 0 {
if match, _ := regexp.MatchString(`.*9\.1\.*`, string(out[0])); match {
cfg.CgroupsV2 = true
}
}
}

// CPU limit percentage
cpuQuotaPtc := cfg.CPULimit
if cpuQuotaPtc <= 0 || cpuQuotaPtc > 100 {
Expand All @@ -61,18 +73,20 @@ func setupCgroup(ctx context.Context, cfg config.Config) error {
// Convert CPU limit percentage to time slice in microseconds
// Refer last example here - https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt
// We are trying to calculate a quota limit considering the period of 1s i.e. 1000000us
cpuQuota := cpuQuotaPtc / 100 * float64((1 * time.Second).Microseconds())
cpuQuotaCmd := append(constants.CgroupQuotaCmd, fmt.Sprintf(constants.CgroupQuotaParam, cpuQuota), constants.CgroupName)
commands = append(commands, constants.CgroupCreateCmd)
commands = append(commands, constants.CgroupPeriodCmd)
commands = append(commands, cpuQuotaCmd)
for _, command := range commands {
exec := command[0]
args := command[1:]
_, err := localCmd.RunCommand(ctx, nil, -1, "", exec, args...)
if err != nil {
zap.S().Warnf("Error running command: %v", command)
return err
if !cfg.CgroupsV2 {
cpuQuota := cpuQuotaPtc / 100 * float64((1 * time.Second).Microseconds())
cpuQuotaCmd := append(constants.CgroupQuotaCmd, fmt.Sprintf(constants.CgroupQuotaParam, cpuQuota), constants.CgroupName)
commands = append(commands, constants.CgroupCreateCmd)
commands = append(commands, constants.CgroupPeriodCmd)
commands = append(commands, cpuQuotaCmd)
for _, command := range commands {
exec := command[0]
args := command[1:]
_, err := localCmd.RunCommand(ctx, nil, -1, "", exec, args...)
if err != nil {
zap.S().Warnf("Error running command: %v", command)
return err
}
}
}
return nil
Expand Down
14 changes: 8 additions & 6 deletions nodelet/pkg/phases/phases_suite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ var _ = Describe("Test phases.go", func() {
fakeCfg.DisableScripts = false
fakeCfg.ClusterRole = "master"
setupCgroupCmdMocks(true, mockCtrl, ctx)
phases, err := phases.InitAndLoadRolePhases(ctx, *fakeCfg)
phases, err := phases.InitAndLoadRolePhases(ctx, fakeCfg)
assert.Nil(GinkgoT(), err)
assert.NotEmpty(GinkgoT(), phases)
})
Expand All @@ -71,7 +71,7 @@ var _ = Describe("Test phases.go", func() {
fakeCfg.DisableScripts = false
fakeCfg.ClusterRole = "worker"
setupCgroupCmdMocks(true, mockCtrl, ctx)
phases, err := phases.InitAndLoadRolePhases(ctx, *fakeCfg)
phases, err := phases.InitAndLoadRolePhases(ctx, fakeCfg)
assert.Nil(GinkgoT(), err)
assert.NotEmpty(GinkgoT(), phases)
})
Expand All @@ -80,7 +80,7 @@ var _ = Describe("Test phases.go", func() {
fakeCfg.DisableScripts = false
fakeCfg.ClusterRole = "none"
setupCgroupCmdMocks(true, mockCtrl, ctx)
phases, err := phases.InitAndLoadRolePhases(ctx, *fakeCfg)
phases, err := phases.InitAndLoadRolePhases(ctx, fakeCfg)
assert.Nil(GinkgoT(), err)
assert.NotEmpty(GinkgoT(), phases)
})
Expand All @@ -92,7 +92,7 @@ var _ = Describe("Test phases.go", func() {
fakeCfg.DisableScripts = false
fakeCfg.ClusterRole = "master"
setupCgroupCmdMocks(false, mockCtrl, ctx)
phases, err := phases.InitAndLoadRolePhases(ctx, *fakeCfg)
phases, err := phases.InitAndLoadRolePhases(ctx, fakeCfg)
assert.Nil(GinkgoT(), err)
assert.NotEmpty(GinkgoT(), phases)
})
Expand All @@ -101,7 +101,7 @@ var _ = Describe("Test phases.go", func() {
fakeCfg.DisableScripts = false
fakeCfg.ClusterRole = "worker"
setupCgroupCmdMocks(false, mockCtrl, ctx)
phases, err := phases.InitAndLoadRolePhases(ctx, *fakeCfg)
phases, err := phases.InitAndLoadRolePhases(ctx, fakeCfg)
assert.Nil(GinkgoT(), err)
assert.NotEmpty(GinkgoT(), phases)
})
Expand All @@ -110,7 +110,7 @@ var _ = Describe("Test phases.go", func() {
fakeCfg.DisableScripts = false
fakeCfg.ClusterRole = "none"
setupCgroupCmdMocks(false, mockCtrl, ctx)
phases, err := phases.InitAndLoadRolePhases(ctx, *fakeCfg)
phases, err := phases.InitAndLoadRolePhases(ctx, fakeCfg)
assert.Nil(GinkgoT(), err)
assert.NotEmpty(GinkgoT(), phases)
})
Expand All @@ -126,6 +126,8 @@ func setupCgroupCmdMocks(enabled bool, mockCtrl *gomock.Controller, ctx context.
if enabled {
cmdCount = 1
}
mockCmd.EXPECT().RunCommand(ctx, nil, -1, "", "grep", gomock.Any(), gomock.Any(), "/etc/os-release").Return(0, nil).Times(cmdCount)
mockCmd.EXPECT().RunCommandWithStdOut(ctx, nil, -1, "", "sed", gomock.Any(), gomock.Any(), "/etc/os-release").Return(0, []string{}, nil).Times(cmdCount)
mockCmd.EXPECT().RunCommand(ctx, nil, -1, "", constants.CgroupCreateCmd[0], constants.CgroupCreateCmd[1], gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(0, nil).Times(cmdCount)
mockCmd.EXPECT().RunCommand(ctx, nil, -1, "", constants.CgroupPeriodCmd[0], constants.CgroupPeriodCmd[1], gomock.Any(), gomock.Any(), gomock.Any()).Return(0, nil).Times(cmdCount)
defaultCPUQuota := fmt.Sprintf(constants.CgroupQuotaParam, 400000)
Expand Down
5 changes: 4 additions & 1 deletion nodelet/pkg/utils/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ var DefaultConfig = Config{
ConnectTimeout: 20,
FullRetryCount: 10,
UseCgroups: true,
CgroupsV2: false,
PhaseRetry: 3,
CPULimit: 40, // percentage
LoopInterval: constants.DefaultLoopInterval, // seconds
Expand Down Expand Up @@ -62,6 +63,7 @@ type Config struct {
KubeServiceState string `mapstructure:"KUBE_SERVICE_STATE"`
FullRetryCount int `mapstructure:"FULL_RETRY_COUNT"`
UseCgroups bool `mapstructure:"USE_CGROUPS"`
CgroupsV2 bool `mapstructure:"CGROUPS_V2"`
PhaseRetry int `mapstructure:"PHASE_RETRY"`
CPULimit float64 `mapstructure:"CPU_LIMIT"`
PF9StatusThresholdSeconds int `mapstructure:"PF9_STATUS_THRESHOLD_SECONDS"`
Expand Down Expand Up @@ -138,7 +140,8 @@ func getDefaultConfig() *Config {

/*
GetConfigFromDir : Tries to load YAML config files from configDir i.e. /etc/pf9/nodelet directory.
This function returns an error if the directory is inaccessible or if no config files could be loaded

This function returns an error if the directory is inaccessible or if no config files could be loaded
*/
func GetConfigFromDir(configDir string) (*Config, error) {
pf9File := fileio.New()
Expand Down
1 change: 1 addition & 0 deletions nodelet/pkg/utils/config/config_suite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ var _ = Describe("Test config.go", func() {
KubeServiceState: constants.ServiceStateTrue,
FullRetryCount: 10,
UseCgroups: true,
CgroupsV2: false,
PhaseRetry: 3,
CPULimit: 40,
PF9StatusThresholdSeconds: 30,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ CONNECTION_TIMEOUT: 30
KUBE_SERVICE_STATE: "true"
FULL_RETRY_COUNT: 10
USE_CGROUPS: True
CGROUPS_V2: False
PHASE_RETRY: 3
CPU_LIMIT: 40.0
PF9_STATUS_THRESHOLD_SECONDS: 30
Loading