diff --git a/Makefile b/Makefile index ffd08c80..a4a74839 100644 --- a/Makefile +++ b/Makefile @@ -299,6 +299,7 @@ $(CNI_PLUGINS_DIR): | $(CNI_PLUGINS_BASE_DIR) cni-plugins: $(CNI_PLUGINS_DIR) + # CONTAINERD CLI # NERDCTL install diff --git a/nodelet/Makefile b/nodelet/Makefile index f8dc0d4f..a5d5400e 100644 --- a/nodelet/Makefile +++ b/nodelet/Makefile @@ -14,6 +14,8 @@ DOCKER_BUILD_CMD=${DOCKER} build DOCKER_RMI_CMD=${DOCKER} rmi RM=rm REPEAT_TESTS ?= "1" +WGET_CMD ?= "wget" + # Using PWD is not guaranteed to be the directory of the Makefile. Use these instead: MAKE_PATH := $(abspath $(lastword $(MAKEFILE_LIST))) @@ -43,8 +45,24 @@ GOLANG_DOCKER_MARKER := ${NODELET_BUILD_DIR}/${COMPILE_IMAGE}.marker .PHONY: nodelet-setup nodelet nodelet-test nodelet-clean .DEFAULT_GOAL:= nodelet +# Download containerd +CONTAINERD_VERSION := 1.6.1 +CONTAINERD_BASE_DIR := $(NODELET_SRC_DIR)/pkg/phases/container_runtime/containerd +CONTAINERD_FILE_NAME := containerd-${CONTAINERD_VERSION}-linux-amd64.tar.gz +CONTAINERD_FILE := $(CONTAINERD_BASE_DIR)/containerd.tar.gz +CONTAINERD_URL := https://github.com/containerd/containerd/releases/download/v${CONTAINERD_VERSION}/${CONTAINERD_FILE_NAME} + +$(CONTAINERD_BASE_DIR): + echo "make CONTAINERD_BASE_DIR: $(CONTAINERD_BASE_DIR)" + mkdir -p $@ + +$(CONTAINERD_FILE): | $(CONTAINERD_BASE_DIR) + echo "make CONTAINERD_FILE: $(CONTAINERD_FILE)" + ${WGET_CMD} ${CONTAINERD_URL} -O ${CONTAINERD_FILE} + + # Download/create build envs -nodelet-setup: ${NODELET_BIN_DIR} ${GOLANG_DOCKER_MARKER} +nodelet-setup: ${NODELET_BIN_DIR} ${GOLANG_DOCKER_MARKER} $(CONTAINERD_FILE) echo "setting up env for nodeletd" cd ${NODELET_SRC_DIR} && go mod vendor || true diff --git a/nodelet/go.mod b/nodelet/go.mod index 19c3a4fb..f04bfa0d 100644 --- a/nodelet/go.mod +++ b/nodelet/go.mod @@ -7,6 +7,7 @@ require ( github.com/cenkalti/backoff v2.2.1+incompatible github.com/choria-io/go-validator v1.1.1 github.com/containerd/containerd v1.6.2 + github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e github.com/erwinvaneyk/goversion v0.1.3 github.com/ghodss/yaml v1.0.0 github.com/golang/mock v1.6.0 @@ -60,6 +61,7 @@ require ( github.com/go-openapi/jsonreference v0.19.5 // indirect github.com/go-openapi/swag v0.19.14 // indirect github.com/gogo/googleapis v1.4.0 // indirect + github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect diff --git a/nodelet/go.sum b/nodelet/go.sum index 44969af4..387687ef 100644 --- a/nodelet/go.sum +++ b/nodelet/go.sum @@ -264,6 +264,7 @@ github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20161114122254-48702e0da86b/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= @@ -390,6 +391,7 @@ github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LB github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/godbus/dbus v0.0.0-20151105175453-c7fdd8b5cd55/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= +github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e h1:BWhy2j3IXJhjCbC68FptL43tDKIq8FladmaTs3Xs7Z8= github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= @@ -998,6 +1000,7 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= diff --git a/nodelet/pkg/phases/container_runtime/containerd_configure.go b/nodelet/pkg/phases/container_runtime/containerd_configure.go new file mode 100644 index 00000000..317f05f8 --- /dev/null +++ b/nodelet/pkg/phases/container_runtime/containerd_configure.go @@ -0,0 +1,120 @@ +package containerruntime + +import ( + "context" + "embed" + "fmt" + + "path" + "path/filepath" + + "github.com/coreos/go-systemd/dbus" + "github.com/platform9/nodelet/nodelet/pkg/untar" + + "github.com/platform9/nodelet/nodelet/pkg/embedutil" + "github.com/platform9/nodelet/nodelet/pkg/utils/config" + "github.com/platform9/nodelet/nodelet/pkg/utils/constants" + sunpikev1alpha1 "github.com/platform9/pf9-qbert/sunpike/apiserver/pkg/apis/sunpike/v1alpha1" + "go.uber.org/zap" +) + +//go:embed containerd/containerd.tar.gz +var containerdZip embed.FS + +type ContainerdConfigPhase struct { + baseDir string + hostPhase *sunpikev1alpha1.HostPhase + embedFs *embedutil.EmbedFS + conn *dbus.Conn + containerdRunPhase *ContainerdRunPhase +} + +// Extract containerd zip to the specified directory +func NewContainerdConfigPhase(baseDir string) (*ContainerdConfigPhase, error) { + + conn, err := dbus.NewSystemConnection() + if err != nil { + return nil, fmt.Errorf("error connecting to dbus: %v", err) + } + + embedFs := embedutil.EmbedFS{ + Fs: containerdZip, + Root: baseDir, + } + containerdRunPhase, err := newContainerdRunPhaseInternal(conn, baseDir) + if err != nil { + return nil, fmt.Errorf("error creating containerd run phase: %v", err) + } + runtimeConfigPhase := &ContainerdConfigPhase{ + baseDir: baseDir, + hostPhase: &sunpikev1alpha1.HostPhase{ + Name: "Configure Container Runtime", + Order: int32(constants.ConfigureRuntimePhaseOrder), + }, + embedFs: &embedFs, + conn: conn, + containerdRunPhase: containerdRunPhase, + } + return runtimeConfigPhase, nil +} + +// PhaseInterface is an interface to interact with the phases +func (cp *ContainerdConfigPhase) GetHostPhase() sunpikev1alpha1.HostPhase { + return *cp.hostPhase +} + +func (cp *ContainerdConfigPhase) GetPhaseName() string { + return cp.hostPhase.Name +} + +func (cp *ContainerdConfigPhase) GetOrder() int { + return int(cp.hostPhase.Order) +} + +// This code assumes the containerd version is tied to the nodelet version +// in future version we should break that tie +// Extract the Containerd zip to the specified directory +func (cp *ContainerdConfigPhase) Start(context.Context, config.Config) error { + + // first make sure if the service exists it is stopped + cp.containerdRunPhase.Stop(context.Background(), config.Config{}) + + zap.S().Infof("Extracting containerd zip to %s", cp.baseDir) + err := cp.embedFs.Extract(cp.baseDir) + if err != nil { + return fmt.Errorf("error extracting containerd zip: to baseDir %s, %v", cp.baseDir, err) + } + + // now extract the tar files + zap.S().Infof("Untarring containerd zip to %s", cp.baseDir) + + matches, err := filepath.Glob(path.Join(cp.baseDir, "containerd*.tar.gz")) + if err != nil { + fmt.Errorf("error finding containerd tar files: %v", err) + } + + for _, match := range matches { + zap.S().Infof("Untarring %s", match) + err = untar.Extract(match, cp.baseDir) + if err != nil { + return fmt.Errorf("error untarring containerd tar file: %v", err) + } + } + // now reload the dbus daemon and start the service + + // Reload dbus daemon to load new services + zap.S().Infof("Reloading dbus") + err = cp.conn.Reload() + if err != nil { + return fmt.Errorf("error reloading dbus: %v", err) + } + return nil +} + +func (cp *ContainerdConfigPhase) Stop(context.Context, config.Config) error { + return nil +} + +func (cp *ContainerdConfigPhase) Status(context.Context, config.Config) error { + return nil +} diff --git a/nodelet/pkg/phases/container_runtime/containerd_runtime.go b/nodelet/pkg/phases/container_runtime/containerd_runtime.go new file mode 100644 index 00000000..f2e645fb --- /dev/null +++ b/nodelet/pkg/phases/container_runtime/containerd_runtime.go @@ -0,0 +1,99 @@ +package containerruntime + +import ( + "context" + "fmt" + + + "github.com/coreos/go-systemd/dbus" + "github.com/platform9/nodelet/nodelet/pkg/utils/config" + "github.com/platform9/nodelet/nodelet/pkg/utils/constants" + sunpikev1alpha1 "github.com/platform9/pf9-qbert/sunpike/apiserver/pkg/apis/sunpike/v1alpha1" + "go.uber.org/zap" +) + + +type ContainerdRunPhase struct { + baseDir string + hostPhase *sunpikev1alpha1.HostPhase + conn *dbus.Conn +} + + +// Extract containerd zip to the specified directory +func NewContainerdRunPhase(baseDir string) (*ContainerdRunPhase, error) { + + conn, err:= dbus.NewSystemConnection() + if err != nil { + return nil, fmt.Errorf("error connecting to dbus: %v", err) + } + + return newContainerdRunPhaseInternal(conn, baseDir) +} + +func newContainerdRunPhaseInternal(conn *dbus.Conn, baseDir string) (*ContainerdRunPhase, error) { + + runtimeConfigPhase := &ContainerdRunPhase{ + baseDir: baseDir, + hostPhase: &sunpikev1alpha1.HostPhase{ + Name: "Configure Container Runtime", + Order: int32(constants.ConfigureRuntimePhaseOrder), + }, + conn: conn, + } + return runtimeConfigPhase, nil +} + +// PhaseInterface is an interface to interact with the phases +func (cp *ContainerdRunPhase) GetHostPhase() sunpikev1alpha1.HostPhase { + return *cp.hostPhase +} + +func (cp *ContainerdRunPhase) GetPhaseName() string { + return cp.hostPhase.Name +} + +func (cp *ContainerdRunPhase) GetOrder() int { + return int(cp.hostPhase.Order) +} + +// This code assumes the containerd version is tied to the nodelet version +// in future version we should break that tie +// Extract the Containerd zip to the specified directory +func (cp *ContainerdRunPhase) Start(context.Context, config.Config) error { + zap.S().Infof("Starting containerd") + + // Start the containerd service + jobId, err := cp.conn.StartUnit("containerd.service", "replace", nil) + if err != nil { + return fmt.Errorf("error starting containerd: %v", err) + } + zap.S().Infof("Started containerd with job id %s", jobId) + return nil +} + +func (cp *ContainerdRunPhase) Stop(context.Context, config.Config) error { + // Stop the containerd service + zap.S().Infof("Stopping containerd") + _, err := cp.conn.StopUnit("containerd.service", "replace", nil) + if err != nil { + return fmt.Errorf("error stopping containerd: %v", err) + } + zap.S().Infof("Stopped containerd") + return nil +} + +func (cp *ContainerdRunPhase) Status(context.Context, config.Config) error { + // Get the containerd service status + zap.S().Infof("Getting containerd status") + unitStatuses, err := cp.conn.ListUnitsByNames([]string{"containerd.service"}) + if err != nil { + return fmt.Errorf("error getting containerd status: %v", err) + } + if len(unitStatuses) == 0 { + return fmt.Errorf("containerd service not found") + } + zap.S().Infof("containerd service status: %s", unitStatuses[0].ActiveState) + // check the actual state of the service + return nil +} diff --git a/nodelet/pkg/untar/untar.go b/nodelet/pkg/untar/untar.go new file mode 100644 index 00000000..4f918a6e --- /dev/null +++ b/nodelet/pkg/untar/untar.go @@ -0,0 +1,69 @@ +package untar + +import ( + "archive/tar" + "compress/gzip" + "fmt" + "io" + "os" + "path/filepath" +) + +// extract the targz file to the destination directory +func Extract(tgzFile string, destDir string) error { + + f, err := os.Open(tgzFile) + if err != nil { + return fmt.Errorf("error opening tgz file: %s %v", tgzFile, err) + } + defer f.Close() + + gzf, err := gzip.NewReader(f) + if err != nil { + return fmt.Errorf("error creating gzip reader: %v", err) + } + defer gzf.Close() + tarReader := tar.NewReader(gzf) + + for { + f, err := tarReader.Next() + if err == io.EOF { + break + } + + if err != nil { + return fmt.Errorf("error reading tar file: %v", err) + } + fpath := filepath.Join(destDir, f.Name) + fi := f.FileInfo() + mode := fi.Mode() + switch { + case mode.IsDir(): + err = os.MkdirAll(fpath, 0755) + if err != nil { + return fmt.Errorf("error creating directory: %s %v", fpath, err) + } + case mode.IsRegular(): + // this is redundant + err = os.MkdirAll(filepath.Dir(fpath), 0755) + if err != nil { + return fmt.Errorf("error creating directory: %s %v", filepath.Dir(fpath), err) + } + destFile, err := os.OpenFile(fpath, os.O_CREATE|os.O_RDWR, 0755) + defer destFile.Close() + if err != nil { + return fmt.Errorf("error creating file: %s %v", fpath, err) + } + n, err := io.Copy(destFile, tarReader) + + if err != nil { + return fmt.Errorf("error copying file: %s %v", fpath, err) + } + if n != f.Size { + return fmt.Errorf("error copying file size written not equal: expected %d, got %d, %s %v", f.Size, n, fpath, err) + } + + } + } + return nil +} diff --git a/nodelet/vendor/modules.txt b/nodelet/vendor/modules.txt index bec26a3d..19e4837f 100644 --- a/nodelet/vendor/modules.txt +++ b/nodelet/vendor/modules.txt @@ -125,6 +125,9 @@ github.com/containerd/ttrpc # github.com/containerd/typeurl v1.0.2 ## explicit; go 1.13 github.com/containerd/typeurl +# github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e +## explicit +github.com/coreos/go-systemd/dbus # github.com/davecgh/go-spew v1.1.1 ## explicit github.com/davecgh/go-spew/spew @@ -165,6 +168,9 @@ github.com/go-openapi/jsonreference # github.com/go-openapi/swag v0.19.14 ## explicit; go 1.11 github.com/go-openapi/swag +# github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e +## explicit; go 1.12 +github.com/godbus/dbus # github.com/gogo/googleapis v1.4.0 ## explicit; go 1.12 github.com/gogo/googleapis/google/rpc