Standalone API server with integration tests

Makefile

@@ -18,6 +18,9 @@ CMDS := controller apiserver pilot-elasticsearch pilot-cassandra
 GOPATH ?= /tmp/go
 GOFLAGS ?= "-a"
 
+# Path of the Navigator API server for use in integration tests
+TEST_ASSET_NAVIGATOR_APISERVER ?= "${CURDIR}/navigator-apiserver_linux_amd64"
+
 help:
 	# all       - runs verify, build and docker_build targets
 	# test      - runs go_test target
@@ -44,7 +47,7 @@ build: $(CMDS)
 
 generate: .generate_files
 
-verify: .hack_verify dep_verify go_verify helm_verify
+verify: .hack_verify dep_verify go_verify helm_verify test_integration
 
 .hack_verify:
 	@echo Running repo-infra verify scripts
@@ -90,7 +93,7 @@ $(CMDS):
 go_build: $(CMDS)
 
 go_test:
-	go test -v $$(go list ./... | grep -v '/vendor/')
+	go test -v $$(go list ./... | grep -v -e '/vendor/'  -e 'github.com/jetstack/navigator/test/')
 
 go_fmt:
 	./hack/verify-lint.sh
@@ -105,3 +108,12 @@ go_fmt:
 # Helm targets
 helm_verify:
 	helm lint contrib/charts/*
+
+.download_integration_test_binaries:
+	mkdir -p vendor/sigs.k8s.io/testing_frameworks/integration/assets/bin
+	DEBUG=1 vendor/sigs.k8s.io/testing_frameworks/integration/scripts/download-binaries.sh
+	touch	.download_integration_test_binaries
+
+test_integration: .download_integration_test_binaries apiserver
+	TEST_ASSET_NAVIGATOR_APISERVER=$(TEST_ASSET_NAVIGATOR_APISERVER) \
+	go test -v ./test/integration/...

cmd/apiserver/start.go

@@ -130,20 +130,47 @@ func (o NavigatorServerOptions) Config() (*apiserver.Config, error) {
 		return nil, fmt.Errorf("error creating self-signed certificates: %v", err)
 	}
 
-	serverConfig := genericapiserver.NewRecommendedConfig(apiserver.Codecs)
-	if err := o.RecommendedOptions.ApplyTo(serverConfig); err != nil {
-		return nil, err
+	config := &apiserver.Config{
+		GenericConfig: genericapiserver.NewRecommendedConfig(apiserver.Codecs),
 	}
 
-	client, err := clientset.NewForConfig(serverConfig.LoopbackClientConfig)
-	if err != nil {
+	// Do not call RecommendedOptions.ApplyTo because some of its sub-options
+	// require kube-apiserver configuration options
+	if err := o.RecommendedOptions.Etcd.ApplyTo(&config.GenericConfig.Config); err != nil {
+		return nil, err
+	}
+	if err := o.RecommendedOptions.SecureServing.ApplyTo(&config.GenericConfig.Config); err != nil {
+		return nil, err
+	}
+	if err := o.RecommendedOptions.Audit.ApplyTo(&config.GenericConfig.Config); err != nil {
+		return nil, err
+	}
+	if err := o.RecommendedOptions.Features.ApplyTo(&config.GenericConfig.Config); err != nil {
 		return nil, err
 	}
-	sharedInformers := informers.NewSharedInformerFactory(client, serverConfig.LoopbackClientConfig.Timeout)
 
 	// only enable admission control when running in-cluster as we require a
 	// kubernetes client
 	if !o.StandaloneMode {
+		// These RecommendedOptions require kube-apiserver configuration and
+		// won't work in a standalone API server
+		if err := o.RecommendedOptions.Authentication.ApplyTo(&config.GenericConfig.Config); err != nil {
+			return nil, err
+		}
+		if err := o.RecommendedOptions.Authorization.ApplyTo(&config.GenericConfig.Config); err != nil {
+			return nil, err
+		}
+		if err := o.RecommendedOptions.CoreAPI.ApplyTo(config.GenericConfig); err != nil {
+			return nil, err
+		}
+
+		client, err := clientset.NewForConfig(config.GenericConfig.LoopbackClientConfig)
+		if err != nil {
+			return nil, err
+		}
+		sharedInformers := informers.NewSharedInformerFactory(client, config.GenericConfig.LoopbackClientConfig.Timeout)
+		config.SharedInformerFactory = sharedInformers
+
 		inClusterConfig, err := restclient.InClusterConfig()
 		if err != nil {
 			glog.Errorf("Failed to get kube client config: %v", err)
@@ -158,18 +185,14 @@ func (o NavigatorServerOptions) Config() (*apiserver.Config, error) {
 		}
 
 		kubeSharedInformers := kubeinformers.NewSharedInformerFactory(kubeClient, 10*time.Minute)
-		serverConfig.SharedInformerFactory = kubeSharedInformers
+		config.GenericConfig.SharedInformerFactory = kubeSharedInformers
 
-		serverConfig.AdmissionControl, err = buildAdmission(&o, client, sharedInformers, kubeClient, kubeSharedInformers)
+		config.GenericConfig.AdmissionControl, err = buildAdmission(&o, client, sharedInformers, kubeClient, kubeSharedInformers)
 		if err != nil {
 			return nil, fmt.Errorf("failed to initialize admission: %v", err)
 		}
 	}
 
-	config := &apiserver.Config{
-		GenericConfig:         serverConfig,
-		SharedInformerFactory: sharedInformers,
-	}
 	return config, nil
 }
 
@@ -201,10 +224,12 @@ func (o NavigatorServerOptions) RunNavigatorServer(stopCh <-chan struct{}) error
 		return err
 	}
 
-	server.GenericAPIServer.AddPostStartHook("start-navigator-server-informers", func(context genericapiserver.PostStartHookContext) error {
-		config.SharedInformerFactory.Start(context.StopCh)
-		return nil
-	})
+	if !o.StandaloneMode {
+		server.GenericAPIServer.AddPostStartHook("start-navigator-server-informers", func(context genericapiserver.PostStartHookContext) error {
+			config.SharedInformerFactory.Start(context.StopCh)
+			return nil
+		})
+	}
 
 	return server.GenericAPIServer.PrepareRun().Run(stopCh)
 }

internal/test/integration/framework/framework.go

@@ -0,0 +1,132 @@
+package framework
+
+import (
+	"fmt"
+	"io"
+	"net/url"
+	"path/filepath"
+	"time"
+
+	"sigs.k8s.io/testing_frameworks/integration"
+
+	"github.com/jetstack/navigator/internal/test/integration/framework/internal"
+)
+
+type NavigatorAPIServer struct {
+	URL          *url.URL
+	Path         string
+	Args         []string
+	StartTimeout time.Duration
+	StopTimeout  time.Duration
+	CertDir      string
+	EtcdURL      *url.URL
+	APIServerURL *url.URL
+	Out          io.Writer
+	Err          io.Writer
+	processState *internal.ProcessState
+}
+
+func (s *NavigatorAPIServer) Start() error {
+	if s.EtcdURL == nil {
+		return fmt.Errorf("expected EtcdURL to be configured")
+	}
+
+	var err error
+
+	s.processState = &internal.ProcessState{}
+
+	s.processState.DefaultedProcessInput, err = internal.DoDefaulting(
+		"navigator-apiserver",
+		s.URL,
+		s.CertDir,
+		s.Path,
+		s.StartTimeout,
+		s.StopTimeout,
+	)
+	if err != nil {
+		return err
+	}
+
+	// s.processState.HealthCheckEndpoint = "/healthz"
+	s.processState.StartMessage = "Serving securely on 127.0.0.1"
+	s.URL = &s.processState.URL
+	s.CertDir = s.processState.Dir
+	s.Path = s.processState.Path
+	s.StartTimeout = s.processState.StartTimeout
+	s.StopTimeout = s.processState.StopTimeout
+
+	s.processState.Args, err = internal.RenderTemplates(
+		append(
+			internal.DoAPIServerArgDefaulting(nil),
+			s.Args...,
+		),
+		s,
+	)
+	if err != nil {
+		return err
+	}
+	return s.processState.Start(s.Out, s.Err)
+}
+
+func (s *NavigatorAPIServer) Stop() error {
+	return s.processState.Stop()
+}
+
+type NavigatorControlPlane struct {
+	*integration.ControlPlane
+	NavigatorAPIServer *NavigatorAPIServer
+}
+
+func (f *NavigatorControlPlane) Start() error {
+	if f.ControlPlane == nil {
+		f.ControlPlane = &integration.ControlPlane{}
+	}
+	err := f.ControlPlane.Start()
+	if err != nil {
+		return err
+	}
+	if f.NavigatorAPIServer == nil {
+		f.NavigatorAPIServer = &NavigatorAPIServer{}
+	}
+	f.NavigatorAPIServer.EtcdURL = f.Etcd.URL
+	f.NavigatorAPIServer.APIServerURL = f.APIServer.URL
+	return f.NavigatorAPIServer.Start()
+}
+
+func (f *NavigatorControlPlane) Stop() error {
+	if f.NavigatorAPIServer != nil {
+		err := f.NavigatorAPIServer.Stop()
+		if err != nil {
+			return err
+		}
+	}
+	return f.ControlPlane.Stop()
+}
+
+func (f *NavigatorControlPlane) NavigatorAPIURL() *url.URL {
+	return &url.URL{
+		Scheme: "https",
+		Host:   f.NavigatorAPIServer.URL.Host,
+	}
+}
+
+func (f NavigatorControlPlane) NavigatorCtl() *NavigatorCtl {
+	return &NavigatorCtl{
+		&integration.KubeCtl{
+			Opts: []string{
+				"--server",
+				f.NavigatorAPIURL().String(),
+				"--client-certificate",
+				filepath.Join(f.NavigatorAPIServer.CertDir, "apiserver.crt"),
+				"--client-key",
+				filepath.Join(f.NavigatorAPIServer.CertDir, "apiserver.key"),
+				"--certificate-authority",
+				filepath.Join(f.NavigatorAPIServer.CertDir, "apiserver.crt"),
+			},
+		},
+	}
+}
+
+type NavigatorCtl struct {
+	*integration.KubeCtl
+}

internal/test/integration/framework/internal/address_manager.go

@@ -0,0 +1,53 @@
+package internal
+
+import (
+	"fmt"
+	"net"
+)
+
+// AddressManager allocates a new address (interface & port) a process
+// can bind and keeps track of that.
+type AddressManager struct {
+	port int
+	host string
+}
+
+// Initialize returns a address a process can listen on. It returns
+// a tuple consisting of a free port and the hostname resolved to its IP.
+func (d *AddressManager) Initialize() (port int, resolvedHost string, err error) {
+	if d.port != 0 {
+		return 0, "", fmt.Errorf("this AddressManager is already initialized")
+	}
+	addr, err := net.ResolveTCPAddr("tcp", "localhost:0")
+	if err != nil {
+		return
+	}
+	l, err := net.ListenTCP("tcp", addr)
+	if err != nil {
+		return
+	}
+	d.port = l.Addr().(*net.TCPAddr).Port
+	defer func() {
+		err = l.Close()
+	}()
+	d.host = addr.IP.String()
+	return d.port, d.host, nil
+}
+
+// Port returns the port that this AddressManager is managing. Port returns an
+// error if this AddressManager has not yet been initialized.
+func (d *AddressManager) Port() (int, error) {
+	if d.port == 0 {
+		return 0, fmt.Errorf("this AdressManager is not initialized yet")
+	}
+	return d.port, nil
+}
+
+// Host returns the host that this AddressManager is managing. Host returns an
+// error if this AddressManager has not yet been initialized.
+func (d *AddressManager) Host() (string, error) {
+	if d.host == "" {
+		return "", fmt.Errorf("this AdressManager is not initialized yet")
+	}
+	return d.host, nil
+}