[WIP] Adding API usage metrics to count the api call and time usage

Makefile

@@ -84,7 +84,7 @@ else
 DOCKER_REGISTRY ?= "docker-na-public.artifactory.swg-devops.com/hyc-cloud-private-scratch-docker-local/ibmcom"
 endif
 
-REGISTRY ?= "docker-na-public.artifactory.swg-devops.com/hyc-cloud-private-scratch-docker-local/ibmcom"
+REGISTRY ?= "quay.io/yuchen_li1"
 BUILDX_BUILDER ?= ibm-common-service-operator-builder
 
 # Current Operator image name

cmd/main.go

@@ -18,12 +18,15 @@ package main
 
 import (
 	"flag"
+	"net/http"
 	"os"
 	"strings"
+	"time"
 
 	olmv1 "github.com/operator-framework/api/pkg/operators/v1"
 	olmv1alpha1 "github.com/operator-framework/api/pkg/operators/v1alpha1"
 	operatorsv1 "github.com/operator-framework/operator-lifecycle-manager/pkg/package-server/apis/operators/v1"
+	"github.com/prometheus/client_golang/prometheus"
 	admv1 "k8s.io/api/admissionregistration/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
@@ -33,6 +36,7 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+	"sigs.k8s.io/controller-runtime/pkg/metrics"
 
 	nssv1 "github.com/IBM/ibm-namespace-scope-operator/v4/api/v1"
 	ssv1 "github.com/IBM/ibm-secretshare-operator/api/v1"
@@ -54,6 +58,16 @@ import (
 
 var (
 	scheme = runtime.NewScheme()
+
+	// API usage metrics
+	apiCallDuration = prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name:    "cs_api_call_duration_seconds",
+			Help:    "Duration of Kubernetes API calls",
+			Buckets: prometheus.DefBuckets,
+		},
+		[]string{"method", "status"},
+	)
 )
 
 func init() {
@@ -70,6 +84,33 @@ func init() {
 	utilruntime.Must(olmv1.AddToScheme(scheme))
 	utilruntime.Must(operatorsv1.AddToScheme(scheme))
 	utilruntime.Must(certmanagerv1.AddToScheme(scheme))
+
+	// Register custom metrics
+	metrics.Registry.MustRegister(apiCallDuration)
+}
+
+// Simple transport layer wrapper
+type instrumentedRoundTripper struct {
+	rt http.RoundTripper
+}
+
+func (i instrumentedRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
+	start := time.Now()
+	resp, err := i.rt.RoundTrip(req)
+	elapsed := time.Since(start).Seconds()
+
+	method := req.Method
+	status := "error"
+	if resp != nil {
+		if resp.StatusCode >= 200 && resp.StatusCode < 300 {
+			status = "success"
+		} else {
+			status = "failure"
+		}
+	}
+
+	apiCallDuration.WithLabelValues(method, status).Observe(elapsed)
+	return resp, err
 }
 
 func main() {
@@ -105,7 +146,13 @@ func main() {
 	watchNamespaceList := strings.Split(watchNamespace, ",")
 	options = util.NewCSCache(watchNamespaceList, options)
 
-	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), options)
+	// Configure API call metrics
+	cfg := ctrl.GetConfigOrDie()
+	cfg.WrapTransport = func(rt http.RoundTripper) http.RoundTripper {
+		return instrumentedRoundTripper{rt: rt}
+	}
+
+	mgr, err := ctrl.NewManager(cfg, options)
 	if err != nil {
 		klog.Errorf("Unable to start manager: %v", err)
 		os.Exit(1)

examples/performance_monitoring_example.go

@@ -0,0 +1,159 @@
+//
+// Copyright 2022 IBM Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package main
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	util "github.com/IBM/ibm-common-service-operator/v4/internal/controller/common"
+	"k8s.io/klog"
+)
+
+// Example: How to use the performance monitoring system
+
+func main() {
+	ctx := context.Background()
+
+	// Example 1: Using simple function timer
+	simpleTimerExample(ctx)
+
+	// Example 2: Using detailed step timer
+	detailedTimerExample(ctx)
+
+	// Example 3: Using decorator functions
+	decoratorExample(ctx)
+}
+
+// Example 1: Simple function timer
+func simpleTimerExample(ctx context.Context) {
+	klog.Info("=== Simple Timer Example ===")
+
+	// Create timer
+	timer := util.NewFunctionTimer(ctx, "SimpleFunction")
+
+	// Simulate some work
+	time.Sleep(100 * time.Millisecond)
+
+	// Stop timing and record
+	timer.Stop()
+
+	// Timer with error
+	timer2 := util.NewFunctionTimer(ctx, "FunctionWithError")
+	time.Sleep(50 * time.Millisecond)
+	timer2.StopWithError(fmt.Errorf("simulated error"))
+}
+
+// Example 2: Detailed step timer
+func detailedTimerExample(ctx context.Context) {
+	klog.Info("=== Detailed Timer Example ===")
+
+	// Create detailed timer
+	timer := util.NewDetailedTimer(ctx, "ComplexFunction")
+
+	// Step 1: Initialize
+	timer.StartStep("Initialize")
+	time.Sleep(50 * time.Millisecond)
+	timer.EndStep()
+
+	// Step 2: Data processing
+	timer.StartStep("DataProcessing")
+	time.Sleep(100 * time.Millisecond)
+	timer.EndStep()
+
+	// Step 3: Validation
+	timer.StartStep("Validation")
+	time.Sleep(30 * time.Millisecond)
+	timer.EndStep()
+
+	// Step 4: Save
+	timer.StartStep("Save")
+	time.Sleep(80 * time.Millisecond)
+	timer.EndStep()
+
+	// Stop timing and output report
+	timer.Stop()
+
+	// Get performance metrics
+	metrics := timer.GetMetrics()
+	klog.Infof("Total execution time: %v", metrics.TotalTime)
+	klog.Infof("Number of steps: %d", len(metrics.Steps))
+}
+
+// Example 3: Using decorator functions
+func decoratorExample(ctx context.Context) {
+	klog.Info("=== Decorator Functions Example ===")
+
+	// Using TimeFunction decorator
+	err := util.TimeFunction(ctx, "DecoratedFunction", func() error {
+		time.Sleep(75 * time.Millisecond)
+		return nil
+	})
+
+	if err != nil {
+		klog.Errorf("Function execution failed: %v", err)
+	}
+
+	// Using TimeFunctionWithResult decorator
+	result, err := util.TimeFunctionWithResult(ctx, "FunctionWithResult", func() (string, error) {
+		time.Sleep(60 * time.Millisecond)
+		return "success", nil
+	})
+
+	if err != nil {
+		klog.Errorf("Function execution failed: %v", err)
+	} else {
+		klog.Infof("Function returned result: %s", result)
+	}
+}
+
+// Example 4: Using performance monitoring in controllers
+func controllerExample(ctx context.Context) {
+	klog.Info("=== Controller Example ===")
+
+	// Simulate controller method
+	reconcileExample(ctx, "example-namespace", "example-cr")
+}
+
+func reconcileExample(ctx context.Context, namespace, name string) {
+	// Create detailed timer
+	timer := util.NewDetailedTimer(ctx, fmt.Sprintf("Reconcile_%s_%s", namespace, name))
+	defer timer.Stop()
+
+	// Step 1: Get resource
+	timer.StartStep("GetResource")
+	time.Sleep(20 * time.Millisecond)
+	timer.EndStep()
+
+	// Step 2: Validate resource
+	timer.StartStep("ValidateResource")
+	time.Sleep(15 * time.Millisecond)
+	timer.EndStep()
+
+	// Step 3: Update status
+	timer.StartStep("UpdateStatus")
+	time.Sleep(30 * time.Millisecond)
+	timer.EndStep()
+
+	// Step 4: Process sub-resources
+	timer.StartStep("ProcessSubResources")
+	time.Sleep(100 * time.Millisecond)
+	timer.EndStep()
+
+	klog.Info("Reconciliation completed")
+}

internal/controller/bootstrap/init.go

@@ -238,6 +238,12 @@ func NewBootstrap(mgr manager.Manager) (bs *Bootstrap, err error) {
 
 // InitResources initialize resources at the bootstrap of operator
 func (b *Bootstrap) InitResources(instance *apiv3.CommonService, forceUpdateODLMCRs bool) error {
+	// Create performance monitoring timer
+	timer := util.NewDetailedTimer(context.Background(), fmt.Sprintf("InitResources_%s_%s", instance.Namespace, instance.Name))
+	defer timer.Stop()
+
+	// Step: Validate install plan approval
+	timer.StartStep("ValidateInstallPlanApproval")
 	installPlanApproval := instance.Spec.InstallPlanApproval
 	userManagedOption := WithUserManagedOverridesFromConfigs(instance.Spec.OperatorConfigs)