Metrics

.github/workflows/on-pull-request.yaml

@@ -16,23 +16,23 @@ jobs:
     strategy:
       matrix:
         go-version:
-          - 1.18.x
-          - 1.19.x
+          - 1.21.x
+          - 1.22.x
         os: [ ubuntu-latest ]
     runs-on: ${{ matrix.os }}
     steps:
       - name: Checkout
-        uses: actions/checkout@master
+        uses: actions/checkout@v4
 
       - name: Set up Go
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v5
         with:
           go-version: ${{ matrix.go-version }}
 
       - run: go env
 
       - name: Cache deps
-        uses: actions/cache@v2
+        uses: actions/cache@v4
         with:
           path: ~/go/pkg/mod
           key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
@@ -43,4 +43,4 @@ jobs:
         run: go mod download
 
       - name: Test
-        run: go test ./...
+        run: go test ./...

example_test.go

@@ -9,7 +9,7 @@ import (
 	"github.com/mailgun/groupcache/v2"
 )
 
-func ExampleUsage() {
+func ExampleNewGroup() {
 	/*
 		// Keep track of peers in our cluster and add our instance to the pool `http://localhost:8080`
 		pool := groupcache.NewHTTPPoolOpts("http://localhost:8080", &groupcache.HTTPPoolOptions{})

go.mod

@@ -1,19 +1,26 @@
 module github.com/mailgun/groupcache/v2
 
-go 1.19
+go 1.21
 
 require (
 	github.com/golang/protobuf v1.5.2
+	github.com/prometheus/client_golang v1.20.5
 	github.com/segmentio/fasthash v1.0.3
 	github.com/sirupsen/logrus v1.9.0
-	github.com/stretchr/testify v1.8.1
+	github.com/stretchr/testify v1.9.0
 )
 
 require (
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/kr/text v0.2.0 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8 // indirect
-	golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect
-	google.golang.org/protobuf v1.28.1 // indirect
+	github.com/prometheus/client_model v0.6.1 // indirect
+	github.com/prometheus/common v0.55.0 // indirect
+	github.com/prometheus/procfs v0.15.1 // indirect
+	golang.org/x/sys v0.22.0 // indirect
+	google.golang.org/protobuf v1.34.2 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -1,37 +1,56 @@
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
 github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
 github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
-github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
+github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y=
+github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=
+github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
+github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
+github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc=
+github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8=
+github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
+github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
+github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
+github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
 github.com/segmentio/fasthash v1.0.3 h1:EI9+KE1EwvMLBWwjpRDc+fEM+prwxDYbslddQGtrmhM=
 github.com/segmentio/fasthash v1.0.3/go.mod h1:waKX8l2N8yckOgmSsXJi7x1ZfdKZ4x7KRMzBtS3oedY=
 github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
 github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
-github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
-github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
-github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8 h1:h+EGohizhe9XlX18rfpa8k8RAc5XyaeamM+0VHRd4lc=
-golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
+golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f h1:uF6paiQQebLeSXkrTqHqz0MXhXXS1KgF41eUdBNvxK0=
-golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=
-google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
+google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

groupcache.go

@@ -35,6 +35,7 @@ import (
 	pb "github.com/mailgun/groupcache/v2/groupcachepb"
 	"github.com/mailgun/groupcache/v2/lru"
 	"github.com/mailgun/groupcache/v2/singleflight"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/sirupsen/logrus"
 )
 
@@ -117,13 +118,14 @@ func newGroup(name string, cacheBytes int64, getter Getter, peers PeerPicker) *G
 		panic("duplicate registration of group " + name)
 	}
 	g := &Group{
-		name:        name,
-		getter:      getter,
-		peers:       peers,
-		cacheBytes:  cacheBytes,
-		loadGroup:   &singleflight.Group{},
-		setGroup:    &singleflight.Group{},
-		removeGroup: &singleflight.Group{},
+		name:                     name,
+		getter:                   getter,
+		peers:                    peers,
+		cacheBytes:               cacheBytes,
+		loadGroup:                &singleflight.Group{},
+		setGroup:                 &singleflight.Group{},
+		removeGroup:              &singleflight.Group{},
+		metricGetFromPeerLatency: metricGetFromPeerLatency.WithLabelValues(name),
 	}
 	if fn := newGroupHook; fn != nil {
 		fn(g)
@@ -201,6 +203,8 @@ type Group struct {
 
 	// Stats are statistics on the group.
 	Stats Stats
+
+	metricGetFromPeerLatency prometheus.Observer
 }
 
 // flightGroup is defined as an interface which flightgroup.Group
@@ -236,7 +240,7 @@ func (g *Group) initPeers() {
 	}
 }
 
-func (g *Group) Get(ctx context.Context, key string, dest Sink) error {
+func (g *Group) Get(ctx context.Context, key string, dest Sink) (err error) {
 	g.peersOnce.Do(g.initPeers)
 	g.Stats.Gets.Add(1)
 	if dest == nil {
@@ -254,7 +258,7 @@ func (g *Group) Get(ctx context.Context, key string, dest Sink) error {
 	// (if local) will set this; the losers will not. The common
 	// case will likely be one caller.
 	destPopulated := false
-	value, destPopulated, err := g.load(ctx, key, dest)
+	value, destPopulated, err = g.load(ctx, key, dest)
 	if err != nil {
 		return err
 	}
@@ -264,20 +268,22 @@ func (g *Group) Get(ctx context.Context, key string, dest Sink) error {
 	return setSinkView(dest, value)
 }
 
-func (g *Group) Set(ctx context.Context, key string, value []byte, expire time.Time, hotCache bool) error {
+func (g *Group) Set(ctx context.Context, key string, value []byte, expire time.Time, hotCache bool) (err error) {
 	g.peersOnce.Do(g.initPeers)
 
 	if key == "" {
 		return errors.New("empty Set() key not allowed")
 	}
 
-	_, err := g.setGroup.Do(key, func() (interface{}, error) {
+	_, err = g.setGroup.Do(key, func() (interface{}, error) {
 		// If remote peer owns this key
 		owner, ok := g.peers.PickPeer(key)
 		if ok {
+			timer := prometheus.NewTimer(metricUpdatePeerLatency.WithLabelValues(g.name, owner.GetURL()))
 			if err := g.setFromPeer(ctx, owner, key, value, expire); err != nil {
 				return nil, err
 			}
+			timer.ObserveDuration()
 			// TODO(thrawn01): Not sure if this is useful outside of tests...
 			//  maybe we should ALWAYS update the local cache?
 			if hotCache {
@@ -294,10 +300,10 @@ func (g *Group) Set(ctx context.Context, key string, value []byte, expire time.T
 
 // Remove clears the key from our cache then forwards the remove
 // request to all peers.
-func (g *Group) Remove(ctx context.Context, key string) error {
+func (g *Group) Remove(ctx context.Context, key string) (err error) {
 	g.peersOnce.Do(g.initPeers)
 
-	_, err := g.removeGroup.Do(key, func() (interface{}, error) {
+	_, err = g.removeGroup.Do(key, func() (interface{}, error) {
 
 		// Remove from key owner first
 		owner, ok := g.peers.PickPeer(key)
@@ -382,12 +388,14 @@ func (g *Group) load(ctx context.Context, key string, dest Sink) (value ByteView
 			value, err = g.getFromPeer(ctx, peer, key)
 
 			// metrics duration compute
-			duration := int64(time.Since(start)) / int64(time.Millisecond)
+			duration := time.Since(start)
+			durationMs := int64(duration / time.Millisecond)
 
 			// metrics only store the slowest duration
-			if g.Stats.GetFromPeersLatencyLower.Get() < duration {
-				g.Stats.GetFromPeersLatencyLower.Store(duration)
+			if g.Stats.GetFromPeersLatencyLower.Get() < durationMs {
+				g.Stats.GetFromPeersLatencyLower.Store(durationMs)
 			}
+			g.metricGetFromPeerLatency.Observe(duration.Seconds())
 
 			if err == nil {
 				g.Stats.PeerLoads.Add(1)
@@ -564,6 +572,17 @@ func (g *Group) populateCache(key string, value ByteView, cache *cache) {
 // CacheType represents a type of cache.
 type CacheType int
 
+func (t CacheType) String() string {
+	switch t {
+	case MainCache:
+		return "main"
+	case HotCache:
+		return "hot"
+	default:
+		return ""
+	}
+}
+
 const (
 	// The MainCache is the cache for items that this peer is the
 	// owner for.
@@ -587,6 +606,15 @@ func (g *Group) CacheStats(which CacheType) CacheStats {
 	}
 }
 
+// GetMetrics returns Prometheus metrics.
+func (g *Group) GetMetrics() []prometheus.Collector {
+	return []prometheus.Collector{
+		&CacheStatsCollector{group: g},
+		metricGetFromPeerLatency,
+		metricUpdatePeerLatency,
+	}
+}
+
 // NowFunc returns the current time which is used by the LRU to
 // determine if the value has expired. This can be overridden by
 // tests to ensure items are evicted when expired.
@@ -713,3 +741,78 @@ type CacheStats struct {
 	Hits      int64
 	Evictions int64
 }
+
+var (
+	statsBytesDesc = prometheus.NewDesc(
+		"groupcache_stats_bytes",
+		"The number of bytes stored in cache",
+		[]string{"group", "type"}, nil,
+	)
+	statsItemsDesc = prometheus.NewDesc(
+		"groupcache_stats_items",
+		"The number of items stored in cache",
+		[]string{"group", "type"}, nil,
+	)
+	statsGetsDesc = prometheus.NewDesc(
+		"groupcache_stats_gets",
+		"The number of get requests",
+		[]string{"group", "type"}, nil,
+	)
+	statsHitsDesc = prometheus.NewDesc(
+		"groupcache_stats_hits",
+		"The number of cache hits",
+		[]string{"group", "type"}, nil,
+	)
+	statsEvictionsDesc = prometheus.NewDesc(
+		"groupcache_stats_evictions",
+		"The number of cache evictions",
+		[]string{"group", "type"}, nil,
+	)
+)
+
+// CacheStats exposed as Prometheus metrics.
+type CacheStatsCollector struct {
+	group *Group
+}
+
+func (c *CacheStatsCollector) Describe(ch chan<- *prometheus.Desc) {
+	prometheus.DescribeByCollect(c, ch)
+}
+
+func (c *CacheStatsCollector) Collect(ch chan<- prometheus.Metric) {
+	types := []CacheType{MainCache, HotCache}
+	for _, t := range types {
+		stats := c.group.CacheStats(t)
+		tstr := t.String()
+		ch <- prometheus.MustNewConstMetric(
+			statsBytesDesc,
+			prometheus.GaugeValue,
+			float64(stats.Bytes),
+			c.group.name, tstr,
+		)
+		ch <- prometheus.MustNewConstMetric(
+			statsItemsDesc,
+			prometheus.GaugeValue,
+			float64(stats.Items),
+			c.group.name, tstr,
+		)
+		ch <- prometheus.MustNewConstMetric(
+			statsGetsDesc,
+			prometheus.CounterValue,
+			float64(stats.Gets),
+			c.group.name, tstr,
+		)
+		ch <- prometheus.MustNewConstMetric(
+			statsHitsDesc,
+			prometheus.CounterValue,
+			float64(stats.Hits),
+			c.group.name, tstr,
+		)
+		ch <- prometheus.MustNewConstMetric(
+			statsEvictionsDesc,
+			prometheus.CounterValue,
+			float64(stats.Evictions),
+			c.group.name, tstr,
+		)
+	}
+}

metrics.go

@@ -0,0 +1,23 @@
+package groupcache
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+var (
+	SummaryObjectives = map[float64]float64{
+		0.5:  0.05,
+		0.99: 0.001,
+		1:    0.001,
+	}
+	metricGetFromPeerLatency = prometheus.NewSummaryVec(prometheus.SummaryOpts{
+		Name:       "groupcache_get_from_peer_latency",
+		Help:       "The latency in seconds getting value from remote peer",
+		Objectives: SummaryObjectives,
+	}, []string{"group"})
+	metricUpdatePeerLatency = prometheus.NewSummaryVec(prometheus.SummaryOpts{
+		Name:       "groupcache_update_peer_latency",
+		Help:       "The latency in seconds updating a remote peer during a Set",
+		Objectives: SummaryObjectives,
+	}, []string{"group", "peer"})
+)