diff --git a/.github/workflows/on-pull-request.yaml b/.github/workflows/on-pull-request.yaml index 756c893..43ef83e 100644 --- a/.github/workflows/on-pull-request.yaml +++ b/.github/workflows/on-pull-request.yaml @@ -16,23 +16,23 @@ jobs: strategy: matrix: go-version: - - 1.18.x - - 1.19.x + - 1.21.x + - 1.22.x os: [ ubuntu-latest ] runs-on: ${{ matrix.os }} steps: - name: Checkout - uses: actions/checkout@master + uses: actions/checkout@v4 - name: Set up Go - uses: actions/setup-go@v2 + uses: actions/setup-go@v5 with: go-version: ${{ matrix.go-version }} - run: go env - name: Cache deps - uses: actions/cache@v2 + uses: actions/cache@v4 with: path: ~/go/pkg/mod key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} @@ -43,4 +43,4 @@ jobs: run: go mod download - name: Test - run: go test ./... \ No newline at end of file + run: go test ./... diff --git a/example_test.go b/example_test.go index 9fd306e..6f6ca97 100644 --- a/example_test.go +++ b/example_test.go @@ -9,7 +9,7 @@ import ( "github.com/mailgun/groupcache/v2" ) -func ExampleUsage() { +func ExampleNewGroup() { /* // Keep track of peers in our cluster and add our instance to the pool `http://localhost:8080` pool := groupcache.NewHTTPPoolOpts("http://localhost:8080", &groupcache.HTTPPoolOptions{}) diff --git a/go.mod b/go.mod index 72812e3..523ee80 100644 --- a/go.mod +++ b/go.mod @@ -1,19 +1,26 @@ module github.com/mailgun/groupcache/v2 -go 1.19 +go 1.21 require ( github.com/golang/protobuf v1.5.2 + github.com/prometheus/client_golang v1.20.5 github.com/segmentio/fasthash v1.0.3 github.com/sirupsen/logrus v1.9.0 - github.com/stretchr/testify v1.8.1 + github.com/stretchr/testify v1.9.0 ) require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8 // indirect - golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect - google.golang.org/protobuf v1.28.1 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.55.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect + golang.org/x/sys v0.22.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index e1d0665..841428b 100644 --- a/go.sum +++ b/go.sum @@ -1,37 +1,56 @@ +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= +github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= +github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/segmentio/fasthash v1.0.3 h1:EI9+KE1EwvMLBWwjpRDc+fEM+prwxDYbslddQGtrmhM= github.com/segmentio/fasthash v1.0.3/go.mod h1:waKX8l2N8yckOgmSsXJi7x1ZfdKZ4x7KRMzBtS3oedY= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8 h1:h+EGohizhe9XlX18rfpa8k8RAc5XyaeamM+0VHRd4lc= -golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f h1:uF6paiQQebLeSXkrTqHqz0MXhXXS1KgF41eUdBNvxK0= -golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/groupcache.go b/groupcache.go index 541e89a..7d7e579 100644 --- a/groupcache.go +++ b/groupcache.go @@ -35,6 +35,7 @@ import ( pb "github.com/mailgun/groupcache/v2/groupcachepb" "github.com/mailgun/groupcache/v2/lru" "github.com/mailgun/groupcache/v2/singleflight" + "github.com/prometheus/client_golang/prometheus" "github.com/sirupsen/logrus" ) @@ -117,13 +118,14 @@ func newGroup(name string, cacheBytes int64, getter Getter, peers PeerPicker) *G panic("duplicate registration of group " + name) } g := &Group{ - name: name, - getter: getter, - peers: peers, - cacheBytes: cacheBytes, - loadGroup: &singleflight.Group{}, - setGroup: &singleflight.Group{}, - removeGroup: &singleflight.Group{}, + name: name, + getter: getter, + peers: peers, + cacheBytes: cacheBytes, + loadGroup: &singleflight.Group{}, + setGroup: &singleflight.Group{}, + removeGroup: &singleflight.Group{}, + metricGetFromPeerLatency: metricGetFromPeerLatency.WithLabelValues(name), } if fn := newGroupHook; fn != nil { fn(g) @@ -201,6 +203,8 @@ type Group struct { // Stats are statistics on the group. Stats Stats + + metricGetFromPeerLatency prometheus.Observer } // flightGroup is defined as an interface which flightgroup.Group @@ -236,7 +240,7 @@ func (g *Group) initPeers() { } } -func (g *Group) Get(ctx context.Context, key string, dest Sink) error { +func (g *Group) Get(ctx context.Context, key string, dest Sink) (err error) { g.peersOnce.Do(g.initPeers) g.Stats.Gets.Add(1) if dest == nil { @@ -254,7 +258,7 @@ func (g *Group) Get(ctx context.Context, key string, dest Sink) error { // (if local) will set this; the losers will not. The common // case will likely be one caller. destPopulated := false - value, destPopulated, err := g.load(ctx, key, dest) + value, destPopulated, err = g.load(ctx, key, dest) if err != nil { return err } @@ -264,20 +268,22 @@ func (g *Group) Get(ctx context.Context, key string, dest Sink) error { return setSinkView(dest, value) } -func (g *Group) Set(ctx context.Context, key string, value []byte, expire time.Time, hotCache bool) error { +func (g *Group) Set(ctx context.Context, key string, value []byte, expire time.Time, hotCache bool) (err error) { g.peersOnce.Do(g.initPeers) if key == "" { return errors.New("empty Set() key not allowed") } - _, err := g.setGroup.Do(key, func() (interface{}, error) { + _, err = g.setGroup.Do(key, func() (interface{}, error) { // If remote peer owns this key owner, ok := g.peers.PickPeer(key) if ok { + timer := prometheus.NewTimer(metricUpdatePeerLatency.WithLabelValues(g.name, owner.GetURL())) if err := g.setFromPeer(ctx, owner, key, value, expire); err != nil { return nil, err } + timer.ObserveDuration() // TODO(thrawn01): Not sure if this is useful outside of tests... // maybe we should ALWAYS update the local cache? if hotCache { @@ -294,10 +300,10 @@ func (g *Group) Set(ctx context.Context, key string, value []byte, expire time.T // Remove clears the key from our cache then forwards the remove // request to all peers. -func (g *Group) Remove(ctx context.Context, key string) error { +func (g *Group) Remove(ctx context.Context, key string) (err error) { g.peersOnce.Do(g.initPeers) - _, err := g.removeGroup.Do(key, func() (interface{}, error) { + _, err = g.removeGroup.Do(key, func() (interface{}, error) { // Remove from key owner first owner, ok := g.peers.PickPeer(key) @@ -382,12 +388,14 @@ func (g *Group) load(ctx context.Context, key string, dest Sink) (value ByteView value, err = g.getFromPeer(ctx, peer, key) // metrics duration compute - duration := int64(time.Since(start)) / int64(time.Millisecond) + duration := time.Since(start) + durationMs := int64(duration / time.Millisecond) // metrics only store the slowest duration - if g.Stats.GetFromPeersLatencyLower.Get() < duration { - g.Stats.GetFromPeersLatencyLower.Store(duration) + if g.Stats.GetFromPeersLatencyLower.Get() < durationMs { + g.Stats.GetFromPeersLatencyLower.Store(durationMs) } + g.metricGetFromPeerLatency.Observe(duration.Seconds()) if err == nil { g.Stats.PeerLoads.Add(1) @@ -564,6 +572,17 @@ func (g *Group) populateCache(key string, value ByteView, cache *cache) { // CacheType represents a type of cache. type CacheType int +func (t CacheType) String() string { + switch t { + case MainCache: + return "main" + case HotCache: + return "hot" + default: + return "" + } +} + const ( // The MainCache is the cache for items that this peer is the // owner for. @@ -587,6 +606,15 @@ func (g *Group) CacheStats(which CacheType) CacheStats { } } +// GetMetrics returns Prometheus metrics. +func (g *Group) GetMetrics() []prometheus.Collector { + return []prometheus.Collector{ + &CacheStatsCollector{group: g}, + metricGetFromPeerLatency, + metricUpdatePeerLatency, + } +} + // NowFunc returns the current time which is used by the LRU to // determine if the value has expired. This can be overridden by // tests to ensure items are evicted when expired. @@ -713,3 +741,78 @@ type CacheStats struct { Hits int64 Evictions int64 } + +var ( + statsBytesDesc = prometheus.NewDesc( + "groupcache_stats_bytes", + "The number of bytes stored in cache", + []string{"group", "type"}, nil, + ) + statsItemsDesc = prometheus.NewDesc( + "groupcache_stats_items", + "The number of items stored in cache", + []string{"group", "type"}, nil, + ) + statsGetsDesc = prometheus.NewDesc( + "groupcache_stats_gets", + "The number of get requests", + []string{"group", "type"}, nil, + ) + statsHitsDesc = prometheus.NewDesc( + "groupcache_stats_hits", + "The number of cache hits", + []string{"group", "type"}, nil, + ) + statsEvictionsDesc = prometheus.NewDesc( + "groupcache_stats_evictions", + "The number of cache evictions", + []string{"group", "type"}, nil, + ) +) + +// CacheStats exposed as Prometheus metrics. +type CacheStatsCollector struct { + group *Group +} + +func (c *CacheStatsCollector) Describe(ch chan<- *prometheus.Desc) { + prometheus.DescribeByCollect(c, ch) +} + +func (c *CacheStatsCollector) Collect(ch chan<- prometheus.Metric) { + types := []CacheType{MainCache, HotCache} + for _, t := range types { + stats := c.group.CacheStats(t) + tstr := t.String() + ch <- prometheus.MustNewConstMetric( + statsBytesDesc, + prometheus.GaugeValue, + float64(stats.Bytes), + c.group.name, tstr, + ) + ch <- prometheus.MustNewConstMetric( + statsItemsDesc, + prometheus.GaugeValue, + float64(stats.Items), + c.group.name, tstr, + ) + ch <- prometheus.MustNewConstMetric( + statsGetsDesc, + prometheus.CounterValue, + float64(stats.Gets), + c.group.name, tstr, + ) + ch <- prometheus.MustNewConstMetric( + statsHitsDesc, + prometheus.CounterValue, + float64(stats.Hits), + c.group.name, tstr, + ) + ch <- prometheus.MustNewConstMetric( + statsEvictionsDesc, + prometheus.CounterValue, + float64(stats.Evictions), + c.group.name, tstr, + ) + } +} diff --git a/metrics.go b/metrics.go new file mode 100644 index 0000000..b42e4f4 --- /dev/null +++ b/metrics.go @@ -0,0 +1,23 @@ +package groupcache + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +var ( + SummaryObjectives = map[float64]float64{ + 0.5: 0.05, + 0.99: 0.001, + 1: 0.001, + } + metricGetFromPeerLatency = prometheus.NewSummaryVec(prometheus.SummaryOpts{ + Name: "groupcache_get_from_peer_latency", + Help: "The latency in seconds getting value from remote peer", + Objectives: SummaryObjectives, + }, []string{"group"}) + metricUpdatePeerLatency = prometheus.NewSummaryVec(prometheus.SummaryOpts{ + Name: "groupcache_update_peer_latency", + Help: "The latency in seconds updating a remote peer during a Set", + Objectives: SummaryObjectives, + }, []string{"group", "peer"}) +)