From 05534d38464bfd887dbfcd1b5278098ec0eae3a2 Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Fri, 30 Jan 2026 14:33:10 +1100 Subject: [PATCH 01/16] Freshen p2p/discover. What a monster --- common/crypto/crypto.go | 22 +- common/crypto/signature_cgo.go | 2 +- common/lru/basiclru.go | 228 +++++ common/lru/basiclru_test.go | 255 +++++ common/lru/blob_lru.go | 84 ++ common/lru/blob_lru_test.go | 155 +++ common/lru/lru.go | 95 ++ common/mclock/alarm.go | 106 ++ common/mclock/alarm_test.go | 116 +++ common/mclock/simclock.go | 4 +- common/metrics/FORK.md | 1 + common/metrics/LICENSE | 29 + common/metrics/README.md | 147 +++ .../metrics/{metrics_enabled.go => config.go} | 0 common/metrics/counter.go | 55 + common/metrics/counter_float64.go | 66 ++ common/metrics/counter_float_64_test.go | 73 ++ common/metrics/counter_test.go | 61 ++ common/metrics/cpu.go | 25 + common/metrics/cpu_disabled.go | 24 + common/metrics/cpu_enabled.go | 44 + common/metrics/cputime_nop.go | 26 + common/metrics/cputime_unix.go | 36 + common/metrics/debug.go | 76 ++ common/metrics/debug_test.go | 48 + common/metrics/disk.go | 25 + common/metrics/disk_linux.go | 72 ++ common/metrics/disk_nop.go | 27 + common/metrics/ewma.go | 91 ++ common/metrics/ewma_test.go | 89 ++ common/metrics/gauge.go | 67 ++ common/metrics/gauge_float64.go | 47 + common/metrics/gauge_float64_test.go | 51 + common/metrics/gauge_info.go | 61 ++ common/metrics/gauge_info_test.go | 36 + common/metrics/gauge_test.go | 31 + common/metrics/healthcheck.go | 35 + common/metrics/histogram.go | 66 ++ common/metrics/histogram_test.go | 95 ++ common/metrics/init_test.go | 5 + common/metrics/json.go | 31 + common/metrics/json_test.go | 28 + common/metrics/log.go | 82 ++ common/metrics/memory.md | 285 ++++++ common/metrics/meter.go | 167 +++ common/metrics/meter_test.go | 83 ++ common/metrics/metrics.go | 204 ++++ common/metrics/metrics_test.go | 62 ++ common/metrics/opentsdb.go | 128 +++ common/metrics/opentsdb_test.go | 66 ++ common/metrics/registry.go | 363 +++++++ common/metrics/registry_test.go | 335 ++++++ common/metrics/resetting_sample.go | 24 + common/metrics/resetting_timer.go | 134 +++ common/metrics/resetting_timer_test.go | 197 ++++ common/metrics/runtimehistogram.go | 298 ++++++ common/metrics/runtimehistogram_test.go | 162 +++ common/metrics/sample.go | 425 ++++++++ common/metrics/sample_test.go | 324 ++++++ common/metrics/syslog.go | 83 ++ common/metrics/timer.go | 149 +++ common/metrics/timer_test.go | 114 +++ common/metrics/validate.sh | 10 + common/metrics/writer.go | 99 ++ common/metrics/writer_test.go | 22 + execution/rlp/decode.go | 610 +++++------ execution/rlp/encbuffer.go | 376 +++++-- execution/rlp/encbuffer_example_test.go | 45 + execution/rlp/encode.go | 305 +++--- execution/rlp/internal/rlpstruct/rlpstruct.go | 4 +- execution/rlp/raw.go | 88 +- execution/rlp/raw_test.go | 343 ++++++- execution/rlp/typecache.go | 217 ++-- go.mod | 1 + go.sum | 2 + p2p/discover/common.go | 124 ++- p2p/discover/lookup.go | 216 ++-- p2p/discover/lookup_util_test.go | 198 ---- p2p/discover/metrics.go | 82 ++ p2p/discover/node.go | 93 +- p2p/discover/ntp.go | 31 +- p2p/discover/table.go | 960 +++++++++--------- p2p/discover/table_integration_test.go | 87 -- p2p/discover/table_reval.go | 248 +++++ p2p/discover/table_reval_test.go | 119 +++ p2p/discover/table_test.go | 636 +++++------- p2p/discover/table_util_test.go | 196 ++-- p2p/discover/v4_lookup_test.go | 301 +++++- p2p/discover/v4_udp.go | 681 +++++-------- p2p/discover/v4_udp_test.go | 398 +++----- p2p/discover/v4wire/v4wire.go | 59 +- p2p/discover/v4wire/v4wire_test.go | 14 +- p2p/discover/v5_lookup_test.go | 134 --- p2p/discover/v5_talk.go | 121 +++ p2p/discover/v5_udp.go | 500 +++++---- p2p/discover/v5_udp_integration_test.go | 121 --- p2p/discover/v5_udp_test.go | 784 +++++++++----- p2p/discover/v5wire/crypto.go | 33 +- p2p/discover/v5wire/crypto_test.go | 17 +- p2p/discover/v5wire/encoding.go | 164 +-- p2p/discover/v5wire/encoding_test.go | 199 ++-- p2p/discover/v5wire/msg.go | 146 ++- p2p/discover/v5wire/session.go | 45 +- p2p/enode/idscheme.go | 25 +- p2p/enode/localnode.go | 91 +- p2p/enode/node.go | 213 ++-- p2p/enode/nodedb.go | 78 +- p2p/enode/nodedb_test.go | 10 +- p2p/enode/urlv4.go | 51 +- p2p/enr/enr.go | 45 +- p2p/enr/enr_test.go | 58 +- p2p/enr/entries.go | 81 +- p2p/event/feedof.go | 164 +++ p2p/event/feedof_test.go | 279 +++++ p2p/netutil/addrutil.go | 49 +- p2p/netutil/iptrack.go | 28 +- p2p/netutil/net.go | 178 ++-- 117 files changed, 12241 insertions(+), 4258 deletions(-) create mode 100644 common/lru/basiclru.go create mode 100644 common/lru/basiclru_test.go create mode 100644 common/lru/blob_lru.go create mode 100644 common/lru/blob_lru_test.go create mode 100644 common/lru/lru.go create mode 100644 common/mclock/alarm.go create mode 100644 common/mclock/alarm_test.go create mode 100644 common/metrics/FORK.md create mode 100644 common/metrics/LICENSE create mode 100644 common/metrics/README.md rename common/metrics/{metrics_enabled.go => config.go} (100%) create mode 100644 common/metrics/counter.go create mode 100644 common/metrics/counter_float64.go create mode 100644 common/metrics/counter_float_64_test.go create mode 100644 common/metrics/counter_test.go create mode 100644 common/metrics/cpu.go create mode 100644 common/metrics/cpu_disabled.go create mode 100644 common/metrics/cpu_enabled.go create mode 100644 common/metrics/cputime_nop.go create mode 100644 common/metrics/cputime_unix.go create mode 100644 common/metrics/debug.go create mode 100644 common/metrics/debug_test.go create mode 100644 common/metrics/disk.go create mode 100644 common/metrics/disk_linux.go create mode 100644 common/metrics/disk_nop.go create mode 100644 common/metrics/ewma.go create mode 100644 common/metrics/ewma_test.go create mode 100644 common/metrics/gauge.go create mode 100644 common/metrics/gauge_float64.go create mode 100644 common/metrics/gauge_float64_test.go create mode 100644 common/metrics/gauge_info.go create mode 100644 common/metrics/gauge_info_test.go create mode 100644 common/metrics/gauge_test.go create mode 100644 common/metrics/healthcheck.go create mode 100644 common/metrics/histogram.go create mode 100644 common/metrics/histogram_test.go create mode 100644 common/metrics/init_test.go create mode 100644 common/metrics/json.go create mode 100644 common/metrics/json_test.go create mode 100644 common/metrics/log.go create mode 100644 common/metrics/memory.md create mode 100644 common/metrics/meter.go create mode 100644 common/metrics/meter_test.go create mode 100644 common/metrics/metrics.go create mode 100644 common/metrics/metrics_test.go create mode 100644 common/metrics/opentsdb.go create mode 100644 common/metrics/opentsdb_test.go create mode 100644 common/metrics/registry.go create mode 100644 common/metrics/registry_test.go create mode 100644 common/metrics/resetting_sample.go create mode 100644 common/metrics/resetting_timer.go create mode 100644 common/metrics/resetting_timer_test.go create mode 100644 common/metrics/runtimehistogram.go create mode 100644 common/metrics/runtimehistogram_test.go create mode 100644 common/metrics/sample.go create mode 100644 common/metrics/sample_test.go create mode 100644 common/metrics/syslog.go create mode 100644 common/metrics/timer.go create mode 100644 common/metrics/timer_test.go create mode 100755 common/metrics/validate.sh create mode 100644 common/metrics/writer.go create mode 100644 common/metrics/writer_test.go create mode 100644 execution/rlp/encbuffer_example_test.go delete mode 100644 p2p/discover/lookup_util_test.go create mode 100644 p2p/discover/metrics.go delete mode 100644 p2p/discover/table_integration_test.go create mode 100644 p2p/discover/table_reval.go create mode 100644 p2p/discover/table_reval_test.go delete mode 100644 p2p/discover/v5_lookup_test.go create mode 100644 p2p/discover/v5_talk.go delete mode 100644 p2p/discover/v5_udp_integration_test.go create mode 100644 p2p/event/feedof.go create mode 100644 p2p/event/feedof_test.go diff --git a/common/crypto/crypto.go b/common/crypto/crypto.go index 522d2f28e8e..adbd7900192 100644 --- a/common/crypto/crypto.go +++ b/common/crypto/crypto.go @@ -58,6 +58,15 @@ var ( var errInvalidPubkey = errors.New("invalid secp256k1 public key") +// EllipticCurve contains curve operations. +type EllipticCurve interface { + elliptic.Curve + + // Point marshaling/unmarshaing. + Marshal(x, y *big.Int) []byte + Unmarshal(data []byte) (x, y *big.Int) +} + // KeccakState wraps sha3.state. In addition to the usual hash methods, it also supports // Read to get a variable amount of data from the hash state. Read is faster than Sum // because it doesn't copy the internal state, but also modifies the internal state. @@ -204,11 +213,20 @@ func MarshalPubkey(pubkey *ecdsa.PublicKey) []byte { return keyBytes[1:] } +// FromECDSAPub converts a secp256k1 public key to bytes. +// Note: it does not use the curve from pub, instead it always +// encodes using secp256k1. +func FromECDSAPub(pub *ecdsa.PublicKey) []byte { + if pub == nil || pub.X == nil || pub.Y == nil { + return nil + } + return S256().Marshal(pub.X, pub.Y) +} + // HexToECDSA parses a secp256k1 private key. func HexToECDSA(hexkey string) (*ecdsa.PrivateKey, error) { b, err := hex.DecodeString(hexkey) - var byteErr hex.InvalidByteError - if errors.As(err, &byteErr) { + if byteErr, ok := err.(hex.InvalidByteError); ok { return nil, fmt.Errorf("invalid hex character %q in private key", byte(byteErr)) } else if err != nil { return nil, errors.New("invalid hex data for private key") diff --git a/common/crypto/signature_cgo.go b/common/crypto/signature_cgo.go index e818aeb3680..ff8dd5cd9e3 100644 --- a/common/crypto/signature_cgo.go +++ b/common/crypto/signature_cgo.go @@ -92,6 +92,6 @@ func CompressPubkey(pubkey *ecdsa.PublicKey) []byte { } // S256 returns an instance of the secp256k1 curve. -func S256() elliptic.Curve { +func S256() EllipticCurve { return secp256k1.S256() } diff --git a/common/lru/basiclru.go b/common/lru/basiclru.go new file mode 100644 index 00000000000..154831a7960 --- /dev/null +++ b/common/lru/basiclru.go @@ -0,0 +1,228 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Package lru implements generically-typed LRU caches. +package lru + +// BasicLRU is a simple LRU cache. +// +// This type is not safe for concurrent use. +// The zero value is not valid, instances must be created using NewCache. +type BasicLRU[K comparable, V any] struct { + list *list[K] + items map[K]cacheItem[K, V] + cap int +} + +type cacheItem[K any, V any] struct { + elem *listElem[K] + value V +} + +// NewBasicLRU creates a new LRU cache. +func NewBasicLRU[K comparable, V any](capacity int) BasicLRU[K, V] { + if capacity <= 0 { + capacity = 1 + } + c := BasicLRU[K, V]{ + items: make(map[K]cacheItem[K, V]), + list: newList[K](), + cap: capacity, + } + return c +} + +// Add adds a value to the cache. Returns true if an item was evicted to store the new item. +func (c *BasicLRU[K, V]) Add(key K, value V) (evicted bool) { + _, _, evicted = c.Add3(key, value) + return evicted +} + +// Add3 adds a value to the cache. If an item was evicted to store the new one, it returns the evicted item. +func (c *BasicLRU[K, V]) Add3(key K, value V) (ek K, ev V, evicted bool) { + item, ok := c.items[key] + if ok { + item.value = value + c.items[key] = item + c.list.moveToFront(item.elem) + return ek, ev, false + } + + var elem *listElem[K] + if c.Len() >= c.cap { + elem = c.list.removeLast() + evicted = true + ek = elem.v + ev = c.items[ek].value + delete(c.items, ek) + } else { + elem = new(listElem[K]) + } + + // Store the new item. + // Note that if another item was evicted, we re-use its list element here. + elem.v = key + c.items[key] = cacheItem[K, V]{elem, value} + c.list.pushElem(elem) + return ek, ev, evicted +} + +// Contains reports whether the given key exists in the cache. +func (c *BasicLRU[K, V]) Contains(key K) bool { + _, ok := c.items[key] + return ok +} + +// Get retrieves a value from the cache. This marks the key as recently used. +func (c *BasicLRU[K, V]) Get(key K) (value V, ok bool) { + item, ok := c.items[key] + if !ok { + return value, false + } + c.list.moveToFront(item.elem) + return item.value, true +} + +// GetOldest retrieves the least-recently-used item. +// Note that this does not update the item's recency. +func (c *BasicLRU[K, V]) GetOldest() (key K, value V, ok bool) { + lastElem := c.list.last() + if lastElem == nil { + return key, value, false + } + key = lastElem.v + item := c.items[key] + return key, item.value, true +} + +// Len returns the current number of items in the cache. +func (c *BasicLRU[K, V]) Len() int { + return len(c.items) +} + +// Peek retrieves a value from the cache, but does not mark the key as recently used. +func (c *BasicLRU[K, V]) Peek(key K) (value V, ok bool) { + item, ok := c.items[key] + return item.value, ok +} + +// Purge empties the cache. +func (c *BasicLRU[K, V]) Purge() { + c.list.init() + clear(c.items) +} + +// Remove drops an item from the cache. Returns true if the key was present in cache. +func (c *BasicLRU[K, V]) Remove(key K) bool { + item, ok := c.items[key] + if ok { + delete(c.items, key) + c.list.remove(item.elem) + } + return ok +} + +// RemoveOldest drops the least recently used item. +func (c *BasicLRU[K, V]) RemoveOldest() (key K, value V, ok bool) { + lastElem := c.list.last() + if lastElem == nil { + return key, value, false + } + + key = lastElem.v + item := c.items[key] + delete(c.items, key) + c.list.remove(lastElem) + return key, item.value, true +} + +// Keys returns all keys in the cache. +func (c *BasicLRU[K, V]) Keys() []K { + keys := make([]K, 0, len(c.items)) + return c.list.appendTo(keys) +} + +// list is a doubly-linked list holding items of type he. +// The zero value is not valid, use newList to create lists. +type list[T any] struct { + root listElem[T] +} + +type listElem[T any] struct { + next *listElem[T] + prev *listElem[T] + v T +} + +func newList[T any]() *list[T] { + l := new(list[T]) + l.init() + return l +} + +// init reinitializes the list, making it empty. +func (l *list[T]) init() { + l.root.next = &l.root + l.root.prev = &l.root +} + +// pushElem adds an element to the front of the list. +func (l *list[T]) pushElem(e *listElem[T]) { + e.prev = &l.root + e.next = l.root.next + l.root.next = e + e.next.prev = e +} + +// moveToFront makes 'node' the head of the list. +func (l *list[T]) moveToFront(e *listElem[T]) { + e.prev.next = e.next + e.next.prev = e.prev + l.pushElem(e) +} + +// remove removes an element from the list. +func (l *list[T]) remove(e *listElem[T]) { + e.prev.next = e.next + e.next.prev = e.prev + e.next, e.prev = nil, nil +} + +// removeLast removes the last element of the list. +func (l *list[T]) removeLast() *listElem[T] { + last := l.last() + if last != nil { + l.remove(last) + } + return last +} + +// last returns the last element of the list, or nil if the list is empty. +func (l *list[T]) last() *listElem[T] { + e := l.root.prev + if e == &l.root { + return nil + } + return e +} + +// appendTo appends all list elements to a slice. +func (l *list[T]) appendTo(slice []T) []T { + for e := l.root.prev; e != &l.root; e = e.prev { + slice = append(slice, e.v) + } + return slice +} diff --git a/common/lru/basiclru_test.go b/common/lru/basiclru_test.go new file mode 100644 index 00000000000..29812bda157 --- /dev/null +++ b/common/lru/basiclru_test.go @@ -0,0 +1,255 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package lru + +import ( + crand "crypto/rand" + "fmt" + "io" + "math/rand" + "testing" +) + +// Some of these test cases were adapted +// from https://github.com/hashicorp/golang-lru/blob/master/simplelru/lru_test.go + +func TestBasicLRU(t *testing.T) { + cache := NewBasicLRU[int, int](128) + + for i := 0; i < 256; i++ { + cache.Add(i, i) + } + if cache.Len() != 128 { + t.Fatalf("bad len: %v", cache.Len()) + } + + // Check that Keys returns least-recent key first. + keys := cache.Keys() + if len(keys) != 128 { + t.Fatal("wrong Keys() length", len(keys)) + } + for i, k := range keys { + v, ok := cache.Peek(k) + if !ok { + t.Fatalf("expected key %d be present", i) + } + if v != k { + t.Fatalf("expected %d == %d", k, v) + } + if v != i+128 { + t.Fatalf("wrong value at key %d: %d, want %d", i, v, i+128) + } + } + + for i := 0; i < 128; i++ { + _, ok := cache.Get(i) + if ok { + t.Fatalf("%d should be evicted", i) + } + } + for i := 128; i < 256; i++ { + _, ok := cache.Get(i) + if !ok { + t.Fatalf("%d should not be evicted", i) + } + } + + for i := 128; i < 192; i++ { + ok := cache.Remove(i) + if !ok { + t.Fatalf("%d should be in cache", i) + } + ok = cache.Remove(i) + if ok { + t.Fatalf("%d should not be in cache", i) + } + _, ok = cache.Get(i) + if ok { + t.Fatalf("%d should be deleted", i) + } + } + + // Request item 192. + cache.Get(192) + // It should be the last item returned by Keys(). + for i, k := range cache.Keys() { + if (i < 63 && k != i+193) || (i == 63 && k != 192) { + t.Fatalf("out of order key: %v", k) + } + } + + cache.Purge() + if cache.Len() != 0 { + t.Fatalf("bad len: %v", cache.Len()) + } + if _, ok := cache.Get(200); ok { + t.Fatalf("should contain nothing") + } +} + +func TestBasicLRUAddExistingKey(t *testing.T) { + cache := NewBasicLRU[int, int](1) + + cache.Add(1, 1) + cache.Add(1, 2) + + v, _ := cache.Get(1) + if v != 2 { + t.Fatal("wrong value:", v) + } +} + +// This test checks GetOldest and RemoveOldest. +func TestBasicLRUGetOldest(t *testing.T) { + cache := NewBasicLRU[int, int](128) + for i := 0; i < 256; i++ { + cache.Add(i, i) + } + + k, _, ok := cache.GetOldest() + if !ok { + t.Fatalf("missing") + } + if k != 128 { + t.Fatalf("bad: %v", k) + } + + k, _, ok = cache.RemoveOldest() + if !ok { + t.Fatalf("missing") + } + if k != 128 { + t.Fatalf("bad: %v", k) + } + + k, _, ok = cache.RemoveOldest() + if !ok { + t.Fatalf("missing oldest item") + } + if k != 129 { + t.Fatalf("wrong oldest item: %v", k) + } +} + +// Test that Add returns true/false if an eviction occurred +func TestBasicLRUAddReturnValue(t *testing.T) { + cache := NewBasicLRU[int, int](1) + if cache.Add(1, 1) { + t.Errorf("first add shouldn't have evicted") + } + if !cache.Add(2, 2) { + t.Errorf("second add should have evicted") + } +} + +// This test verifies that Contains doesn't change item recency. +func TestBasicLRUContains(t *testing.T) { + cache := NewBasicLRU[int, int](2) + cache.Add(1, 1) + cache.Add(2, 2) + if !cache.Contains(1) { + t.Errorf("1 should be in the cache") + } + cache.Add(3, 3) + if cache.Contains(1) { + t.Errorf("Contains should not have updated recency of 1") + } +} + +// Test that Peek doesn't update recent-ness +func TestBasicLRUPeek(t *testing.T) { + cache := NewBasicLRU[int, int](2) + cache.Add(1, 1) + cache.Add(2, 2) + if v, ok := cache.Peek(1); !ok || v != 1 { + t.Errorf("1 should be set to 1") + } + cache.Add(3, 3) + if cache.Contains(1) { + t.Errorf("should not have updated recent-ness of 1") + } +} + +func BenchmarkLRU(b *testing.B) { + var ( + capacity = 1000 + indexes = make([]int, capacity*20) + keys = make([]string, capacity) + values = make([][]byte, capacity) + ) + for i := range indexes { + indexes[i] = rand.Intn(capacity) + } + for i := range keys { + b := make([]byte, 32) + crand.Read(b) + keys[i] = string(b) + crand.Read(b) + values[i] = b + } + + var sink []byte + + b.Run("Add/BasicLRU", func(b *testing.B) { + cache := NewBasicLRU[int, int](capacity) + for i := 0; i < b.N; i++ { + cache.Add(i, i) + } + }) + b.Run("Get/BasicLRU", func(b *testing.B) { + cache := NewBasicLRU[string, []byte](capacity) + for i := 0; i < capacity; i++ { + index := indexes[i] + cache.Add(keys[index], values[index]) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + k := keys[indexes[i%len(indexes)]] + v, ok := cache.Get(k) + if ok { + sink = v + } + } + }) + + // // vs. github.com/hashicorp/golang-lru/simplelru + // b.Run("Add/simplelru.LRU", func(b *testing.B) { + // cache, _ := simplelru.NewLRU(capacity, nil) + // for i := 0; i < b.N; i++ { + // cache.Add(i, i) + // } + // }) + // b.Run("Get/simplelru.LRU", func(b *testing.B) { + // cache, _ := simplelru.NewLRU(capacity, nil) + // for i := 0; i < capacity; i++ { + // index := indexes[i] + // cache.Add(keys[index], values[index]) + // } + // + // b.ResetTimer() + // for i := 0; i < b.N; i++ { + // k := keys[indexes[i%len(indexes)]] + // v, ok := cache.Get(k) + // if ok { + // sink = v.([]byte) + // } + // } + // }) + + fmt.Fprintln(io.Discard, sink) +} diff --git a/common/lru/blob_lru.go b/common/lru/blob_lru.go new file mode 100644 index 00000000000..c9b33985032 --- /dev/null +++ b/common/lru/blob_lru.go @@ -0,0 +1,84 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package lru + +import ( + "math" + "sync" +) + +// blobType is the type constraint for values stored in SizeConstrainedCache. +type blobType interface { + ~[]byte | ~string +} + +// SizeConstrainedCache is a cache where capacity is in bytes (instead of item count). When the cache +// is at capacity, and a new item is added, older items are evicted until the size +// constraint is met. +// +// OBS: This cache assumes that items are content-addressed: keys are unique per content. +// In other words: two Add(..) with the same key K, will always have the same value V. +type SizeConstrainedCache[K comparable, V blobType] struct { + size uint64 + maxSize uint64 + lru BasicLRU[K, V] + lock sync.Mutex +} + +// NewSizeConstrainedCache creates a new size-constrained LRU cache. +func NewSizeConstrainedCache[K comparable, V blobType](maxSize uint64) *SizeConstrainedCache[K, V] { + return &SizeConstrainedCache[K, V]{ + size: 0, + maxSize: maxSize, + lru: NewBasicLRU[K, V](math.MaxInt), + } +} + +// Add adds a value to the cache. Returns true if an eviction occurred. +// OBS: This cache assumes that items are content-addressed: keys are unique per content. +// In other words: two Add(..) with the same key K, will always have the same value V. +// OBS: The value is _not_ copied on Add, so the caller must not modify it afterwards. +func (c *SizeConstrainedCache[K, V]) Add(key K, value V) (evicted bool) { + c.lock.Lock() + defer c.lock.Unlock() + + // Unless it is already present, might need to evict something. + // OBS: If it is present, we still call Add internally to bump the recentness. + if !c.lru.Contains(key) { + targetSize := c.size + uint64(len(value)) + for targetSize > c.maxSize { + evicted = true + _, v, ok := c.lru.RemoveOldest() + if !ok { + // list is now empty. Break + break + } + targetSize -= uint64(len(v)) + } + c.size = targetSize + } + c.lru.Add(key, value) + return evicted +} + +// Get looks up a key's value from the cache. +func (c *SizeConstrainedCache[K, V]) Get(key K) (V, bool) { + c.lock.Lock() + defer c.lock.Unlock() + + return c.lru.Get(key) +} diff --git a/common/lru/blob_lru_test.go b/common/lru/blob_lru_test.go new file mode 100644 index 00000000000..ca1b0ddd742 --- /dev/null +++ b/common/lru/blob_lru_test.go @@ -0,0 +1,155 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package lru + +import ( + "encoding/binary" + "fmt" + "testing" +) + +type testKey [8]byte + +func mkKey(i int) (key testKey) { + binary.LittleEndian.PutUint64(key[:], uint64(i)) + return key +} + +func TestSizeConstrainedCache(t *testing.T) { + lru := NewSizeConstrainedCache[testKey, []byte](100) + var want uint64 + // Add 11 items of 10 byte each. First item should be swapped out + for i := 0; i < 11; i++ { + k := mkKey(i) + v := fmt.Sprintf("value-%04d", i) + lru.Add(k, []byte(v)) + want += uint64(len(v)) + if want > 100 { + want = 100 + } + if have := lru.size; have != want { + t.Fatalf("size wrong, have %d want %d", have, want) + } + } + // Zero:th should be evicted + { + k := mkKey(0) + if _, ok := lru.Get(k); ok { + t.Fatalf("should be evicted: %v", k) + } + } + // Elems 1-11 should be present + for i := 1; i < 11; i++ { + k := mkKey(i) + want := fmt.Sprintf("value-%04d", i) + have, ok := lru.Get(k) + if !ok { + t.Fatalf("missing key %v", k) + } + if string(have) != want { + t.Fatalf("wrong value, have %v want %v", have, want) + } + } +} + +// This test adds inserting an element exceeding the max size. +func TestSizeConstrainedCacheOverflow(t *testing.T) { + lru := NewSizeConstrainedCache[testKey, []byte](100) + + // Add 10 items of 10 byte each, filling the cache + for i := 0; i < 10; i++ { + k := mkKey(i) + v := fmt.Sprintf("value-%04d", i) + lru.Add(k, []byte(v)) + } + // Add one single large elem. We expect it to swap out all entries. + { + k := mkKey(1337) + v := make([]byte, 200) + lru.Add(k, v) + } + // Elems 0-9 should be missing + for i := 1; i < 10; i++ { + k := mkKey(i) + if _, ok := lru.Get(k); ok { + t.Fatalf("should be evicted: %v", k) + } + } + // The size should be accurate + if have, want := lru.size, uint64(200); have != want { + t.Fatalf("size wrong, have %d want %d", have, want) + } + // Adding one small item should swap out the large one + { + i := 0 + k := mkKey(i) + v := fmt.Sprintf("value-%04d", i) + lru.Add(k, []byte(v)) + if have, want := lru.size, uint64(10); have != want { + t.Fatalf("size wrong, have %d want %d", have, want) + } + } +} + +// This checks what happens when inserting the same k/v multiple times. +func TestSizeConstrainedCacheSameItem(t *testing.T) { + lru := NewSizeConstrainedCache[testKey, []byte](100) + + // Add one 10 byte-item 10 times. + k := mkKey(0) + v := fmt.Sprintf("value-%04d", 0) + for i := 0; i < 10; i++ { + lru.Add(k, []byte(v)) + } + + // The size should be accurate. + if have, want := lru.size, uint64(10); have != want { + t.Fatalf("size wrong, have %d want %d", have, want) + } +} + +// This tests that empty/nil values are handled correctly. +func TestSizeConstrainedCacheEmpties(t *testing.T) { + lru := NewSizeConstrainedCache[testKey, []byte](100) + + // This test abuses the lru a bit, using different keys for identical value(s). + for i := 0; i < 10; i++ { + lru.Add(testKey{byte(i)}, []byte{}) + lru.Add(testKey{byte(255 - i)}, nil) + } + + // The size should not count, only the values count. So this could be a DoS + // since it basically has no cap, and it is intentionally overloaded with + // different-keyed 0-length values. + if have, want := lru.size, uint64(0); have != want { + t.Fatalf("size wrong, have %d want %d", have, want) + } + + for i := 0; i < 10; i++ { + if v, ok := lru.Get(testKey{byte(i)}); !ok { + t.Fatalf("test %d: expected presence", i) + } else if v == nil { + t.Fatalf("test %d, v is nil", i) + } + + if v, ok := lru.Get(testKey{byte(255 - i)}); !ok { + t.Fatalf("test %d: expected presence", i) + } else if v != nil { + t.Fatalf("test %d, v is not nil", i) + } + } +} diff --git a/common/lru/lru.go b/common/lru/lru.go new file mode 100644 index 00000000000..45965adb0df --- /dev/null +++ b/common/lru/lru.go @@ -0,0 +1,95 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package lru + +import "sync" + +// Cache is a LRU cache. +// This type is safe for concurrent use. +type Cache[K comparable, V any] struct { + cache BasicLRU[K, V] + mu sync.Mutex +} + +// NewCache creates an LRU cache. +func NewCache[K comparable, V any](capacity int) *Cache[K, V] { + return &Cache[K, V]{cache: NewBasicLRU[K, V](capacity)} +} + +// Add adds a value to the cache. Returns true if an item was evicted to store the new item. +func (c *Cache[K, V]) Add(key K, value V) (evicted bool) { + c.mu.Lock() + defer c.mu.Unlock() + + return c.cache.Add(key, value) +} + +// Contains reports whether the given key exists in the cache. +func (c *Cache[K, V]) Contains(key K) bool { + c.mu.Lock() + defer c.mu.Unlock() + + return c.cache.Contains(key) +} + +// Get retrieves a value from the cache. This marks the key as recently used. +func (c *Cache[K, V]) Get(key K) (value V, ok bool) { + c.mu.Lock() + defer c.mu.Unlock() + + return c.cache.Get(key) +} + +// Len returns the current number of items in the cache. +func (c *Cache[K, V]) Len() int { + c.mu.Lock() + defer c.mu.Unlock() + + return c.cache.Len() +} + +// Peek retrieves a value from the cache, but does not mark the key as recently used. +func (c *Cache[K, V]) Peek(key K) (value V, ok bool) { + c.mu.Lock() + defer c.mu.Unlock() + + return c.cache.Peek(key) +} + +// Purge empties the cache. +func (c *Cache[K, V]) Purge() { + c.mu.Lock() + defer c.mu.Unlock() + + c.cache.Purge() +} + +// Remove drops an item from the cache. Returns true if the key was present in cache. +func (c *Cache[K, V]) Remove(key K) bool { + c.mu.Lock() + defer c.mu.Unlock() + + return c.cache.Remove(key) +} + +// Keys returns all keys of items currently in the LRU. +func (c *Cache[K, V]) Keys() []K { + c.mu.Lock() + defer c.mu.Unlock() + + return c.cache.Keys() +} diff --git a/common/mclock/alarm.go b/common/mclock/alarm.go new file mode 100644 index 00000000000..e83810a6a07 --- /dev/null +++ b/common/mclock/alarm.go @@ -0,0 +1,106 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package mclock + +import ( + "time" +) + +// Alarm sends timed notifications on a channel. This is very similar to a regular timer, +// but is easier to use in code that needs to re-schedule the same timer over and over. +// +// When scheduling an Alarm, the channel returned by C() will receive a value no later +// than the scheduled time. An Alarm can be reused after it has fired and can also be +// canceled by calling Stop. +type Alarm struct { + ch chan struct{} + clock Clock + timer Timer + deadline AbsTime +} + +// NewAlarm creates an Alarm. +func NewAlarm(clock Clock) *Alarm { + if clock == nil { + panic("nil clock") + } + return &Alarm{ + ch: make(chan struct{}, 1), + clock: clock, + } +} + +// C returns the alarm notification channel. This channel remains identical for +// the entire lifetime of the alarm, and is never closed. +func (e *Alarm) C() <-chan struct{} { + return e.ch +} + +// Stop cancels the alarm and drains the channel. +// This method is not safe for concurrent use. +func (e *Alarm) Stop() { + // Clear timer. + if e.timer != nil { + e.timer.Stop() + } + e.deadline = 0 + + // Drain the channel. + select { + case <-e.ch: + default: + } +} + +// Schedule sets the alarm to fire no later than the given time. If the alarm was already +// scheduled but has not fired yet, it may fire earlier than the newly-scheduled time. +func (e *Alarm) Schedule(time AbsTime) { + now := e.clock.Now() + e.schedule(now, time) +} + +func (e *Alarm) schedule(now, newDeadline AbsTime) { + if e.timer != nil { + if e.deadline > now && e.deadline <= newDeadline { + // Here, the current timer can be reused because it is already scheduled to + // occur earlier than the new deadline. + // + // The e.deadline > now part of the condition is important. If the old + // deadline lies in the past, we assume the timer has already fired and needs + // to be rescheduled. + return + } + e.timer.Stop() + } + + // Set the timer. + d := time.Duration(0) + if newDeadline < now { + newDeadline = now + } else { + d = newDeadline.Sub(now) + } + e.timer = e.clock.AfterFunc(d, e.send) + e.deadline = newDeadline +} + +func (e *Alarm) send() { + select { + case e.ch <- struct{}{}: + default: + } +} diff --git a/common/mclock/alarm_test.go b/common/mclock/alarm_test.go new file mode 100644 index 00000000000..d2ad9913fd2 --- /dev/null +++ b/common/mclock/alarm_test.go @@ -0,0 +1,116 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package mclock + +import "testing" + +// This test checks basic functionality of Alarm. +func TestAlarm(t *testing.T) { + clk := new(Simulated) + clk.Run(20) + a := NewAlarm(clk) + + a.Schedule(clk.Now() + 10) + if recv(a.C()) { + t.Fatal("Alarm fired before scheduled deadline") + } + if ntimers := clk.ActiveTimers(); ntimers != 1 { + t.Fatal("clock has", ntimers, "active timers, want", 1) + } + clk.Run(5) + if recv(a.C()) { + t.Fatal("Alarm fired too early") + } + + clk.Run(5) + if !recv(a.C()) { + t.Fatal("Alarm did not fire") + } + if recv(a.C()) { + t.Fatal("Alarm fired twice") + } + if ntimers := clk.ActiveTimers(); ntimers != 0 { + t.Fatal("clock has", ntimers, "active timers, want", 0) + } + + a.Schedule(clk.Now() + 5) + if recv(a.C()) { + t.Fatal("Alarm fired before scheduled deadline when scheduling the second event") + } + + clk.Run(5) + if !recv(a.C()) { + t.Fatal("Alarm did not fire when scheduling the second event") + } + if recv(a.C()) { + t.Fatal("Alarm fired twice when scheduling the second event") + } +} + +// This test checks that scheduling an Alarm to an earlier time than the +// one already scheduled works properly. +func TestAlarmScheduleEarlier(t *testing.T) { + clk := new(Simulated) + clk.Run(20) + a := NewAlarm(clk) + + a.Schedule(clk.Now() + 50) + clk.Run(5) + a.Schedule(clk.Now() + 1) + clk.Run(3) + if !recv(a.C()) { + t.Fatal("Alarm did not fire") + } +} + +// This test checks that scheduling an Alarm to a later time than the +// one already scheduled works properly. +func TestAlarmScheduleLater(t *testing.T) { + clk := new(Simulated) + clk.Run(20) + a := NewAlarm(clk) + + a.Schedule(clk.Now() + 50) + clk.Run(5) + a.Schedule(clk.Now() + 100) + clk.Run(50) + if !recv(a.C()) { + t.Fatal("Alarm did not fire") + } +} + +// This test checks that scheduling an Alarm in the past makes it fire immediately. +func TestAlarmNegative(t *testing.T) { + clk := new(Simulated) + clk.Run(50) + a := NewAlarm(clk) + + a.Schedule(-1) + clk.Run(1) // needed to process timers + if !recv(a.C()) { + t.Fatal("Alarm did not fire for negative time") + } +} + +func recv(ch <-chan struct{}) bool { + select { + case <-ch: + return true + default: + return false + } +} diff --git a/common/mclock/simclock.go b/common/mclock/simclock.go index 727855a0e1a..24ef865216a 100644 --- a/common/mclock/simclock.go +++ b/common/mclock/simclock.go @@ -61,7 +61,7 @@ func (s *Simulated) Run(d time.Duration) { s.mu.Lock() s.init() - end := s.now + AbsTime(d) + end := s.now.Add(d) var do []func() for len(s.scheduled) > 0 && s.scheduled[0].at <= end { ev := heap.Pop(&s.scheduled).(*simTimer) @@ -137,7 +137,7 @@ func (s *Simulated) AfterFunc(d time.Duration, fn func()) Timer { func (s *Simulated) schedule(d time.Duration, fn func()) *simTimer { s.init() - at := s.now + AbsTime(d) + at := s.now.Add(d) ev := &simTimer{do: fn, at: at, s: s} heap.Push(&s.scheduled, ev) s.cond.Broadcast() diff --git a/common/metrics/FORK.md b/common/metrics/FORK.md new file mode 100644 index 00000000000..b19985bf56e --- /dev/null +++ b/common/metrics/FORK.md @@ -0,0 +1 @@ +This repo has been forked from https://github.com/rcrowley/go-metrics at commit e181e09 diff --git a/common/metrics/LICENSE b/common/metrics/LICENSE new file mode 100644 index 00000000000..363fa9ee77b --- /dev/null +++ b/common/metrics/LICENSE @@ -0,0 +1,29 @@ +Copyright 2012 Richard Crowley. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + +THIS SOFTWARE IS PROVIDED BY RICHARD CROWLEY ``AS IS'' AND ANY EXPRESS +OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL RICHARD CROWLEY OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +THE POSSIBILITY OF SUCH DAMAGE. + +The views and conclusions contained in the software and documentation +are those of the authors and should not be interpreted as representing +official policies, either expressed or implied, of Richard Crowley. diff --git a/common/metrics/README.md b/common/metrics/README.md new file mode 100644 index 00000000000..85b119470a9 --- /dev/null +++ b/common/metrics/README.md @@ -0,0 +1,147 @@ +go-metrics +========== + +![travis build status](https://travis-ci.org/rcrowley/go-metrics.svg?branch=master) + +Go port of Coda Hale's Metrics library: . + +Documentation: . + +Usage +----- + +Create and update metrics: + +```go +c := metrics.NewCounter() +metrics.Register("foo", c) +c.Inc(47) + +g := metrics.NewGauge() +metrics.Register("bar", g) +g.Update(47) + +r := NewRegistry() +g := metrics.NewRegisteredFunctionalGauge("cache-evictions", r, func() int64 { return cache.getEvictionsCount() }) + +s := metrics.NewExpDecaySample(1028, 0.015) // or metrics.NewUniformSample(1028) +h := metrics.NewHistogram(s) +metrics.Register("baz", h) +h.Update(47) + +m := metrics.NewMeter() +metrics.Register("quux", m) +m.Mark(47) + +t := metrics.NewTimer() +metrics.Register("bang", t) +t.Time(func() {}) +t.Update(47) +``` + +Register() is not threadsafe. For threadsafe metric registration use +GetOrRegister: + +```go +t := metrics.GetOrRegisterTimer("account.create.latency", nil) +t.Time(func() {}) +t.Update(47) +``` + +**NOTE:** Be sure to unregister short-lived meters and timers otherwise they will +leak memory: + +```go +// Will call Stop() on the Meter to allow for garbage collection +metrics.Unregister("quux") +// Or similarly for a Timer that embeds a Meter +metrics.Unregister("bang") +``` + +Periodically log every metric in human-readable form to standard error: + +```go +go metrics.Log(metrics.DefaultRegistry, 5 * time.Second, log.New(os.Stderr, "metrics: ", log.Lmicroseconds)) +``` + +Periodically log every metric in slightly-more-parseable form to syslog: + +```go +w, _ := syslog.Dial("unixgram", "/dev/log", syslog.LOG_INFO, "metrics") +go metrics.Syslog(metrics.DefaultRegistry, 60e9, w) +``` + +Periodically emit every metric to Graphite using the [Graphite client](https://github.com/cyberdelia/go-metrics-graphite): + +```go + +import "github.com/cyberdelia/go-metrics-graphite" + +addr, _ := net.ResolveTCPAddr("tcp", "127.0.0.1:2003") +go graphite.Graphite(metrics.DefaultRegistry, 10e9, "metrics", addr) +``` + +Periodically emit every metric into InfluxDB: + +**NOTE:** this has been pulled out of the library due to constant fluctuations +in the InfluxDB API. In fact, all client libraries are on their way out. see +issues [#121](https://github.com/rcrowley/go-metrics/issues/121) and +[#124](https://github.com/rcrowley/go-metrics/issues/124) for progress and details. + +```go +import "github.com/vrischmann/go-metrics-influxdb" + +go influxdb.InfluxDB(metrics.DefaultRegistry, + 10e9, + "127.0.0.1:8086", + "database-name", + "username", + "password" +) +``` + +Periodically emit every metric to StatHat: + +```go +import "github.com/rcrowley/go-metrics/stathat" + +go stathat.Stathat(metrics.DefaultRegistry, 10e9, "example@example.com") +``` + +Maintain all metrics along with expvars at `/debug/metrics`: + +This uses the same mechanism as [the official expvar](https://golang.org/pkg/expvar/) +but exposed under `/debug/metrics`, which shows a json representation of all your usual expvars +as well as all your go-metrics. + + +```go +import "github.com/rcrowley/go-metrics/exp" + +exp.Exp(metrics.DefaultRegistry) +``` + +Installation +------------ + +```sh +go get github.com/rcrowley/go-metrics +``` + +StatHat support additionally requires their Go client: + +```sh +go get github.com/stathat/go +``` + +Publishing Metrics +------------------ + +Clients are available for the following destinations: + +* Graphite - https://github.com/cyberdelia/go-metrics-graphite +* InfluxDB - https://github.com/vrischmann/go-metrics-influxdb +* Ganglia - https://github.com/appscode/metlia +* Prometheus - https://github.com/deathowl/go-metrics-prometheus +* DataDog - https://github.com/syntaqx/go-metrics-datadog +* SignalFX - https://github.com/pascallouisperez/go-metrics-signalfx diff --git a/common/metrics/metrics_enabled.go b/common/metrics/config.go similarity index 100% rename from common/metrics/metrics_enabled.go rename to common/metrics/config.go diff --git a/common/metrics/counter.go b/common/metrics/counter.go new file mode 100644 index 00000000000..c884e9a1784 --- /dev/null +++ b/common/metrics/counter.go @@ -0,0 +1,55 @@ +package metrics + +import ( + "sync/atomic" +) + +// GetOrRegisterCounter returns an existing Counter or constructs and registers +// a new Counter. +func GetOrRegisterCounter(name string, r Registry) *Counter { + return getOrRegister(name, NewCounter, r) +} + +// NewCounter constructs a new Counter. +func NewCounter() *Counter { + return new(Counter) +} + +// NewRegisteredCounter constructs and registers a new Counter. +func NewRegisteredCounter(name string, r Registry) *Counter { + c := NewCounter() + if r == nil { + r = DefaultRegistry + } + r.Register(name, c) + return c +} + +// CounterSnapshot is a read-only copy of a Counter. +type CounterSnapshot int64 + +// Count returns the count at the time the snapshot was taken. +func (c CounterSnapshot) Count() int64 { return int64(c) } + +// Counter hold an int64 value that can be incremented and decremented. +type Counter atomic.Int64 + +// Clear sets the counter to zero. +func (c *Counter) Clear() { + (*atomic.Int64)(c).Store(0) +} + +// Dec decrements the counter by the given amount. +func (c *Counter) Dec(i int64) { + (*atomic.Int64)(c).Add(-i) +} + +// Inc increments the counter by the given amount. +func (c *Counter) Inc(i int64) { + (*atomic.Int64)(c).Add(i) +} + +// Snapshot returns a read-only copy of the counter. +func (c *Counter) Snapshot() CounterSnapshot { + return CounterSnapshot((*atomic.Int64)(c).Load()) +} diff --git a/common/metrics/counter_float64.go b/common/metrics/counter_float64.go new file mode 100644 index 00000000000..6cc73d89a29 --- /dev/null +++ b/common/metrics/counter_float64.go @@ -0,0 +1,66 @@ +package metrics + +import ( + "math" + "sync/atomic" +) + +// GetOrRegisterCounterFloat64 returns an existing *CounterFloat64 or constructs and registers +// a new CounterFloat64. +func GetOrRegisterCounterFloat64(name string, r Registry) *CounterFloat64 { + return getOrRegister(name, NewCounterFloat64, r) +} + +// NewCounterFloat64 constructs a new CounterFloat64. +func NewCounterFloat64() *CounterFloat64 { + return new(CounterFloat64) +} + +// NewRegisteredCounterFloat64 constructs and registers a new CounterFloat64. +func NewRegisteredCounterFloat64(name string, r Registry) *CounterFloat64 { + c := NewCounterFloat64() + if r == nil { + r = DefaultRegistry + } + r.Register(name, c) + return c +} + +// CounterFloat64Snapshot is a read-only copy of a float64 counter. +type CounterFloat64Snapshot float64 + +// Count returns the value at the time the snapshot was taken. +func (c CounterFloat64Snapshot) Count() float64 { return float64(c) } + +// CounterFloat64 holds a float64 value that can be incremented and decremented. +type CounterFloat64 atomic.Uint64 + +// Clear sets the counter to zero. +func (c *CounterFloat64) Clear() { + (*atomic.Uint64)(c).Store(0) +} + +// Dec decrements the counter by the given amount. +func (c *CounterFloat64) Dec(v float64) { + atomicAddFloat((*atomic.Uint64)(c), -v) +} + +// Inc increments the counter by the given amount. +func (c *CounterFloat64) Inc(v float64) { + atomicAddFloat((*atomic.Uint64)(c), v) +} + +// Snapshot returns a read-only copy of the counter. +func (c *CounterFloat64) Snapshot() CounterFloat64Snapshot { + return CounterFloat64Snapshot(math.Float64frombits((*atomic.Uint64)(c).Load())) +} + +func atomicAddFloat(fbits *atomic.Uint64, v float64) { + for { + loadedBits := fbits.Load() + newBits := math.Float64bits(math.Float64frombits(loadedBits) + v) + if fbits.CompareAndSwap(loadedBits, newBits) { + break + } + } +} diff --git a/common/metrics/counter_float_64_test.go b/common/metrics/counter_float_64_test.go new file mode 100644 index 00000000000..618cbbbc2b0 --- /dev/null +++ b/common/metrics/counter_float_64_test.go @@ -0,0 +1,73 @@ +package metrics + +import ( + "sync" + "testing" +) + +func BenchmarkCounterFloat64(b *testing.B) { + c := NewCounterFloat64() + b.ResetTimer() + for i := 0; i < b.N; i++ { + c.Inc(1.0) + } +} + +func BenchmarkCounterFloat64Parallel(b *testing.B) { + c := NewCounterFloat64() + b.ResetTimer() + var wg sync.WaitGroup + for i := 0; i < 10; i++ { + wg.Add(1) + go func() { + for i := 0; i < b.N; i++ { + c.Inc(1.0) + } + wg.Done() + }() + } + wg.Wait() + if have, want := c.Snapshot().Count(), 10.0*float64(b.N); have != want { + b.Fatalf("have %f want %f", have, want) + } +} + +func TestCounterFloat64(t *testing.T) { + c := NewCounterFloat64() + if count := c.Snapshot().Count(); count != 0 { + t.Errorf("wrong count: %v", count) + } + c.Dec(1.0) + if count := c.Snapshot().Count(); count != -1.0 { + t.Errorf("wrong count: %v", count) + } + snapshot := c.Snapshot() + c.Dec(2.0) + if count := c.Snapshot().Count(); count != -3.0 { + t.Errorf("wrong count: %v", count) + } + c.Inc(1.0) + if count := c.Snapshot().Count(); count != -2.0 { + t.Errorf("wrong count: %v", count) + } + c.Inc(2.0) + if count := c.Snapshot().Count(); count != 0.0 { + t.Errorf("wrong count: %v", count) + } + if count := snapshot.Count(); count != -1.0 { + t.Errorf("snapshot count wrong: %v", count) + } + c.Inc(1.0) + c.Clear() + if count := c.Snapshot().Count(); count != 0.0 { + t.Errorf("wrong count: %v", count) + } +} + +func TestGetOrRegisterCounterFloat64(t *testing.T) { + r := NewRegistry() + NewRegisteredCounterFloat64("foo", r).Inc(47.0) + if c := GetOrRegisterCounterFloat64("foo", r).Snapshot(); c.Count() != 47.0 { + t.Fatal(c) + } +} diff --git a/common/metrics/counter_test.go b/common/metrics/counter_test.go new file mode 100644 index 00000000000..bf0ca6bae44 --- /dev/null +++ b/common/metrics/counter_test.go @@ -0,0 +1,61 @@ +package metrics + +import "testing" + +func BenchmarkCounter(b *testing.B) { + c := NewCounter() + b.ResetTimer() + for i := 0; i < b.N; i++ { + c.Inc(1) + } +} + +func TestCounterClear(t *testing.T) { + c := NewCounter() + c.Inc(1) + c.Clear() + if count := c.Snapshot().Count(); count != 0 { + t.Errorf("c.Count(): 0 != %v\n", count) + } +} + +func TestCounter(t *testing.T) { + c := NewCounter() + if count := c.Snapshot().Count(); count != 0 { + t.Errorf("wrong count: %v", count) + } + c.Dec(1) + if count := c.Snapshot().Count(); count != -1 { + t.Errorf("wrong count: %v", count) + } + c.Dec(2) + if count := c.Snapshot().Count(); count != -3 { + t.Errorf("wrong count: %v", count) + } + c.Inc(1) + if count := c.Snapshot().Count(); count != -2 { + t.Errorf("wrong count: %v", count) + } + c.Inc(2) + if count := c.Snapshot().Count(); count != 0 { + t.Errorf("wrong count: %v", count) + } +} + +func TestCounterSnapshot(t *testing.T) { + c := NewCounter() + c.Inc(1) + snapshot := c.Snapshot() + c.Inc(1) + if count := snapshot.Count(); count != 1 { + t.Errorf("c.Count(): 1 != %v\n", count) + } +} + +func TestGetOrRegisterCounter(t *testing.T) { + r := NewRegistry() + NewRegisteredCounter("foo", r).Inc(47) + if c := GetOrRegisterCounter("foo", r).Snapshot(); c.Count() != 47 { + t.Fatal(c) + } +} diff --git a/common/metrics/cpu.go b/common/metrics/cpu.go new file mode 100644 index 00000000000..3a49cd42493 --- /dev/null +++ b/common/metrics/cpu.go @@ -0,0 +1,25 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package metrics + +// CPUStats is the system and process CPU stats. +// All values are in seconds. +type CPUStats struct { + GlobalTime float64 // Time spent by the CPU working on all processes + GlobalWait float64 // Time spent by waiting on disk for all processes + LocalTime float64 // Time spent by the CPU working on this process +} diff --git a/common/metrics/cpu_disabled.go b/common/metrics/cpu_disabled.go new file mode 100644 index 00000000000..37c3e1b8f7f --- /dev/null +++ b/common/metrics/cpu_disabled.go @@ -0,0 +1,24 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +//go:build ios || js || wasip1 || tinygo +// +build ios js wasip1 tinygo + +package metrics + +// ReadCPUStats retrieves the current CPU stats. Internally this uses `gosigar`, +// which is not supported on the platforms in this file. +func ReadCPUStats(stats *CPUStats) {} diff --git a/common/metrics/cpu_enabled.go b/common/metrics/cpu_enabled.go new file mode 100644 index 00000000000..7daadf1dc21 --- /dev/null +++ b/common/metrics/cpu_enabled.go @@ -0,0 +1,44 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +//go:build !ios && !js && !wasip1 && !tinygo +// +build !ios,!js,!wasip1,!tinygo + +package metrics + +import ( + "github.com/erigontech/erigon/common/log/v3" + "github.com/shirou/gopsutil/cpu" +) + +// ReadCPUStats retrieves the current CPU stats. +func ReadCPUStats(stats *CPUStats) { + // passing false to request all cpu times + timeStats, err := cpu.Times(false) + if err != nil { + log.Error("Could not read cpu stats", "err", err) + return + } + if len(timeStats) == 0 { + log.Error("Empty cpu stats") + return + } + // requesting all cpu times will always return an array with only one time stats entry + timeStat := timeStats[0] + stats.GlobalTime = timeStat.User + timeStat.Nice + timeStat.System + stats.GlobalWait = timeStat.Iowait + stats.LocalTime = getProcessCPUTime() +} diff --git a/common/metrics/cputime_nop.go b/common/metrics/cputime_nop.go new file mode 100644 index 00000000000..a6285ec10ad --- /dev/null +++ b/common/metrics/cputime_nop.go @@ -0,0 +1,26 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +//go:build windows || js || tinygo +// +build windows js tinygo + +package metrics + +// getProcessCPUTime returns 0 on Windows as there is no system call to resolve +// the actual process' CPU time. +func getProcessCPUTime() float64 { + return 0 +} diff --git a/common/metrics/cputime_unix.go b/common/metrics/cputime_unix.go new file mode 100644 index 00000000000..970d3076747 --- /dev/null +++ b/common/metrics/cputime_unix.go @@ -0,0 +1,36 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +//go:build !windows && !js && !wasip1 && !tinygo +// +build !windows,!js,!wasip1,!tinygo + +package metrics + +import ( + syscall "golang.org/x/sys/unix" + + "github.com/erigontech/erigon/common/log/v3" +) + +// getProcessCPUTime retrieves the process' CPU time since program startup. +func getProcessCPUTime() float64 { + var usage syscall.Rusage + if err := syscall.Getrusage(syscall.RUSAGE_SELF, &usage); err != nil { + log.Warn("Failed to retrieve CPU time", "err", err) + return 0 + } + return float64(usage.Utime.Sec+usage.Stime.Sec) + float64(usage.Utime.Usec+usage.Stime.Usec)/1000000 //nolint:unconvert +} diff --git a/common/metrics/debug.go b/common/metrics/debug.go new file mode 100644 index 00000000000..5d0d3992f10 --- /dev/null +++ b/common/metrics/debug.go @@ -0,0 +1,76 @@ +package metrics + +import ( + "runtime/debug" + "time" +) + +var ( + debugMetrics struct { + GCStats struct { + LastGC *Gauge + NumGC *Gauge + Pause Histogram + //PauseQuantiles Histogram + PauseTotal *Gauge + } + ReadGCStats *Timer + } + gcStats debug.GCStats +) + +// CaptureDebugGCStats captures new values for the Go garbage collector statistics +// exported in debug.GCStats. This is designed to be called as a goroutine. +func CaptureDebugGCStats(r Registry, d time.Duration) { + for range time.Tick(d) { + CaptureDebugGCStatsOnce(r) + } +} + +// CaptureDebugGCStatsOnce captures new values for the Go garbage collector +// statistics exported in debug.GCStats. This is designed to be called in +// a background goroutine. Giving a registry which has not been given to +// RegisterDebugGCStats will panic. +// +// Be careful (but much less so) with this because debug.ReadGCStats calls +// the C function runtime·lock(runtime·mheap) which, while not a stop-the-world +// operation, isn't something you want to be doing all the time. +func CaptureDebugGCStatsOnce(r Registry) { + lastGC := gcStats.LastGC + t := time.Now() + debug.ReadGCStats(&gcStats) + debugMetrics.ReadGCStats.UpdateSince(t) + + debugMetrics.GCStats.LastGC.Update(gcStats.LastGC.UnixNano()) + debugMetrics.GCStats.NumGC.Update(gcStats.NumGC) + if lastGC != gcStats.LastGC && 0 < len(gcStats.Pause) { + debugMetrics.GCStats.Pause.Update(int64(gcStats.Pause[0])) + } + //debugMetrics.GCStats.PauseQuantiles.Update(gcStats.PauseQuantiles) + debugMetrics.GCStats.PauseTotal.Update(int64(gcStats.PauseTotal)) +} + +// RegisterDebugGCStats registers metrics for the Go garbage collector statistics +// exported in debug.GCStats. The metrics are named by their fully-qualified Go +// symbols, i.e. debug.GCStats.PauseTotal. +func RegisterDebugGCStats(r Registry) { + debugMetrics.GCStats.LastGC = NewGauge() + debugMetrics.GCStats.NumGC = NewGauge() + debugMetrics.GCStats.Pause = NewHistogram(NewExpDecaySample(1028, 0.015)) + //debugMetrics.GCStats.PauseQuantiles = NewHistogram(NewExpDecaySample(1028, 0.015)) + debugMetrics.GCStats.PauseTotal = NewGauge() + debugMetrics.ReadGCStats = NewTimer() + + r.Register("debug.GCStats.LastGC", debugMetrics.GCStats.LastGC) + r.Register("debug.GCStats.NumGC", debugMetrics.GCStats.NumGC) + r.Register("debug.GCStats.Pause", debugMetrics.GCStats.Pause) + //r.Register("debug.GCStats.PauseQuantiles", debugMetrics.GCStats.PauseQuantiles) + r.Register("debug.GCStats.PauseTotal", debugMetrics.GCStats.PauseTotal) + r.Register("debug.ReadGCStats", debugMetrics.ReadGCStats) +} + +// Allocate an initial slice for gcStats.Pause to avoid allocations during +// normal operation. +func init() { + gcStats.Pause = make([]time.Duration, 11) +} diff --git a/common/metrics/debug_test.go b/common/metrics/debug_test.go new file mode 100644 index 00000000000..07eb8678416 --- /dev/null +++ b/common/metrics/debug_test.go @@ -0,0 +1,48 @@ +package metrics + +import ( + "runtime" + "runtime/debug" + "testing" + "time" +) + +func BenchmarkDebugGCStats(b *testing.B) { + r := NewRegistry() + RegisterDebugGCStats(r) + b.ResetTimer() + for i := 0; i < b.N; i++ { + CaptureDebugGCStatsOnce(r) + } +} + +func TestDebugGCStatsBlocking(t *testing.T) { + if g := runtime.GOMAXPROCS(0); g < 2 { + t.Skipf("skipping TestDebugGCMemStatsBlocking with GOMAXPROCS=%d\n", g) + return + } + ch := make(chan int) + go testDebugGCStatsBlocking(ch) + var gcStats debug.GCStats + t0 := time.Now() + debug.ReadGCStats(&gcStats) + t1 := time.Now() + t.Log("i++ during debug.ReadGCStats:", <-ch) + go testDebugGCStatsBlocking(ch) + d := t1.Sub(t0) + t.Log(d) + time.Sleep(d) + t.Log("i++ during time.Sleep:", <-ch) +} + +func testDebugGCStatsBlocking(ch chan int) { + i := 0 + for { + select { + case ch <- i: + return + default: + i++ + } + } +} diff --git a/common/metrics/disk.go b/common/metrics/disk.go new file mode 100644 index 00000000000..25142d2ad1e --- /dev/null +++ b/common/metrics/disk.go @@ -0,0 +1,25 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package metrics + +// DiskStats is the per process disk io stats. +type DiskStats struct { + ReadCount int64 // Number of read operations executed + ReadBytes int64 // Total number of bytes read + WriteCount int64 // Number of write operations executed + WriteBytes int64 // Total number of byte written +} diff --git a/common/metrics/disk_linux.go b/common/metrics/disk_linux.go new file mode 100644 index 00000000000..8d610cd6749 --- /dev/null +++ b/common/metrics/disk_linux.go @@ -0,0 +1,72 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains the Linux implementation of process disk IO counter retrieval. + +package metrics + +import ( + "bufio" + "fmt" + "io" + "os" + "strconv" + "strings" +) + +// ReadDiskStats retrieves the disk IO stats belonging to the current process. +func ReadDiskStats(stats *DiskStats) error { + // Open the process disk IO counter file + inf, err := os.Open(fmt.Sprintf("/proc/%d/io", os.Getpid())) + if err != nil { + return err + } + defer inf.Close() + in := bufio.NewReader(inf) + + // Iterate over the IO counter, and extract what we need + for { + // Read the next line and split to key and value + line, err := in.ReadString('\n') + if err != nil { + if err == io.EOF { + return nil + } + return err + } + parts := strings.Split(line, ":") + if len(parts) != 2 { + continue + } + key := strings.TrimSpace(parts[0]) + value, err := strconv.ParseInt(strings.TrimSpace(parts[1]), 10, 64) + if err != nil { + return err + } + + // Update the counter based on the key + switch key { + case "syscr": + stats.ReadCount = value + case "syscw": + stats.WriteCount = value + case "rchar": + stats.ReadBytes = value + case "wchar": + stats.WriteBytes = value + } + } +} diff --git a/common/metrics/disk_nop.go b/common/metrics/disk_nop.go new file mode 100644 index 00000000000..41bbe9adb2d --- /dev/null +++ b/common/metrics/disk_nop.go @@ -0,0 +1,27 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +//go:build !linux +// +build !linux + +package metrics + +import "errors" + +// ReadDiskStats retrieves the disk IO stats belonging to the current process. +func ReadDiskStats(stats *DiskStats) error { + return errors.New("not implemented") +} diff --git a/common/metrics/ewma.go b/common/metrics/ewma.go new file mode 100644 index 00000000000..194527a7989 --- /dev/null +++ b/common/metrics/ewma.go @@ -0,0 +1,91 @@ +package metrics + +import ( + "math" + "sync" + "sync/atomic" + "time" +) + +// EWMASnapshot is a read-only copy of an EWMA. +type EWMASnapshot float64 + +// Rate returns the rate of events per second at the time the snapshot was +// taken. +func (a EWMASnapshot) Rate() float64 { return float64(a) } + +// NewEWMA constructs a new EWMA with the given alpha. +func NewEWMA(alpha float64) *EWMA { + return &EWMA{alpha: alpha} +} + +// NewEWMA1 constructs a new EWMA for a one-minute moving average. +func NewEWMA1() *EWMA { + return NewEWMA(1 - math.Exp(-5.0/60.0/1)) +} + +// NewEWMA5 constructs a new EWMA for a five-minute moving average. +func NewEWMA5() *EWMA { + return NewEWMA(1 - math.Exp(-5.0/60.0/5)) +} + +// NewEWMA15 constructs a new EWMA for a fifteen-minute moving average. +func NewEWMA15() *EWMA { + return NewEWMA(1 - math.Exp(-5.0/60.0/15)) +} + +// EWMA continuously calculate an exponentially-weighted moving average +// based on an outside source of clock ticks. +type EWMA struct { + uncounted atomic.Int64 + alpha float64 + rate atomic.Uint64 + init atomic.Bool + mutex sync.Mutex +} + +// Snapshot returns a read-only copy of the EWMA. +func (a *EWMA) Snapshot() EWMASnapshot { + r := math.Float64frombits(a.rate.Load()) * float64(time.Second) + return EWMASnapshot(r) +} + +// tick ticks the clock to update the moving average. It assumes it is called +// every five seconds. +func (a *EWMA) tick() { + // Optimization to avoid mutex locking in the hot-path. + if a.init.Load() { + a.updateRate(a.fetchInstantRate()) + return + } + // Slow-path: this is only needed on the first tick() and preserves transactional updating + // of init and rate in the else block. The first conditional is needed below because + // a different thread could have set a.init = 1 between the time of the first atomic load and when + // the lock was acquired. + a.mutex.Lock() + if a.init.Load() { + // The fetchInstantRate() uses atomic loading, which is unnecessary in this critical section + // but again, this section is only invoked on the first successful tick() operation. + a.updateRate(a.fetchInstantRate()) + } else { + a.init.Store(true) + a.rate.Store(math.Float64bits(a.fetchInstantRate())) + } + a.mutex.Unlock() +} + +func (a *EWMA) fetchInstantRate() float64 { + count := a.uncounted.Swap(0) + return float64(count) / float64(5*time.Second) +} + +func (a *EWMA) updateRate(instantRate float64) { + currentRate := math.Float64frombits(a.rate.Load()) + currentRate += a.alpha * (instantRate - currentRate) + a.rate.Store(math.Float64bits(currentRate)) +} + +// Update adds n uncounted events. +func (a *EWMA) Update(n int64) { + a.uncounted.Add(n) +} diff --git a/common/metrics/ewma_test.go b/common/metrics/ewma_test.go new file mode 100644 index 00000000000..4b9bde3a4b3 --- /dev/null +++ b/common/metrics/ewma_test.go @@ -0,0 +1,89 @@ +package metrics + +import ( + "math" + "testing" +) + +const epsilon = 0.0000000000000001 + +func BenchmarkEWMA(b *testing.B) { + a := NewEWMA1() + b.ResetTimer() + for i := 0; i < b.N; i++ { + a.Update(1) + a.tick() + } +} + +func BenchmarkEWMAParallel(b *testing.B) { + a := NewEWMA1() + b.ResetTimer() + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + a.Update(1) + a.tick() + } + }) +} + +func TestEWMA1(t *testing.T) { + a := NewEWMA1() + a.Update(3) + a.tick() + for i, want := range []float64{0.6, + 0.22072766470286553, 0.08120116994196772, 0.029872241020718428, + 0.01098938333324054, 0.004042768199451294, 0.0014872513059998212, + 0.0005471291793327122, 0.00020127757674150815, 7.404588245200814e-05, + 2.7239957857491083e-05, 1.0021020474147462e-05, 3.6865274119969525e-06, + 1.3561976441886433e-06, 4.989172314621449e-07, 1.8354139230109722e-07, + } { + if rate := a.Snapshot().Rate(); math.Abs(want-rate) > epsilon { + t.Errorf("%d minute a.Snapshot().Rate(): %f != %v\n", i, want, rate) + } + elapseMinute(a) + } +} + +func TestEWMA5(t *testing.T) { + a := NewEWMA5() + a.Update(3) + a.tick() + for i, want := range []float64{ + 0.6, 0.49123845184678905, 0.4021920276213837, 0.32928698165641596, + 0.269597378470333, 0.2207276647028654, 0.18071652714732128, + 0.14795817836496392, 0.12113791079679326, 0.09917933293295193, + 0.08120116994196763, 0.06648189501740036, 0.05443077197364752, + 0.04456414692860035, 0.03648603757513079, 0.0298722410207183831020718428, + } { + if rate := a.Snapshot().Rate(); math.Abs(want-rate) > epsilon { + t.Errorf("%d minute a.Snapshot().Rate(): %f != %v\n", i, want, rate) + } + elapseMinute(a) + } +} + +func TestEWMA15(t *testing.T) { + a := NewEWMA15() + a.Update(3) + a.tick() + for i, want := range []float64{ + 0.6, 0.5613041910189706, 0.5251039914257684, 0.4912384518467888184678905, + 0.459557003018789, 0.4299187863442732, 0.4021920276213831, + 0.37625345116383313, 0.3519877317060185, 0.3292869816564153165641596, + 0.3080502714195546, 0.2881831806538789, 0.26959737847033216, + 0.2522102307052083, 0.23594443252115815, 0.2207276647028646247028654470286553, + } { + if rate := a.Snapshot().Rate(); math.Abs(want-rate) > epsilon { + t.Errorf("%d minute a.Snapshot().Rate(): %f != %v\n", i, want, rate) + } + elapseMinute(a) + } +} + +func elapseMinute(a *EWMA) { + for i := 0; i < 12; i++ { + a.tick() + } +} diff --git a/common/metrics/gauge.go b/common/metrics/gauge.go new file mode 100644 index 00000000000..20de95255bd --- /dev/null +++ b/common/metrics/gauge.go @@ -0,0 +1,67 @@ +package metrics + +import "sync/atomic" + +// GaugeSnapshot is a read-only copy of a Gauge. +type GaugeSnapshot int64 + +// Value returns the value at the time the snapshot was taken. +func (g GaugeSnapshot) Value() int64 { return int64(g) } + +// GetOrRegisterGauge returns an existing Gauge or constructs and registers a +// new Gauge. +func GetOrRegisterGauge(name string, r Registry) *Gauge { + return getOrRegister(name, NewGauge, r) +} + +// NewGauge constructs a new Gauge. +func NewGauge() *Gauge { + return &Gauge{} +} + +// NewRegisteredGauge constructs and registers a new Gauge. +func NewRegisteredGauge(name string, r Registry) *Gauge { + c := NewGauge() + if r == nil { + r = DefaultRegistry + } + r.Register(name, c) + return c +} + +// Gauge holds an int64 value that can be set arbitrarily. +type Gauge atomic.Int64 + +// Snapshot returns a read-only copy of the gauge. +func (g *Gauge) Snapshot() GaugeSnapshot { + return GaugeSnapshot((*atomic.Int64)(g).Load()) +} + +// Update updates the gauge's value. +func (g *Gauge) Update(v int64) { + (*atomic.Int64)(g).Store(v) +} + +// UpdateIfGt updates the gauge's value if v is larger then the current value. +func (g *Gauge) UpdateIfGt(v int64) { + value := (*atomic.Int64)(g) + for { + exist := value.Load() + if exist >= v { + break + } + if value.CompareAndSwap(exist, v) { + break + } + } +} + +// Dec decrements the gauge's current value by the given amount. +func (g *Gauge) Dec(i int64) { + (*atomic.Int64)(g).Add(-i) +} + +// Inc increments the gauge's current value by the given amount. +func (g *Gauge) Inc(i int64) { + (*atomic.Int64)(g).Add(i) +} diff --git a/common/metrics/gauge_float64.go b/common/metrics/gauge_float64.go new file mode 100644 index 00000000000..48524e4c3f7 --- /dev/null +++ b/common/metrics/gauge_float64.go @@ -0,0 +1,47 @@ +package metrics + +import ( + "math" + "sync/atomic" +) + +// GetOrRegisterGaugeFloat64 returns an existing GaugeFloat64 or constructs and registers a +// new GaugeFloat64. +func GetOrRegisterGaugeFloat64(name string, r Registry) *GaugeFloat64 { + return getOrRegister(name, NewGaugeFloat64, r) +} + +// GaugeFloat64Snapshot is a read-only copy of a GaugeFloat64. +type GaugeFloat64Snapshot float64 + +// Value returns the value at the time the snapshot was taken. +func (g GaugeFloat64Snapshot) Value() float64 { return float64(g) } + +// NewGaugeFloat64 constructs a new GaugeFloat64. +func NewGaugeFloat64() *GaugeFloat64 { + return new(GaugeFloat64) +} + +// NewRegisteredGaugeFloat64 constructs and registers a new GaugeFloat64. +func NewRegisteredGaugeFloat64(name string, r Registry) *GaugeFloat64 { + c := NewGaugeFloat64() + if nil == r { + r = DefaultRegistry + } + r.Register(name, c) + return c +} + +// GaugeFloat64 hold a float64 value that can be set arbitrarily. +type GaugeFloat64 atomic.Uint64 + +// Snapshot returns a read-only copy of the gauge. +func (g *GaugeFloat64) Snapshot() GaugeFloat64Snapshot { + v := math.Float64frombits((*atomic.Uint64)(g).Load()) + return GaugeFloat64Snapshot(v) +} + +// Update updates the gauge's value. +func (g *GaugeFloat64) Update(v float64) { + (*atomic.Uint64)(g).Store(math.Float64bits(v)) +} diff --git a/common/metrics/gauge_float64_test.go b/common/metrics/gauge_float64_test.go new file mode 100644 index 00000000000..194a18821f8 --- /dev/null +++ b/common/metrics/gauge_float64_test.go @@ -0,0 +1,51 @@ +package metrics + +import ( + "sync" + "testing" +) + +func BenchmarkGaugeFloat64(b *testing.B) { + g := NewGaugeFloat64() + b.ResetTimer() + for i := 0; i < b.N; i++ { + g.Update(float64(i)) + } +} + +func BenchmarkGaugeFloat64Parallel(b *testing.B) { + c := NewGaugeFloat64() + var wg sync.WaitGroup + for i := 0; i < 10; i++ { + wg.Add(1) + go func() { + for i := 0; i < b.N; i++ { + c.Update(float64(i)) + } + wg.Done() + }() + } + wg.Wait() + if have, want := c.Snapshot().Value(), float64(b.N-1); have != want { + b.Fatalf("have %f want %f", have, want) + } +} + +func TestGaugeFloat64Snapshot(t *testing.T) { + g := NewGaugeFloat64() + g.Update(47.0) + snapshot := g.Snapshot() + g.Update(float64(0)) + if v := snapshot.Value(); v != 47.0 { + t.Errorf("g.Value(): 47.0 != %v\n", v) + } +} + +func TestGetOrRegisterGaugeFloat64(t *testing.T) { + r := NewRegistry() + NewRegisteredGaugeFloat64("foo", r).Update(47.0) + t.Logf("registry: %v", r) + if g := GetOrRegisterGaugeFloat64("foo", r).Snapshot(); g.Value() != 47.0 { + t.Fatal(g) + } +} diff --git a/common/metrics/gauge_info.go b/common/metrics/gauge_info.go new file mode 100644 index 00000000000..34ac9179194 --- /dev/null +++ b/common/metrics/gauge_info.go @@ -0,0 +1,61 @@ +package metrics + +import ( + "encoding/json" + "sync" +) + +// GaugeInfoValue is a mapping of keys to values +type GaugeInfoValue map[string]string + +func (val GaugeInfoValue) String() string { + data, _ := json.Marshal(val) + return string(data) +} + +// GetOrRegisterGaugeInfo returns an existing GaugeInfo or constructs and registers a +// new GaugeInfo. +func GetOrRegisterGaugeInfo(name string, r Registry) *GaugeInfo { + return getOrRegister(name, NewGaugeInfo, r) +} + +// NewGaugeInfo constructs a new GaugeInfo. +func NewGaugeInfo() *GaugeInfo { + return &GaugeInfo{ + value: GaugeInfoValue{}, + } +} + +// NewRegisteredGaugeInfo constructs and registers a new GaugeInfo. +func NewRegisteredGaugeInfo(name string, r Registry) *GaugeInfo { + c := NewGaugeInfo() + if nil == r { + r = DefaultRegistry + } + r.Register(name, c) + return c +} + +// GaugeInfoSnapshot is a read-only copy of another GaugeInfo. +type GaugeInfoSnapshot GaugeInfoValue + +// Value returns the value at the time the snapshot was taken. +func (g GaugeInfoSnapshot) Value() GaugeInfoValue { return GaugeInfoValue(g) } + +// GaugeInfo maintains a set of key/value mappings. +type GaugeInfo struct { + mutex sync.Mutex + value GaugeInfoValue +} + +// Snapshot returns a read-only copy of the gauge. +func (g *GaugeInfo) Snapshot() GaugeInfoSnapshot { + return GaugeInfoSnapshot(g.value) +} + +// Update updates the gauge's value. +func (g *GaugeInfo) Update(v GaugeInfoValue) { + g.mutex.Lock() + defer g.mutex.Unlock() + g.value = v +} diff --git a/common/metrics/gauge_info_test.go b/common/metrics/gauge_info_test.go new file mode 100644 index 00000000000..319afbf92e8 --- /dev/null +++ b/common/metrics/gauge_info_test.go @@ -0,0 +1,36 @@ +package metrics + +import ( + "testing" +) + +func TestGaugeInfoJsonString(t *testing.T) { + g := NewGaugeInfo() + g.Update(GaugeInfoValue{ + "chain_id": "5", + "anotherKey": "any_string_value", + "third_key": "anything", + }, + ) + want := `{"anotherKey":"any_string_value","chain_id":"5","third_key":"anything"}` + + original := g.Snapshot() + g.Update(GaugeInfoValue{"value": "updated"}) + + if have := original.Value().String(); have != want { + t.Errorf("\nhave: %v\nwant: %v\n", have, want) + } + if have, want := g.Snapshot().Value().String(), `{"value":"updated"}`; have != want { + t.Errorf("\nhave: %v\nwant: %v\n", have, want) + } +} + +func TestGetOrRegisterGaugeInfo(t *testing.T) { + r := NewRegistry() + NewRegisteredGaugeInfo("foo", r).Update( + GaugeInfoValue{"chain_id": "5"}) + g := GetOrRegisterGaugeInfo("foo", r).Snapshot() + if have, want := g.Value().String(), `{"chain_id":"5"}`; have != want { + t.Errorf("have\n%v\nwant\n%v\n", have, want) + } +} diff --git a/common/metrics/gauge_test.go b/common/metrics/gauge_test.go new file mode 100644 index 00000000000..f2ba930bc46 --- /dev/null +++ b/common/metrics/gauge_test.go @@ -0,0 +1,31 @@ +package metrics + +import ( + "testing" +) + +func BenchmarkGauge(b *testing.B) { + g := NewGauge() + b.ResetTimer() + for i := 0; i < b.N; i++ { + g.Update(int64(i)) + } +} + +func TestGaugeSnapshot(t *testing.T) { + g := NewGauge() + g.Update(int64(47)) + snapshot := g.Snapshot() + g.Update(int64(0)) + if v := snapshot.Value(); v != 47 { + t.Errorf("g.Value(): 47 != %v\n", v) + } +} + +func TestGetOrRegisterGauge(t *testing.T) { + r := NewRegistry() + NewRegisteredGauge("foo", r).Update(47) + if g := GetOrRegisterGauge("foo", r); g.Snapshot().Value() != 47 { + t.Fatal(g) + } +} diff --git a/common/metrics/healthcheck.go b/common/metrics/healthcheck.go new file mode 100644 index 00000000000..435e5e0bf93 --- /dev/null +++ b/common/metrics/healthcheck.go @@ -0,0 +1,35 @@ +package metrics + +// NewHealthcheck constructs a new Healthcheck which will use the given +// function to update its status. +func NewHealthcheck(f func(*Healthcheck)) *Healthcheck { + return &Healthcheck{nil, f} +} + +// Healthcheck is the standard implementation of a Healthcheck and +// stores the status and a function to call to update the status. +type Healthcheck struct { + err error + f func(*Healthcheck) +} + +// Check runs the healthcheck function to update the healthcheck's status. +func (h *Healthcheck) Check() { + h.f(h) +} + +// Error returns the healthcheck's status, which will be nil if it is healthy. +func (h *Healthcheck) Error() error { + return h.err +} + +// Healthy marks the healthcheck as healthy. +func (h *Healthcheck) Healthy() { + h.err = nil +} + +// Unhealthy marks the healthcheck as unhealthy. The error is stored and +// may be retrieved by the Error method. +func (h *Healthcheck) Unhealthy(err error) { + h.err = err +} diff --git a/common/metrics/histogram.go b/common/metrics/histogram.go new file mode 100644 index 00000000000..18bf6e3d2b7 --- /dev/null +++ b/common/metrics/histogram.go @@ -0,0 +1,66 @@ +package metrics + +type HistogramSnapshot interface { + Count() int64 + Max() int64 + Mean() float64 + Min() int64 + Percentile(float64) float64 + Percentiles([]float64) []float64 + Size() int + StdDev() float64 + Sum() int64 + Variance() float64 +} + +// Histogram calculates distribution statistics from a series of int64 values. +type Histogram interface { + Clear() + Update(int64) + Snapshot() HistogramSnapshot +} + +// GetOrRegisterHistogram returns an existing Histogram or constructs and +// registers a new StandardHistogram. +func GetOrRegisterHistogram(name string, r Registry, s Sample) Histogram { + return getOrRegister(name, func() Histogram { return NewHistogram(s) }, r) +} + +// GetOrRegisterHistogramLazy returns an existing Histogram or constructs and +// registers a new StandardHistogram. +func GetOrRegisterHistogramLazy(name string, r Registry, s func() Sample) Histogram { + return getOrRegister(name, func() Histogram { return NewHistogram(s()) }, r) +} + +// NewHistogram constructs a new StandardHistogram from a Sample. +func NewHistogram(s Sample) Histogram { + return &StandardHistogram{s} +} + +// NewRegisteredHistogram constructs and registers a new StandardHistogram from +// a Sample. +func NewRegisteredHistogram(name string, r Registry, s Sample) Histogram { + c := NewHistogram(s) + if nil == r { + r = DefaultRegistry + } + r.Register(name, c) + return c +} + +// StandardHistogram is the standard implementation of a Histogram and uses a +// Sample to bound its memory use. +type StandardHistogram struct { + sample Sample +} + +// Clear clears the histogram and its sample. +func (h *StandardHistogram) Clear() { h.sample.Clear() } + +// Snapshot returns a read-only copy of the histogram. +func (h *StandardHistogram) Snapshot() HistogramSnapshot { + return h.sample.Snapshot() +} + +// Update samples a new value. +func (h *StandardHistogram) Update(v int64) { h.sample.Update(v) } diff --git a/common/metrics/histogram_test.go b/common/metrics/histogram_test.go new file mode 100644 index 00000000000..22fc5468b0b --- /dev/null +++ b/common/metrics/histogram_test.go @@ -0,0 +1,95 @@ +package metrics + +import "testing" + +func BenchmarkHistogram(b *testing.B) { + h := NewHistogram(NewUniformSample(100)) + b.ResetTimer() + for i := 0; i < b.N; i++ { + h.Update(int64(i)) + } +} + +func TestGetOrRegisterHistogram(t *testing.T) { + r := NewRegistry() + s := NewUniformSample(100) + NewRegisteredHistogram("foo", r, s).Update(47) + if h := GetOrRegisterHistogram("foo", r, s).Snapshot(); h.Count() != 1 { + t.Fatal(h) + } +} + +func TestHistogram10000(t *testing.T) { + h := NewHistogram(NewUniformSample(100000)) + for i := 1; i <= 10000; i++ { + h.Update(int64(i)) + } + testHistogram10000(t, h.Snapshot()) +} + +func TestHistogramEmpty(t *testing.T) { + h := NewHistogram(NewUniformSample(100)).Snapshot() + if count := h.Count(); count != 0 { + t.Errorf("h.Count(): 0 != %v\n", count) + } + if min := h.Min(); min != 0 { + t.Errorf("h.Min(): 0 != %v\n", min) + } + if max := h.Max(); max != 0 { + t.Errorf("h.Max(): 0 != %v\n", max) + } + if mean := h.Mean(); mean != 0.0 { + t.Errorf("h.Mean(): 0.0 != %v\n", mean) + } + if stdDev := h.StdDev(); stdDev != 0.0 { + t.Errorf("h.StdDev(): 0.0 != %v\n", stdDev) + } + ps := h.Percentiles([]float64{0.5, 0.75, 0.99}) + if ps[0] != 0.0 { + t.Errorf("median: 0.0 != %v\n", ps[0]) + } + if ps[1] != 0.0 { + t.Errorf("75th percentile: 0.0 != %v\n", ps[1]) + } + if ps[2] != 0.0 { + t.Errorf("99th percentile: 0.0 != %v\n", ps[2]) + } +} + +func TestHistogramSnapshot(t *testing.T) { + h := NewHistogram(NewUniformSample(100000)) + for i := 1; i <= 10000; i++ { + h.Update(int64(i)) + } + snapshot := h.Snapshot() + h.Update(0) + testHistogram10000(t, snapshot) +} + +func testHistogram10000(t *testing.T, h HistogramSnapshot) { + if count := h.Count(); count != 10000 { + t.Errorf("h.Count(): 10000 != %v\n", count) + } + if min := h.Min(); min != 1 { + t.Errorf("h.Min(): 1 != %v\n", min) + } + if max := h.Max(); max != 10000 { + t.Errorf("h.Max(): 10000 != %v\n", max) + } + if mean := h.Mean(); mean != 5000.5 { + t.Errorf("h.Mean(): 5000.5 != %v\n", mean) + } + if stdDev := h.StdDev(); stdDev != 2886.751331514372 { + t.Errorf("h.StdDev(): 2886.751331514372 != %v\n", stdDev) + } + ps := h.Percentiles([]float64{0.5, 0.75, 0.99}) + if ps[0] != 5000.5 { + t.Errorf("median: 5000.5 != %v\n", ps[0]) + } + if ps[1] != 7500.75 { + t.Errorf("75th percentile: 7500.75 != %v\n", ps[1]) + } + if ps[2] != 9900.99 { + t.Errorf("99th percentile: 9900.99 != %v\n", ps[2]) + } +} diff --git a/common/metrics/init_test.go b/common/metrics/init_test.go new file mode 100644 index 00000000000..af75bee425b --- /dev/null +++ b/common/metrics/init_test.go @@ -0,0 +1,5 @@ +package metrics + +func init() { + metricsEnabled = true +} diff --git a/common/metrics/json.go b/common/metrics/json.go new file mode 100644 index 00000000000..6b134d477b6 --- /dev/null +++ b/common/metrics/json.go @@ -0,0 +1,31 @@ +package metrics + +import ( + "encoding/json" + "io" + "time" +) + +// MarshalJSON returns a byte slice containing a JSON representation of all +// the metrics in the Registry. +func (r *StandardRegistry) MarshalJSON() ([]byte, error) { + return json.Marshal(r.GetAll()) +} + +// WriteJSON writes metrics from the given registry periodically to the +// specified io.Writer as JSON. +func WriteJSON(r Registry, d time.Duration, w io.Writer) { + for range time.Tick(d) { + WriteJSONOnce(r, w) + } +} + +// WriteJSONOnce writes metrics from the given registry to the specified +// io.Writer as JSON. +func WriteJSONOnce(r Registry, w io.Writer) { + json.NewEncoder(w).Encode(r) +} + +func (r *PrefixedRegistry) MarshalJSON() ([]byte, error) { + return json.Marshal(r.GetAll()) +} diff --git a/common/metrics/json_test.go b/common/metrics/json_test.go new file mode 100644 index 00000000000..811bc29f11e --- /dev/null +++ b/common/metrics/json_test.go @@ -0,0 +1,28 @@ +package metrics + +import ( + "bytes" + "encoding/json" + "testing" +) + +func TestRegistryMarshallJSON(t *testing.T) { + b := &bytes.Buffer{} + enc := json.NewEncoder(b) + r := NewRegistry() + r.Register("counter", NewCounter()) + enc.Encode(r) + if s := b.String(); s != "{\"counter\":{\"count\":0}}\n" { + t.Fatal(s) + } +} + +func TestRegistryWriteJSONOnce(t *testing.T) { + r := NewRegistry() + r.Register("counter", NewCounter()) + b := &bytes.Buffer{} + WriteJSONOnce(r, b) + if s := b.String(); s != "{\"counter\":{\"count\":0}}\n" { + t.Fail() + } +} diff --git a/common/metrics/log.go b/common/metrics/log.go new file mode 100644 index 00000000000..08f3effb81c --- /dev/null +++ b/common/metrics/log.go @@ -0,0 +1,82 @@ +package metrics + +import ( + "time" +) + +type Logger interface { + Printf(format string, v ...interface{}) +} + +func Log(r Registry, freq time.Duration, l Logger) { + LogScaled(r, freq, time.Nanosecond, l) +} + +// LogScaled outputs each metric in the given registry periodically using the given +// logger. Print timings in `scale` units (eg time.Millisecond) rather than nanos. +func LogScaled(r Registry, freq time.Duration, scale time.Duration, l Logger) { + du := float64(scale) + duSuffix := scale.String()[1:] + + for range time.Tick(freq) { + r.Each(func(name string, i interface{}) { + switch metric := i.(type) { + case *Counter: + l.Printf("counter %s\n", name) + l.Printf(" count: %9d\n", metric.Snapshot().Count()) + case *CounterFloat64: + l.Printf("counter %s\n", name) + l.Printf(" count: %f\n", metric.Snapshot().Count()) + case *Gauge: + l.Printf("gauge %s\n", name) + l.Printf(" value: %9d\n", metric.Snapshot().Value()) + case *GaugeFloat64: + l.Printf("gauge %s\n", name) + l.Printf(" value: %f\n", metric.Snapshot().Value()) + case *GaugeInfo: + l.Printf("gauge %s\n", name) + l.Printf(" value: %s\n", metric.Snapshot().Value()) + case Histogram: + h := metric.Snapshot() + ps := h.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999}) + l.Printf("histogram %s\n", name) + l.Printf(" count: %9d\n", h.Count()) + l.Printf(" min: %9d\n", h.Min()) + l.Printf(" max: %9d\n", h.Max()) + l.Printf(" mean: %12.2f\n", h.Mean()) + l.Printf(" stddev: %12.2f\n", h.StdDev()) + l.Printf(" median: %12.2f\n", ps[0]) + l.Printf(" 75%%: %12.2f\n", ps[1]) + l.Printf(" 95%%: %12.2f\n", ps[2]) + l.Printf(" 99%%: %12.2f\n", ps[3]) + l.Printf(" 99.9%%: %12.2f\n", ps[4]) + case *Meter: + m := metric.Snapshot() + l.Printf("meter %s\n", name) + l.Printf(" count: %9d\n", m.Count()) + l.Printf(" 1-min rate: %12.2f\n", m.Rate1()) + l.Printf(" 5-min rate: %12.2f\n", m.Rate5()) + l.Printf(" 15-min rate: %12.2f\n", m.Rate15()) + l.Printf(" mean rate: %12.2f\n", m.RateMean()) + case *Timer: + t := metric.Snapshot() + ps := t.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999}) + l.Printf("timer %s\n", name) + l.Printf(" count: %9d\n", t.Count()) + l.Printf(" min: %12.2f%s\n", float64(t.Min())/du, duSuffix) + l.Printf(" max: %12.2f%s\n", float64(t.Max())/du, duSuffix) + l.Printf(" mean: %12.2f%s\n", t.Mean()/du, duSuffix) + l.Printf(" stddev: %12.2f%s\n", t.StdDev()/du, duSuffix) + l.Printf(" median: %12.2f%s\n", ps[0]/du, duSuffix) + l.Printf(" 75%%: %12.2f%s\n", ps[1]/du, duSuffix) + l.Printf(" 95%%: %12.2f%s\n", ps[2]/du, duSuffix) + l.Printf(" 99%%: %12.2f%s\n", ps[3]/du, duSuffix) + l.Printf(" 99.9%%: %12.2f%s\n", ps[4]/du, duSuffix) + l.Printf(" 1-min rate: %12.2f\n", t.Rate1()) + l.Printf(" 5-min rate: %12.2f\n", t.Rate5()) + l.Printf(" 15-min rate: %12.2f\n", t.Rate15()) + l.Printf(" mean rate: %12.2f\n", t.RateMean()) + } + }) + } +} diff --git a/common/metrics/memory.md b/common/metrics/memory.md new file mode 100644 index 00000000000..47454f54b64 --- /dev/null +++ b/common/metrics/memory.md @@ -0,0 +1,285 @@ +Memory usage +============ + +(Highly unscientific.) + +Command used to gather static memory usage: + +```sh +grep ^Vm "/proc/$(ps fax | grep [m]etrics-bench | awk '{print $1}')/status" +``` + +Program used to gather baseline memory usage: + +```go +package main + +import "time" + +func main() { + time.Sleep(600e9) +} +``` + +Baseline +-------- + +``` +VmPeak: 42604 kB +VmSize: 42604 kB +VmLck: 0 kB +VmHWM: 1120 kB +VmRSS: 1120 kB +VmData: 35460 kB +VmStk: 136 kB +VmExe: 1020 kB +VmLib: 1848 kB +VmPTE: 36 kB +VmSwap: 0 kB +``` + +Program used to gather metric memory usage (with other metrics being similar): + +```go +package main + +import ( + "fmt" + "metrics" + "time" +) + +func main() { + fmt.Sprintf("foo") + metrics.NewRegistry() + time.Sleep(600e9) +} +``` + +1000 counters registered +------------------------ + +``` +VmPeak: 44016 kB +VmSize: 44016 kB +VmLck: 0 kB +VmHWM: 1928 kB +VmRSS: 1928 kB +VmData: 36868 kB +VmStk: 136 kB +VmExe: 1024 kB +VmLib: 1848 kB +VmPTE: 40 kB +VmSwap: 0 kB +``` + +**1.412 kB virtual, TODO 0.808 kB resident per counter.** + +100000 counters registered +-------------------------- + +``` +VmPeak: 55024 kB +VmSize: 55024 kB +VmLck: 0 kB +VmHWM: 12440 kB +VmRSS: 12440 kB +VmData: 47876 kB +VmStk: 136 kB +VmExe: 1024 kB +VmLib: 1848 kB +VmPTE: 64 kB +VmSwap: 0 kB +``` + +**0.1242 kB virtual, 0.1132 kB resident per counter.** + +1000 gauges registered +---------------------- + +``` +VmPeak: 44012 kB +VmSize: 44012 kB +VmLck: 0 kB +VmHWM: 1928 kB +VmRSS: 1928 kB +VmData: 36868 kB +VmStk: 136 kB +VmExe: 1020 kB +VmLib: 1848 kB +VmPTE: 40 kB +VmSwap: 0 kB +``` + +**1.408 kB virtual, 0.808 kB resident per counter.** + +100000 gauges registered +------------------------ + +``` +VmPeak: 55020 kB +VmSize: 55020 kB +VmLck: 0 kB +VmHWM: 12432 kB +VmRSS: 12432 kB +VmData: 47876 kB +VmStk: 136 kB +VmExe: 1020 kB +VmLib: 1848 kB +VmPTE: 60 kB +VmSwap: 0 kB +``` + +**0.12416 kB virtual, 0.11312 resident per gauge.** + +1000 histograms with a uniform sample size of 1028 +-------------------------------------------------- + +``` +VmPeak: 72272 kB +VmSize: 72272 kB +VmLck: 0 kB +VmHWM: 16204 kB +VmRSS: 16204 kB +VmData: 65100 kB +VmStk: 136 kB +VmExe: 1048 kB +VmLib: 1848 kB +VmPTE: 80 kB +VmSwap: 0 kB +``` + +**29.668 kB virtual, TODO 15.084 resident per histogram.** + +10000 histograms with a uniform sample size of 1028 +--------------------------------------------------- + +``` +VmPeak: 256912 kB +VmSize: 256912 kB +VmLck: 0 kB +VmHWM: 146204 kB +VmRSS: 146204 kB +VmData: 249740 kB +VmStk: 136 kB +VmExe: 1048 kB +VmLib: 1848 kB +VmPTE: 448 kB +VmSwap: 0 kB +``` + +**21.4308 kB virtual, 14.5084 kB resident per histogram.** + +50000 histograms with a uniform sample size of 1028 +--------------------------------------------------- + +``` +VmPeak: 908112 kB +VmSize: 908112 kB +VmLck: 0 kB +VmHWM: 645832 kB +VmRSS: 645588 kB +VmData: 900940 kB +VmStk: 136 kB +VmExe: 1048 kB +VmLib: 1848 kB +VmPTE: 1716 kB +VmSwap: 1544 kB +``` + +**17.31016 kB virtual, 12.88936 kB resident per histogram.** + +1000 histograms with an exponentially-decaying sample size of 1028 and alpha of 0.015 +------------------------------------------------------------------------------------- + +``` +VmPeak: 62480 kB +VmSize: 62480 kB +VmLck: 0 kB +VmHWM: 11572 kB +VmRSS: 11572 kB +VmData: 55308 kB +VmStk: 136 kB +VmExe: 1048 kB +VmLib: 1848 kB +VmPTE: 64 kB +VmSwap: 0 kB +``` + +**19.876 kB virtual, 10.452 kB resident per histogram.** + +10000 histograms with an exponentially-decaying sample size of 1028 and alpha of 0.015 +-------------------------------------------------------------------------------------- + +``` +VmPeak: 153296 kB +VmSize: 153296 kB +VmLck: 0 kB +VmHWM: 101176 kB +VmRSS: 101176 kB +VmData: 146124 kB +VmStk: 136 kB +VmExe: 1048 kB +VmLib: 1848 kB +VmPTE: 240 kB +VmSwap: 0 kB +``` + +**11.0692 kB virtual, 10.0056 kB resident per histogram.** + +50000 histograms with an exponentially-decaying sample size of 1028 and alpha of 0.015 +-------------------------------------------------------------------------------------- + +``` +VmPeak: 557264 kB +VmSize: 557264 kB +VmLck: 0 kB +VmHWM: 501056 kB +VmRSS: 501056 kB +VmData: 550092 kB +VmStk: 136 kB +VmExe: 1048 kB +VmLib: 1848 kB +VmPTE: 1032 kB +VmSwap: 0 kB +``` + +**10.2932 kB virtual, 9.99872 kB resident per histogram.** + +1000 meters +----------- + +``` +VmPeak: 74504 kB +VmSize: 74504 kB +VmLck: 0 kB +VmHWM: 24124 kB +VmRSS: 24124 kB +VmData: 67340 kB +VmStk: 136 kB +VmExe: 1040 kB +VmLib: 1848 kB +VmPTE: 92 kB +VmSwap: 0 kB +``` + +**31.9 kB virtual, 23.004 kB resident per meter.** + +10000 meters +------------ + +``` +VmPeak: 278920 kB +VmSize: 278920 kB +VmLck: 0 kB +VmHWM: 227300 kB +VmRSS: 227300 kB +VmData: 271756 kB +VmStk: 136 kB +VmExe: 1040 kB +VmLib: 1848 kB +VmPTE: 488 kB +VmSwap: 0 kB +``` + +**23.6316 kB virtual, 22.618 kB resident per meter.** diff --git a/common/metrics/meter.go b/common/metrics/meter.go new file mode 100644 index 00000000000..ee23af10ebe --- /dev/null +++ b/common/metrics/meter.go @@ -0,0 +1,167 @@ +package metrics + +import ( + "math" + "sync" + "sync/atomic" + "time" +) + +// GetOrRegisterMeter returns an existing Meter or constructs and registers a +// new Meter. +// Be sure to unregister the meter from the registry once it is of no use to +// allow for garbage collection. +func GetOrRegisterMeter(name string, r Registry) *Meter { + return getOrRegister(name, NewMeter, r) +} + +// NewMeter constructs a new Meter and launches a goroutine. +// Be sure to call Stop() once the meter is of no use to allow for garbage collection. +func NewMeter() *Meter { + m := newMeter() + arbiter.add(m) + return m +} + +// NewInactiveMeter returns a meter but does not start any goroutines. This +// method is mainly intended for testing. +func NewInactiveMeter() *Meter { + return newMeter() +} + +// NewRegisteredMeter constructs and registers a new Meter +// and launches a goroutine. +// Be sure to unregister the meter from the registry once it is of no use to +// allow for garbage collection. +func NewRegisteredMeter(name string, r Registry) *Meter { + return GetOrRegisterMeter(name, r) +} + +// MeterSnapshot is a read-only copy of the meter's internal values. +type MeterSnapshot struct { + count int64 + rate1, rate5, rate15, rateMean float64 +} + +// Count returns the count of events at the time the snapshot was taken. +func (m *MeterSnapshot) Count() int64 { return m.count } + +// Rate1 returns the one-minute moving average rate of events per second at the +// time the snapshot was taken. +func (m *MeterSnapshot) Rate1() float64 { return m.rate1 } + +// Rate5 returns the five-minute moving average rate of events per second at +// the time the snapshot was taken. +func (m *MeterSnapshot) Rate5() float64 { return m.rate5 } + +// Rate15 returns the fifteen-minute moving average rate of events per second +// at the time the snapshot was taken. +func (m *MeterSnapshot) Rate15() float64 { return m.rate15 } + +// RateMean returns the meter's mean rate of events per second at the time the +// snapshot was taken. +func (m *MeterSnapshot) RateMean() float64 { return m.rateMean } + +// Meter count events to produce exponentially-weighted moving average rates +// at one-, five-, and fifteen-minutes and a mean rate. +type Meter struct { + count atomic.Int64 + uncounted atomic.Int64 // not yet added to the EWMAs + rateMean atomic.Uint64 + + a1, a5, a15 *EWMA + startTime time.Time + stopped atomic.Bool +} + +func newMeter() *Meter { + return &Meter{ + a1: NewEWMA1(), + a5: NewEWMA5(), + a15: NewEWMA15(), + startTime: time.Now(), + } +} + +// Stop stops the meter, Mark() will be a no-op if you use it after being stopped. +func (m *Meter) Stop() { + if stopped := m.stopped.Swap(true); !stopped { + arbiter.remove(m) + } +} + +// Mark records the occurrence of n events. +func (m *Meter) Mark(n int64) { + m.uncounted.Add(n) +} + +// Snapshot returns a read-only copy of the meter. +func (m *Meter) Snapshot() *MeterSnapshot { + return &MeterSnapshot{ + count: m.count.Load() + m.uncounted.Load(), + rate1: m.a1.Snapshot().Rate(), + rate5: m.a5.Snapshot().Rate(), + rate15: m.a15.Snapshot().Rate(), + rateMean: math.Float64frombits(m.rateMean.Load()), + } +} + +func (m *Meter) tick() { + // Take the uncounted values, add to count + n := m.uncounted.Swap(0) + count := m.count.Add(n) + m.rateMean.Store(math.Float64bits(float64(count) / time.Since(m.startTime).Seconds())) + // Update the EWMA's internal state + m.a1.Update(n) + m.a5.Update(n) + m.a15.Update(n) + // And trigger them to calculate the rates + m.a1.tick() + m.a5.tick() + m.a15.tick() +} + +var arbiter = meterTicker{meters: make(map[*Meter]struct{})} + +// meterTicker ticks meters every 5s from a single goroutine. +// meters are references in a set for future stopping. +type meterTicker struct { + mu sync.RWMutex + + once sync.Once + meters map[*Meter]struct{} +} + +// add a *Meter to the arbiter +func (ma *meterTicker) add(m *Meter) { + ma.mu.Lock() + defer ma.mu.Unlock() + ma.meters[m] = struct{}{} +} + +// remove removes a meter from the set of ticked meters. +func (ma *meterTicker) remove(m *Meter) { + ma.mu.Lock() + delete(ma.meters, m) + ma.mu.Unlock() +} + +// loop ticks meters on a 5-second interval. +func (ma *meterTicker) loop() { + ticker := time.NewTicker(5 * time.Second) + for range ticker.C { + if !metricsEnabled { + continue + } + ma.mu.RLock() + for meter := range ma.meters { + meter.tick() + } + ma.mu.RUnlock() + } +} + +// startMeterTickerLoop will start the arbiter ticker. +func startMeterTickerLoop() { + arbiter.once.Do(func() { go arbiter.loop() }) +} diff --git a/common/metrics/meter_test.go b/common/metrics/meter_test.go new file mode 100644 index 00000000000..e3f39684bd2 --- /dev/null +++ b/common/metrics/meter_test.go @@ -0,0 +1,83 @@ +package metrics + +import ( + "testing" + "time" +) + +func BenchmarkMeter(b *testing.B) { + m := NewMeter() + b.ResetTimer() + for i := 0; i < b.N; i++ { + m.Mark(1) + } +} +func TestMeter(t *testing.T) { + m := NewMeter() + m.Mark(47) + if v := m.Snapshot().Count(); v != 47 { + t.Fatalf("have %d want %d", v, 47) + } +} +func TestGetOrRegisterMeter(t *testing.T) { + r := NewRegistry() + NewRegisteredMeter("foo", r).Mark(47) + if m := GetOrRegisterMeter("foo", r).Snapshot(); m.Count() != 47 { + t.Fatal(m.Count()) + } +} + +func TestMeterDecay(t *testing.T) { + m := newMeter() + m.Mark(1) + m.tick() + rateMean := m.Snapshot().RateMean() + time.Sleep(100 * time.Millisecond) + m.tick() + if m.Snapshot().RateMean() >= rateMean { + t.Error("m.RateMean() didn't decrease") + } +} + +func TestMeterNonzero(t *testing.T) { + m := NewMeter() + m.Mark(3) + if count := m.Snapshot().Count(); count != 3 { + t.Errorf("m.Count(): 3 != %v\n", count) + } +} + +func TestMeterStop(t *testing.T) { + l := len(arbiter.meters) + m := NewMeter() + if l+1 != len(arbiter.meters) { + t.Errorf("arbiter.meters: %d != %d\n", l+1, len(arbiter.meters)) + } + m.Stop() + if l != len(arbiter.meters) { + t.Errorf("arbiter.meters: %d != %d\n", l, len(arbiter.meters)) + } +} + +func TestMeterZero(t *testing.T) { + m := NewMeter().Snapshot() + if count := m.Count(); count != 0 { + t.Errorf("m.Count(): 0 != %v\n", count) + } +} + +func TestMeterRepeat(t *testing.T) { + m := NewMeter() + for i := 0; i < 101; i++ { + m.Mark(int64(i)) + } + if count := m.Snapshot().Count(); count != 5050 { + t.Errorf("m.Count(): 5050 != %v\n", count) + } + for i := 0; i < 101; i++ { + m.Mark(int64(i)) + } + if count := m.Snapshot().Count(); count != 10100 { + t.Errorf("m.Count(): 10100 != %v\n", count) + } +} diff --git a/common/metrics/metrics.go b/common/metrics/metrics.go new file mode 100644 index 00000000000..088948d4034 --- /dev/null +++ b/common/metrics/metrics.go @@ -0,0 +1,204 @@ +// Go port of Coda Hale's Metrics library +// +// +// +// Coda Hale's original work: + +package metrics + +import ( + "runtime/metrics" + "runtime/pprof" + "time" +) + +var ( + metricsEnabled = false +) + +// Enabled is checked by functions that are deemed 'expensive', e.g. if a +// meter-type does locking and/or non-trivial math operations during update. +func Enabled() bool { + return metricsEnabled +} + +// Enable enables the metrics system. +// The Enabled-flag is expected to be set, once, during startup, but toggling off and on +// is not supported. +// +// Enable is not safe to call concurrently. You need to call this as early as possible in +// the program, before any metrics collection will happen. +func Enable() { + metricsEnabled = true + startMeterTickerLoop() +} + +var threadCreateProfile = pprof.Lookup("threadcreate") + +type runtimeStats struct { + GCPauses *metrics.Float64Histogram + GCAllocBytes uint64 + GCFreedBytes uint64 + + MemTotal uint64 + HeapObjects uint64 + HeapFree uint64 + HeapReleased uint64 + HeapUnused uint64 + + Goroutines uint64 + SchedLatency *metrics.Float64Histogram +} + +var runtimeSamples = []metrics.Sample{ + {Name: "/gc/pauses:seconds"}, // histogram + {Name: "/gc/heap/allocs:bytes"}, + {Name: "/gc/heap/frees:bytes"}, + {Name: "/memory/classes/total:bytes"}, + {Name: "/memory/classes/heap/objects:bytes"}, + {Name: "/memory/classes/heap/free:bytes"}, + {Name: "/memory/classes/heap/released:bytes"}, + {Name: "/memory/classes/heap/unused:bytes"}, + {Name: "/sched/goroutines:goroutines"}, + {Name: "/sched/latencies:seconds"}, // histogram +} + +func ReadRuntimeStats() *runtimeStats { + r := new(runtimeStats) + readRuntimeStats(r) + return r +} + +func readRuntimeStats(v *runtimeStats) { + metrics.Read(runtimeSamples) + for _, s := range runtimeSamples { + // Skip invalid/unknown metrics. This is needed because some metrics + // are unavailable in older Go versions, and attempting to read a 'bad' + // metric panics. + if s.Value.Kind() == metrics.KindBad { + continue + } + + switch s.Name { + case "/gc/pauses:seconds": + v.GCPauses = s.Value.Float64Histogram() + case "/gc/heap/allocs:bytes": + v.GCAllocBytes = s.Value.Uint64() + case "/gc/heap/frees:bytes": + v.GCFreedBytes = s.Value.Uint64() + case "/memory/classes/total:bytes": + v.MemTotal = s.Value.Uint64() + case "/memory/classes/heap/objects:bytes": + v.HeapObjects = s.Value.Uint64() + case "/memory/classes/heap/free:bytes": + v.HeapFree = s.Value.Uint64() + case "/memory/classes/heap/released:bytes": + v.HeapReleased = s.Value.Uint64() + case "/memory/classes/heap/unused:bytes": + v.HeapUnused = s.Value.Uint64() + case "/sched/goroutines:goroutines": + v.Goroutines = s.Value.Uint64() + case "/sched/latencies:seconds": + v.SchedLatency = s.Value.Float64Histogram() + } + } +} + +// CollectProcessMetrics periodically collects various metrics about the running process. +func CollectProcessMetrics(refresh time.Duration) { + // Short circuit if the metrics system is disabled + if !metricsEnabled { + return + } + + // Create the various data collectors + var ( + cpustats = make([]CPUStats, 2) + diskstats = make([]DiskStats, 2) + rstats = make([]runtimeStats, 2) + ) + + // This scale factor is used for the runtime's time metrics. It's useful to convert to + // ns here because the runtime gives times in float seconds, but runtimeHistogram can + // only provide integers for the minimum and maximum values. + const secondsToNs = float64(time.Second) + + // Define the various metrics to collect + var ( + cpuSysLoad = GetOrRegisterGauge("system/cpu/sysload", DefaultRegistry) + cpuSysWait = GetOrRegisterGauge("system/cpu/syswait", DefaultRegistry) + cpuProcLoad = GetOrRegisterGauge("system/cpu/procload", DefaultRegistry) + cpuSysLoadTotal = GetOrRegisterCounterFloat64("system/cpu/sysload/total", DefaultRegistry) + cpuSysWaitTotal = GetOrRegisterCounterFloat64("system/cpu/syswait/total", DefaultRegistry) + cpuProcLoadTotal = GetOrRegisterCounterFloat64("system/cpu/procload/total", DefaultRegistry) + cpuThreads = GetOrRegisterGauge("system/cpu/threads", DefaultRegistry) + cpuGoroutines = GetOrRegisterGauge("system/cpu/goroutines", DefaultRegistry) + cpuSchedLatency = getOrRegisterRuntimeHistogram("system/cpu/schedlatency", secondsToNs, nil) + memPauses = getOrRegisterRuntimeHistogram("system/memory/pauses", secondsToNs, nil) + memAllocs = GetOrRegisterMeter("system/memory/allocs", DefaultRegistry) + memFrees = GetOrRegisterMeter("system/memory/frees", DefaultRegistry) + memTotal = GetOrRegisterGauge("system/memory/held", DefaultRegistry) + heapUsed = GetOrRegisterGauge("system/memory/used", DefaultRegistry) + heapObjects = GetOrRegisterGauge("system/memory/objects", DefaultRegistry) + diskReads = GetOrRegisterMeter("system/disk/readcount", DefaultRegistry) + diskReadBytes = GetOrRegisterMeter("system/disk/readdata", DefaultRegistry) + diskReadBytesCounter = GetOrRegisterCounter("system/disk/readbytes", DefaultRegistry) + diskWrites = GetOrRegisterMeter("system/disk/writecount", DefaultRegistry) + diskWriteBytes = GetOrRegisterMeter("system/disk/writedata", DefaultRegistry) + diskWriteBytesCounter = GetOrRegisterCounter("system/disk/writebytes", DefaultRegistry) + ) + + var lastCollectTime time.Time + + // Iterate loading the different stats and updating the meters. + now, prev := 0, 1 + for ; ; now, prev = prev, now { + // Gather CPU times. + ReadCPUStats(&cpustats[now]) + collectTime := time.Now() + secondsSinceLastCollect := collectTime.Sub(lastCollectTime).Seconds() + lastCollectTime = collectTime + if secondsSinceLastCollect > 0 { + sysLoad := cpustats[now].GlobalTime - cpustats[prev].GlobalTime + sysWait := cpustats[now].GlobalWait - cpustats[prev].GlobalWait + procLoad := cpustats[now].LocalTime - cpustats[prev].LocalTime + // Convert to integer percentage. + cpuSysLoad.Update(int64(sysLoad / secondsSinceLastCollect * 100)) + cpuSysWait.Update(int64(sysWait / secondsSinceLastCollect * 100)) + cpuProcLoad.Update(int64(procLoad / secondsSinceLastCollect * 100)) + // increment counters (ms) + cpuSysLoadTotal.Inc(sysLoad) + cpuSysWaitTotal.Inc(sysWait) + cpuProcLoadTotal.Inc(procLoad) + } + + // Threads + cpuThreads.Update(int64(threadCreateProfile.Count())) + + // Go runtime metrics + readRuntimeStats(&rstats[now]) + + cpuGoroutines.Update(int64(rstats[now].Goroutines)) + cpuSchedLatency.update(rstats[now].SchedLatency) + memPauses.update(rstats[now].GCPauses) + + memAllocs.Mark(int64(rstats[now].GCAllocBytes - rstats[prev].GCAllocBytes)) + memFrees.Mark(int64(rstats[now].GCFreedBytes - rstats[prev].GCFreedBytes)) + + memTotal.Update(int64(rstats[now].MemTotal)) + heapUsed.Update(int64(rstats[now].MemTotal - rstats[now].HeapUnused - rstats[now].HeapFree - rstats[now].HeapReleased)) + heapObjects.Update(int64(rstats[now].HeapObjects)) + + // Disk + if ReadDiskStats(&diskstats[now]) == nil { + diskReads.Mark(diskstats[now].ReadCount - diskstats[prev].ReadCount) + diskReadBytes.Mark(diskstats[now].ReadBytes - diskstats[prev].ReadBytes) + diskWrites.Mark(diskstats[now].WriteCount - diskstats[prev].WriteCount) + diskWriteBytes.Mark(diskstats[now].WriteBytes - diskstats[prev].WriteBytes) + diskReadBytesCounter.Inc(diskstats[now].ReadBytes - diskstats[prev].ReadBytes) + diskWriteBytesCounter.Inc(diskstats[now].WriteBytes - diskstats[prev].WriteBytes) + } + + time.Sleep(refresh) + } +} diff --git a/common/metrics/metrics_test.go b/common/metrics/metrics_test.go new file mode 100644 index 00000000000..dc144f2425a --- /dev/null +++ b/common/metrics/metrics_test.go @@ -0,0 +1,62 @@ +package metrics + +import ( + "fmt" + "sync" + "testing" + "time" +) + +func TestReadRuntimeValues(t *testing.T) { + var v runtimeStats + readRuntimeStats(&v) + t.Logf("%+v", v) +} + +func BenchmarkMetrics(b *testing.B) { + var ( + r = NewRegistry() + c = NewRegisteredCounter("counter", r) + cf = NewRegisteredCounterFloat64("counterfloat64", r) + g = NewRegisteredGauge("gauge", r) + gf = NewRegisteredGaugeFloat64("gaugefloat64", r) + h = NewRegisteredHistogram("histogram", r, NewUniformSample(100)) + m = NewRegisteredMeter("meter", r) + t = NewRegisteredTimer("timer", r) + ) + RegisterDebugGCStats(r) + b.ResetTimer() + var wg sync.WaitGroup + wg.Add(128) + for i := 0; i < 128; i++ { + go func() { + defer wg.Done() + for i := 0; i < b.N; i++ { + c.Inc(1) + cf.Inc(1.0) + g.Update(int64(i)) + gf.Update(float64(i)) + h.Update(int64(i)) + m.Mark(1) + t.Update(1) + } + }() + } + wg.Wait() +} + +func Example() { + c := NewCounter() + Register("money", c) + c.Inc(17) + + // Threadsafe registration + t := GetOrRegisterTimer("db.get.latency", nil) + t.Time(func() { time.Sleep(10 * time.Millisecond) }) + t.Update(1) + + fmt.Println(c.Snapshot().Count()) + fmt.Println(t.Snapshot().Min()) + // Output: 17 + // 1 +} diff --git a/common/metrics/opentsdb.go b/common/metrics/opentsdb.go new file mode 100644 index 00000000000..57af3d025e6 --- /dev/null +++ b/common/metrics/opentsdb.go @@ -0,0 +1,128 @@ +package metrics + +import ( + "bufio" + "fmt" + "io" + "log" + "net" + "os" + "strings" + "time" +) + +var shortHostName = "" + +// OpenTSDBConfig provides a container with configuration parameters for +// the OpenTSDB exporter +type OpenTSDBConfig struct { + Addr *net.TCPAddr // Network address to connect to + Registry Registry // Registry to be exported + FlushInterval time.Duration // Flush interval + DurationUnit time.Duration // Time conversion unit for durations + Prefix string // Prefix to be prepended to metric names +} + +// OpenTSDB is a blocking exporter function which reports metrics in r +// to a TSDB server located at addr, flushing them every d duration +// and prepending metric names with prefix. +func OpenTSDB(r Registry, d time.Duration, prefix string, addr *net.TCPAddr) { + OpenTSDBWithConfig(OpenTSDBConfig{ + Addr: addr, + Registry: r, + FlushInterval: d, + DurationUnit: time.Nanosecond, + Prefix: prefix, + }) +} + +// OpenTSDBWithConfig is a blocking exporter function just like OpenTSDB, +// but it takes a OpenTSDBConfig instead. +func OpenTSDBWithConfig(c OpenTSDBConfig) { + for range time.Tick(c.FlushInterval) { + if err := openTSDB(&c); nil != err { + log.Println(err) + } + } +} + +func getShortHostname() string { + if shortHostName == "" { + host, _ := os.Hostname() + if index := strings.Index(host, "."); index > 0 { + shortHostName = host[:index] + } else { + shortHostName = host + } + } + return shortHostName +} + +// writeRegistry writes the registry-metrics on the opentsb format. +func (c *OpenTSDBConfig) writeRegistry(w io.Writer, now int64, shortHostname string) { + du := float64(c.DurationUnit) + + c.Registry.Each(func(name string, i interface{}) { + switch metric := i.(type) { + case *Counter: + fmt.Fprintf(w, "put %s.%s.count %d %d host=%s\n", c.Prefix, name, now, metric.Snapshot().Count(), shortHostname) + case *CounterFloat64: + fmt.Fprintf(w, "put %s.%s.count %d %f host=%s\n", c.Prefix, name, now, metric.Snapshot().Count(), shortHostname) + case *Gauge: + fmt.Fprintf(w, "put %s.%s.value %d %d host=%s\n", c.Prefix, name, now, metric.Snapshot().Value(), shortHostname) + case *GaugeFloat64: + fmt.Fprintf(w, "put %s.%s.value %d %f host=%s\n", c.Prefix, name, now, metric.Snapshot().Value(), shortHostname) + case *GaugeInfo: + fmt.Fprintf(w, "put %s.%s.value %d %s host=%s\n", c.Prefix, name, now, metric.Snapshot().Value().String(), shortHostname) + case Histogram: + h := metric.Snapshot() + ps := h.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999}) + fmt.Fprintf(w, "put %s.%s.count %d %d host=%s\n", c.Prefix, name, now, h.Count(), shortHostname) + fmt.Fprintf(w, "put %s.%s.min %d %d host=%s\n", c.Prefix, name, now, h.Min(), shortHostname) + fmt.Fprintf(w, "put %s.%s.max %d %d host=%s\n", c.Prefix, name, now, h.Max(), shortHostname) + fmt.Fprintf(w, "put %s.%s.mean %d %.2f host=%s\n", c.Prefix, name, now, h.Mean(), shortHostname) + fmt.Fprintf(w, "put %s.%s.std-dev %d %.2f host=%s\n", c.Prefix, name, now, h.StdDev(), shortHostname) + fmt.Fprintf(w, "put %s.%s.50-percentile %d %.2f host=%s\n", c.Prefix, name, now, ps[0], shortHostname) + fmt.Fprintf(w, "put %s.%s.75-percentile %d %.2f host=%s\n", c.Prefix, name, now, ps[1], shortHostname) + fmt.Fprintf(w, "put %s.%s.95-percentile %d %.2f host=%s\n", c.Prefix, name, now, ps[2], shortHostname) + fmt.Fprintf(w, "put %s.%s.99-percentile %d %.2f host=%s\n", c.Prefix, name, now, ps[3], shortHostname) + fmt.Fprintf(w, "put %s.%s.999-percentile %d %.2f host=%s\n", c.Prefix, name, now, ps[4], shortHostname) + case *Meter: + m := metric.Snapshot() + fmt.Fprintf(w, "put %s.%s.count %d %d host=%s\n", c.Prefix, name, now, m.Count(), shortHostname) + fmt.Fprintf(w, "put %s.%s.one-minute %d %.2f host=%s\n", c.Prefix, name, now, m.Rate1(), shortHostname) + fmt.Fprintf(w, "put %s.%s.five-minute %d %.2f host=%s\n", c.Prefix, name, now, m.Rate5(), shortHostname) + fmt.Fprintf(w, "put %s.%s.fifteen-minute %d %.2f host=%s\n", c.Prefix, name, now, m.Rate15(), shortHostname) + fmt.Fprintf(w, "put %s.%s.mean %d %.2f host=%s\n", c.Prefix, name, now, m.RateMean(), shortHostname) + case *Timer: + t := metric.Snapshot() + ps := t.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999}) + fmt.Fprintf(w, "put %s.%s.count %d %d host=%s\n", c.Prefix, name, now, t.Count(), shortHostname) + fmt.Fprintf(w, "put %s.%s.min %d %d host=%s\n", c.Prefix, name, now, t.Min()/int64(du), shortHostname) + fmt.Fprintf(w, "put %s.%s.max %d %d host=%s\n", c.Prefix, name, now, t.Max()/int64(du), shortHostname) + fmt.Fprintf(w, "put %s.%s.mean %d %.2f host=%s\n", c.Prefix, name, now, t.Mean()/du, shortHostname) + fmt.Fprintf(w, "put %s.%s.std-dev %d %.2f host=%s\n", c.Prefix, name, now, t.StdDev()/du, shortHostname) + fmt.Fprintf(w, "put %s.%s.50-percentile %d %.2f host=%s\n", c.Prefix, name, now, ps[0]/du, shortHostname) + fmt.Fprintf(w, "put %s.%s.75-percentile %d %.2f host=%s\n", c.Prefix, name, now, ps[1]/du, shortHostname) + fmt.Fprintf(w, "put %s.%s.95-percentile %d %.2f host=%s\n", c.Prefix, name, now, ps[2]/du, shortHostname) + fmt.Fprintf(w, "put %s.%s.99-percentile %d %.2f host=%s\n", c.Prefix, name, now, ps[3]/du, shortHostname) + fmt.Fprintf(w, "put %s.%s.999-percentile %d %.2f host=%s\n", c.Prefix, name, now, ps[4]/du, shortHostname) + fmt.Fprintf(w, "put %s.%s.one-minute %d %.2f host=%s\n", c.Prefix, name, now, t.Rate1(), shortHostname) + fmt.Fprintf(w, "put %s.%s.five-minute %d %.2f host=%s\n", c.Prefix, name, now, t.Rate5(), shortHostname) + fmt.Fprintf(w, "put %s.%s.fifteen-minute %d %.2f host=%s\n", c.Prefix, name, now, t.Rate15(), shortHostname) + fmt.Fprintf(w, "put %s.%s.mean-rate %d %.2f host=%s\n", c.Prefix, name, now, t.RateMean(), shortHostname) + } + }) +} + +func openTSDB(c *OpenTSDBConfig) error { + conn, err := net.DialTCP("tcp", nil, c.Addr) + if nil != err { + return err + } + defer conn.Close() + w := bufio.NewWriter(conn) + c.writeRegistry(w, time.Now().Unix(), getShortHostname()) + w.Flush() + return nil +} diff --git a/common/metrics/opentsdb_test.go b/common/metrics/opentsdb_test.go new file mode 100644 index 00000000000..4548309f9c2 --- /dev/null +++ b/common/metrics/opentsdb_test.go @@ -0,0 +1,66 @@ +package metrics + +import ( + "fmt" + "net" + "os" + "strings" + "testing" + "time" +) + +func ExampleOpenTSDB() { + addr, _ := net.ResolveTCPAddr("net", ":2003") + go OpenTSDB(DefaultRegistry, 1*time.Second, "some.prefix", addr) +} + +func ExampleOpenTSDBWithConfig() { + addr, _ := net.ResolveTCPAddr("net", ":2003") + go OpenTSDBWithConfig(OpenTSDBConfig{ + Addr: addr, + Registry: DefaultRegistry, + FlushInterval: 1 * time.Second, + DurationUnit: time.Millisecond, + }) +} + +func TestExampleOpenTSB(t *testing.T) { + r := NewOrderedRegistry() + NewRegisteredGaugeInfo("foo", r).Update(GaugeInfoValue{"chain_id": "5"}) + NewRegisteredGaugeFloat64("pi", r).Update(3.14) + NewRegisteredCounter("months", r).Inc(12) + NewRegisteredCounterFloat64("tau", r).Inc(1.57) + NewRegisteredMeter("elite", r).Mark(1337) + NewRegisteredTimer("second", r).Update(time.Second) + NewRegisteredCounterFloat64("tau", r).Inc(1.57) + NewRegisteredCounterFloat64("tau", r).Inc(1.57) + + w := new(strings.Builder) + (&OpenTSDBConfig{ + Registry: r, + DurationUnit: time.Millisecond, + Prefix: "pre", + }).writeRegistry(w, 978307200, "hal9000") + + wantB, err := os.ReadFile("./testdata/opentsb.want") + if err != nil { + t.Fatal(err) + } + if have, want := w.String(), string(wantB); have != want { + t.Errorf("\nhave:\n%v\nwant:\n%v\n", have, want) + t.Logf("have vs want:\n%v", findFirstDiffPos(have, want)) + } +} + +func findFirstDiffPos(a, b string) string { + yy := strings.Split(b, "\n") + for i, x := range strings.Split(a, "\n") { + if i >= len(yy) { + return fmt.Sprintf("have:%d: %s\nwant:%d: ", i, x, i) + } + if y := yy[i]; x != y { + return fmt.Sprintf("have:%d: %s\nwant:%d: %s", i, x, i, y) + } + } + return "" +} diff --git a/common/metrics/registry.go b/common/metrics/registry.go new file mode 100644 index 00000000000..6070f3d0e9e --- /dev/null +++ b/common/metrics/registry.go @@ -0,0 +1,363 @@ +package metrics + +import ( + "errors" + "fmt" + "sort" + "strings" + "sync" +) + +// ErrDuplicateMetric is the error returned by Registry.Register when a metric +// already exists. If you mean to Register that metric you must first +// Unregister the existing metric. +var ErrDuplicateMetric = errors.New("duplicate metric") + +// A Registry holds references to a set of metrics by name and can iterate +// over them, calling callback functions provided by the user. +// +// This is an interface to encourage other structs to implement +// the Registry API as appropriate. +type Registry interface { + + // Each call the given function for each registered metric. + Each(func(string, interface{})) + + // Get the metric by the given name or nil if none is registered. + Get(string) interface{} + + // GetAll metrics in the Registry. + GetAll() map[string]map[string]interface{} + + // GetOrRegister returns an existing metric or registers the one returned + // by the given constructor. + GetOrRegister(string, func() interface{}) interface{} + + // Register the given metric under the given name. + Register(string, interface{}) error + + // RunHealthchecks run all registered healthchecks. + RunHealthchecks() + + // Unregister the metric with the given name. + Unregister(string) +} + +type orderedRegistry struct { + StandardRegistry +} + +// Each call the given function for each registered metric. +func (r *orderedRegistry) Each(f func(string, interface{})) { + var names []string + reg := r.registered() + for name := range reg { + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + f(name, reg[name]) + } +} + +// NewRegistry creates a new registry. +func NewRegistry() Registry { + return new(StandardRegistry) +} + +// NewOrderedRegistry creates a new ordered registry (for testing). +func NewOrderedRegistry() Registry { + return new(orderedRegistry) +} + +// StandardRegistry the standard implementation of a Registry uses sync.map +// of names to metrics. +type StandardRegistry struct { + metrics sync.Map +} + +// Each call the given function for each registered metric. +func (r *StandardRegistry) Each(f func(string, interface{})) { + for name, i := range r.registered() { + f(name, i) + } +} + +// Get the metric by the given name or nil if none is registered. +func (r *StandardRegistry) Get(name string) interface{} { + item, _ := r.metrics.Load(name) + return item +} + +// GetOrRegister gets an existing metric or creates and registers a new one. Threadsafe +// alternative to calling Get and Register on failure. +// The interface can be the metric to register if not found in registry, +// or a function returning the metric for lazy instantiation. +func (r *StandardRegistry) GetOrRegister(name string, ctor func() interface{}) interface{} { + // fast path + cached, ok := r.metrics.Load(name) + if ok { + return cached + } + item, _, _ := r.loadOrRegister(name, ctor()) + return item +} + +// Register the given metric under the given name. Returns a ErrDuplicateMetric +// if a metric by the given name is already registered. +func (r *StandardRegistry) Register(name string, i interface{}) error { + // fast path + _, ok := r.metrics.Load(name) + if ok { + return fmt.Errorf("%w: %v", ErrDuplicateMetric, name) + } + + _, loaded, _ := r.loadOrRegister(name, i) + if loaded { + return fmt.Errorf("%w: %v", ErrDuplicateMetric, name) + } + return nil +} + +// RunHealthchecks run all registered healthchecks. +func (r *StandardRegistry) RunHealthchecks() { + r.metrics.Range(func(key, value any) bool { + if h, ok := value.(*Healthcheck); ok { + h.Check() + } + return true + }) +} + +// GetAll metrics in the Registry +func (r *StandardRegistry) GetAll() map[string]map[string]interface{} { + data := make(map[string]map[string]interface{}) + r.Each(func(name string, i interface{}) { + values := make(map[string]interface{}) + switch metric := i.(type) { + case *Counter: + values["count"] = metric.Snapshot().Count() + case *CounterFloat64: + values["count"] = metric.Snapshot().Count() + case *Gauge: + values["value"] = metric.Snapshot().Value() + case *GaugeFloat64: + values["value"] = metric.Snapshot().Value() + case *Healthcheck: + values["error"] = nil + metric.Check() + if err := metric.Error(); nil != err { + values["error"] = metric.Error().Error() + } + case Histogram: + h := metric.Snapshot() + ps := h.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999}) + values["count"] = h.Count() + values["min"] = h.Min() + values["max"] = h.Max() + values["mean"] = h.Mean() + values["stddev"] = h.StdDev() + values["median"] = ps[0] + values["75%"] = ps[1] + values["95%"] = ps[2] + values["99%"] = ps[3] + values["99.9%"] = ps[4] + case *Meter: + m := metric.Snapshot() + values["count"] = m.Count() + values["1m.rate"] = m.Rate1() + values["5m.rate"] = m.Rate5() + values["15m.rate"] = m.Rate15() + values["mean.rate"] = m.RateMean() + case *Timer: + t := metric.Snapshot() + ps := t.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999}) + values["count"] = t.Count() + values["min"] = t.Min() + values["max"] = t.Max() + values["mean"] = t.Mean() + values["stddev"] = t.StdDev() + values["median"] = ps[0] + values["75%"] = ps[1] + values["95%"] = ps[2] + values["99%"] = ps[3] + values["99.9%"] = ps[4] + values["1m.rate"] = t.Rate1() + values["5m.rate"] = t.Rate5() + values["15m.rate"] = t.Rate15() + values["mean.rate"] = t.RateMean() + } + data[name] = values + }) + return data +} + +// Unregister the metric with the given name. +func (r *StandardRegistry) Unregister(name string) { + r.stop(name) + r.metrics.LoadAndDelete(name) +} + +func (r *StandardRegistry) loadOrRegister(name string, i interface{}) (interface{}, bool, bool) { + switch i.(type) { + case *Counter, *CounterFloat64, *Gauge, *GaugeFloat64, *GaugeInfo, *Healthcheck, Histogram, *Meter, *Timer, *ResettingTimer: + default: + return nil, false, false + } + item, loaded := r.metrics.LoadOrStore(name, i) + return item, loaded, true +} + +func (r *StandardRegistry) registered() map[string]interface{} { + metrics := make(map[string]interface{}) + r.metrics.Range(func(key, value any) bool { + metrics[key.(string)] = value + return true + }) + return metrics +} + +func (r *StandardRegistry) stop(name string) { + if i, ok := r.metrics.Load(name); ok { + if s, ok := i.(Stoppable); ok { + s.Stop() + } + } +} + +// Stoppable defines the metrics which has to be stopped. +type Stoppable interface { + Stop() +} + +type PrefixedRegistry struct { + underlying Registry + prefix string +} + +func NewPrefixedRegistry(prefix string) Registry { + return &PrefixedRegistry{ + underlying: NewRegistry(), + prefix: prefix, + } +} + +func NewPrefixedChildRegistry(parent Registry, prefix string) Registry { + return &PrefixedRegistry{ + underlying: parent, + prefix: prefix, + } +} + +// Each call the given function for each registered metric. +func (r *PrefixedRegistry) Each(fn func(string, interface{})) { + wrappedFn := func(prefix string) func(string, interface{}) { + return func(name string, iface interface{}) { + if strings.HasPrefix(name, prefix) { + fn(name, iface) + } else { + return + } + } + } + + baseRegistry, prefix := findPrefix(r, "") + baseRegistry.Each(wrappedFn(prefix)) +} + +func findPrefix(registry Registry, prefix string) (Registry, string) { + switch r := registry.(type) { + case *PrefixedRegistry: + return findPrefix(r.underlying, r.prefix+prefix) + case *StandardRegistry: + return r, prefix + } + return nil, "" +} + +// Get the metric by the given name or nil if none is registered. +func (r *PrefixedRegistry) Get(name string) interface{} { + realName := r.prefix + name + return r.underlying.Get(realName) +} + +// GetOrRegister gets an existing metric or registers the given one. +// The interface can be the metric to register if not found in registry, +// or a function returning the metric for lazy instantiation. +func (r *PrefixedRegistry) GetOrRegister(name string, ctor func() interface{}) interface{} { + realName := r.prefix + name + return r.underlying.GetOrRegister(realName, ctor) +} + +// Register the given metric under the given name. The name will be prefixed. +func (r *PrefixedRegistry) Register(name string, metric interface{}) error { + realName := r.prefix + name + return r.underlying.Register(realName, metric) +} + +// RunHealthchecks run all registered healthchecks. +func (r *PrefixedRegistry) RunHealthchecks() { + r.underlying.RunHealthchecks() +} + +// GetAll metrics in the Registry +func (r *PrefixedRegistry) GetAll() map[string]map[string]interface{} { + return r.underlying.GetAll() +} + +// Unregister the metric with the given name. The name will be prefixed. +func (r *PrefixedRegistry) Unregister(name string) { + realName := r.prefix + name + r.underlying.Unregister(realName) +} + +var ( + DefaultRegistry = NewRegistry() +) + +// Each call the given function for each registered metric. +func Each(f func(string, interface{})) { + DefaultRegistry.Each(f) +} + +// Get the metric by the given name or nil if none is registered. +func Get(name string) interface{} { + return DefaultRegistry.Get(name) +} + +// GetOrRegister gets an existing metric or creates and registers a new one. Threadsafe +// alternative to calling Get and Register on failure. +func GetOrRegister(name string, i func() interface{}) interface{} { + return DefaultRegistry.GetOrRegister(name, i) +} + +func getOrRegister[T any](name string, ctor func() T, r Registry) T { + if r == nil { + r = DefaultRegistry + } + return r.GetOrRegister(name, func() any { return ctor() }).(T) +} + +// Register the given metric under the given name. Returns a ErrDuplicateMetric +// if a metric by the given name is already registered. +func Register(name string, i interface{}) error { + return DefaultRegistry.Register(name, i) +} + +// MustRegister register the given metric under the given name. Panics if a metric by the +// given name is already registered. +func MustRegister(name string, i interface{}) { + if err := Register(name, i); err != nil { + panic(err) + } +} + +// RunHealthchecks run all registered healthchecks. +func RunHealthchecks() { + DefaultRegistry.RunHealthchecks() +} + +// Unregister the metric with the given name. +func Unregister(name string) { + DefaultRegistry.Unregister(name) +} diff --git a/common/metrics/registry_test.go b/common/metrics/registry_test.go new file mode 100644 index 00000000000..6af0796da95 --- /dev/null +++ b/common/metrics/registry_test.go @@ -0,0 +1,335 @@ +package metrics + +import ( + "sync" + "testing" +) + +func BenchmarkRegistry(b *testing.B) { + r := NewRegistry() + r.Register("foo", NewCounter()) + b.ResetTimer() + for i := 0; i < b.N; i++ { + r.Each(func(string, interface{}) {}) + } +} + +func BenchmarkRegistryGetOrRegisterParallel_8(b *testing.B) { + benchmarkRegistryGetOrRegisterParallel(b, 8) +} + +func BenchmarkRegistryGetOrRegisterParallel_32(b *testing.B) { + benchmarkRegistryGetOrRegisterParallel(b, 32) +} + +func benchmarkRegistryGetOrRegisterParallel(b *testing.B, amount int) { + r := NewRegistry() + b.ResetTimer() + var wg sync.WaitGroup + for i := 0; i < amount; i++ { + wg.Add(1) + go func() { + for i := 0; i < b.N; i++ { + GetOrRegisterMeter("foo", r) + } + wg.Done() + }() + } + wg.Wait() +} + +func TestRegistry(t *testing.T) { + r := NewRegistry() + r.Register("foo", NewCounter()) + i := 0 + r.Each(func(name string, iface interface{}) { + i++ + if name != "foo" { + t.Fatal(name) + } + if _, ok := iface.(*Counter); !ok { + t.Fatal(iface) + } + }) + if i != 1 { + t.Fatal(i) + } + r.Unregister("foo") + i = 0 + r.Each(func(string, interface{}) { i++ }) + if i != 0 { + t.Fatal(i) + } +} + +func TestRegistryDuplicate(t *testing.T) { + r := NewRegistry() + if err := r.Register("foo", NewCounter()); nil != err { + t.Fatal(err) + } + if err := r.Register("foo", NewGauge()); nil == err { + t.Fatal(err) + } + i := 0 + r.Each(func(name string, iface interface{}) { + i++ + if _, ok := iface.(*Counter); !ok { + t.Fatal(iface) + } + }) + if i != 1 { + t.Fatal(i) + } +} + +func TestRegistryGet(t *testing.T) { + r := NewRegistry() + r.Register("foo", NewCounter()) + if count := r.Get("foo").(*Counter).Snapshot().Count(); count != 0 { + t.Fatal(count) + } + r.Get("foo").(*Counter).Inc(1) + if count := r.Get("foo").(*Counter).Snapshot().Count(); count != 1 { + t.Fatal(count) + } +} + +func TestRegistryGetOrRegister(t *testing.T) { + r := NewRegistry() + + // First metric wins with GetOrRegister + c1 := GetOrRegisterCounter("foo", r) + c2 := GetOrRegisterCounter("foo", r) + if c1 != c2 { + t.Fatal("counters should've matched") + } + + i := 0 + r.Each(func(name string, iface interface{}) { + i++ + if name != "foo" { + t.Fatal(name) + } + if _, ok := iface.(*Counter); !ok { + t.Fatal(iface) + } + }) + if i != 1 { + t.Fatal(i) + } +} + +func TestRegistryGetOrRegisterWithLazyInstantiation(t *testing.T) { + r := NewRegistry() + + // First metric wins with GetOrRegister + c1 := GetOrRegisterCounter("foo", r) + c2 := GetOrRegisterCounter("foo", r) + if c1 != c2 { + t.Fatal("counters should've matched") + } + + i := 0 + r.Each(func(name string, iface interface{}) { + i++ + if name != "foo" { + t.Fatal(name) + } + if _, ok := iface.(*Counter); !ok { + t.Fatal(iface) + } + }) + if i != 1 { + t.Fatal(i) + } +} + +func TestRegistryUnregister(t *testing.T) { + l := len(arbiter.meters) + r := NewRegistry() + r.Register("foo", NewCounter()) + r.Register("bar", NewMeter()) + r.Register("baz", NewTimer()) + if len(arbiter.meters) != l+2 { + t.Errorf("arbiter.meters: %d != %d\n", l+2, len(arbiter.meters)) + } + r.Unregister("foo") + r.Unregister("bar") + r.Unregister("baz") + if len(arbiter.meters) != l { + t.Errorf("arbiter.meters: %d != %d\n", l+2, len(arbiter.meters)) + } +} + +func TestPrefixedChildRegistryGetOrRegister(t *testing.T) { + r := NewRegistry() + pr := NewPrefixedChildRegistry(r, "prefix.") + + _ = GetOrRegisterCounter("foo", pr) + + i := 0 + r.Each(func(name string, m interface{}) { + i++ + if name != "prefix.foo" { + t.Fatal(name) + } + }) + if i != 1 { + t.Fatal(i) + } +} + +func TestPrefixedRegistryGetOrRegister(t *testing.T) { + r := NewPrefixedRegistry("prefix.") + + _ = GetOrRegisterCounter("foo", r) + + i := 0 + r.Each(func(name string, m interface{}) { + i++ + if name != "prefix.foo" { + t.Fatal(name) + } + }) + if i != 1 { + t.Fatal(i) + } +} + +func TestPrefixedRegistryRegister(t *testing.T) { + r := NewPrefixedRegistry("prefix.") + err := r.Register("foo", NewCounter()) + c := NewCounter() + Register("bar", c) + if err != nil { + t.Fatal(err.Error()) + } + + i := 0 + r.Each(func(name string, m interface{}) { + i++ + if name != "prefix.foo" { + t.Fatal(name) + } + }) + if i != 1 { + t.Fatal(i) + } +} + +func TestPrefixedRegistryUnregister(t *testing.T) { + r := NewPrefixedRegistry("prefix.") + + _ = r.Register("foo", NewCounter()) + + i := 0 + r.Each(func(name string, m interface{}) { + i++ + if name != "prefix.foo" { + t.Fatal(name) + } + }) + if i != 1 { + t.Fatal(i) + } + + r.Unregister("foo") + + i = 0 + r.Each(func(name string, m interface{}) { + i++ + }) + + if i != 0 { + t.Fatal(i) + } +} + +func TestPrefixedRegistryGet(t *testing.T) { + pr := NewPrefixedRegistry("prefix.") + name := "foo" + pr.Register(name, NewCounter()) + + fooCounter := pr.Get(name) + if fooCounter == nil { + t.Fatal(name) + } +} + +func TestPrefixedChildRegistryGet(t *testing.T) { + r := NewRegistry() + pr := NewPrefixedChildRegistry(r, "prefix.") + name := "foo" + pr.Register(name, NewCounter()) + fooCounter := pr.Get(name) + if fooCounter == nil { + t.Fatal(name) + } +} + +func TestChildPrefixedRegistryRegister(t *testing.T) { + r := NewPrefixedChildRegistry(DefaultRegistry, "prefix.") + err := r.Register("foo", NewCounter()) + c := NewCounter() + Register("bar", c) + if err != nil { + t.Fatal(err.Error()) + } + + i := 0 + r.Each(func(name string, m interface{}) { + i++ + if name != "prefix.foo" { + t.Fatal(name) + } + }) + if i != 1 { + t.Fatal(i) + } +} + +func TestChildPrefixedRegistryOfChildRegister(t *testing.T) { + r := NewPrefixedChildRegistry(NewRegistry(), "prefix.") + r2 := NewPrefixedChildRegistry(r, "prefix2.") + err := r.Register("foo2", NewCounter()) + if err != nil { + t.Fatal(err.Error()) + } + err = r2.Register("baz", NewCounter()) + if err != nil { + t.Fatal(err.Error()) + } + c := NewCounter() + Register("bars", c) + + i := 0 + r2.Each(func(name string, m interface{}) { + i++ + if name != "prefix.prefix2.baz" { + t.Fatal(name) + } + }) + if i != 1 { + t.Fatal(i) + } +} + +func TestWalkRegistries(t *testing.T) { + r := NewPrefixedChildRegistry(NewRegistry(), "prefix.") + r2 := NewPrefixedChildRegistry(r, "prefix2.") + err := r.Register("foo2", NewCounter()) + if err != nil { + t.Fatal(err.Error()) + } + err = r2.Register("baz", NewCounter()) + if err != nil { + t.Fatal(err.Error()) + } + c := NewCounter() + Register("bars", c) + + _, prefix := findPrefix(r2, "") + if prefix != "prefix.prefix2." { + t.Fatal(prefix) + } +} diff --git a/common/metrics/resetting_sample.go b/common/metrics/resetting_sample.go new file mode 100644 index 00000000000..730ef93416d --- /dev/null +++ b/common/metrics/resetting_sample.go @@ -0,0 +1,24 @@ +package metrics + +// ResettingSample converts an ordinary sample into one that resets whenever its +// snapshot is retrieved. This will break for multi-monitor systems, but when only +// a single metric is being pushed out, this ensure that low-frequency events don't +// skew th charts indefinitely. +func ResettingSample(sample Sample) Sample { + return &resettingSample{ + Sample: sample, + } +} + +// resettingSample is a simple wrapper around a sample that resets it upon the +// snapshot retrieval. +type resettingSample struct { + Sample +} + +// Snapshot returns a read-only copy of the sample with the original reset. +func (rs *resettingSample) Snapshot() *sampleSnapshot { + s := rs.Sample.Snapshot() + rs.Sample.Clear() + return s +} diff --git a/common/metrics/resetting_timer.go b/common/metrics/resetting_timer.go new file mode 100644 index 00000000000..8aa7dc1488f --- /dev/null +++ b/common/metrics/resetting_timer.go @@ -0,0 +1,134 @@ +package metrics + +import ( + "sync" + "time" +) + +// GetOrRegisterResettingTimer returns an existing ResettingTimer or constructs and registers a +// new ResettingTimer. +func GetOrRegisterResettingTimer(name string, r Registry) *ResettingTimer { + return getOrRegister(name, NewResettingTimer, r) +} + +// NewRegisteredResettingTimer constructs and registers a new ResettingTimer. +func NewRegisteredResettingTimer(name string, r Registry) *ResettingTimer { + c := NewResettingTimer() + if nil == r { + r = DefaultRegistry + } + r.Register(name, c) + return c +} + +// NewResettingTimer constructs a new ResettingTimer +func NewResettingTimer() *ResettingTimer { + return &ResettingTimer{ + values: make([]int64, 0, 10), + } +} + +// ResettingTimer is used for storing aggregated values for timers, which are reset on every flush interval. +type ResettingTimer struct { + values []int64 + sum int64 // sum is a running count of the total sum, used later to calculate mean + + mutex sync.Mutex +} + +// Snapshot resets the timer and returns a read-only copy of its contents. +func (t *ResettingTimer) Snapshot() *ResettingTimerSnapshot { + t.mutex.Lock() + defer t.mutex.Unlock() + snapshot := &ResettingTimerSnapshot{} + if len(t.values) > 0 { + snapshot.mean = float64(t.sum) / float64(len(t.values)) + snapshot.values = t.values + t.values = make([]int64, 0, 10) + } + t.sum = 0 + return snapshot +} + +// Time records the duration of the execution of the given function. +func (t *ResettingTimer) Time(f func()) { + ts := time.Now() + f() + t.Update(time.Since(ts)) +} + +// Update records the duration of an event. +func (t *ResettingTimer) Update(d time.Duration) { + if !metricsEnabled { + return + } + t.mutex.Lock() + defer t.mutex.Unlock() + t.values = append(t.values, int64(d)) + t.sum += int64(d) +} + +// UpdateSince records the duration of an event that started at a time and ends now. +func (t *ResettingTimer) UpdateSince(ts time.Time) { + t.Update(time.Since(ts)) +} + +// ResettingTimerSnapshot is a point-in-time copy of another ResettingTimer. +type ResettingTimerSnapshot struct { + values []int64 + mean float64 + max int64 + min int64 + thresholdBoundaries []float64 + calculated bool +} + +// Count return the length of the values from snapshot. +func (t *ResettingTimerSnapshot) Count() int { + return len(t.values) +} + +// Percentiles returns the boundaries for the input percentiles. +// note: this method is not thread safe +func (t *ResettingTimerSnapshot) Percentiles(percentiles []float64) []float64 { + t.calc(percentiles) + return t.thresholdBoundaries +} + +// Mean returns the mean of the snapshotted values +// note: this method is not thread safe +func (t *ResettingTimerSnapshot) Mean() float64 { + if !t.calculated { + t.calc(nil) + } + + return t.mean +} + +// Max returns the max of the snapshotted values +// note: this method is not thread safe +func (t *ResettingTimerSnapshot) Max() int64 { + if !t.calculated { + t.calc(nil) + } + return t.max +} + +// Min returns the min of the snapshotted values +// note: this method is not thread safe +func (t *ResettingTimerSnapshot) Min() int64 { + if !t.calculated { + t.calc(nil) + } + return t.min +} + +func (t *ResettingTimerSnapshot) calc(percentiles []float64) { + scores := CalculatePercentiles(t.values, percentiles) + t.thresholdBoundaries = scores + if len(t.values) == 0 { + return + } + t.min = t.values[0] + t.max = t.values[len(t.values)-1] +} diff --git a/common/metrics/resetting_timer_test.go b/common/metrics/resetting_timer_test.go new file mode 100644 index 00000000000..4571fc8eb05 --- /dev/null +++ b/common/metrics/resetting_timer_test.go @@ -0,0 +1,197 @@ +package metrics + +import ( + "testing" + "time" +) + +func TestResettingTimer(t *testing.T) { + tests := []struct { + values []int64 + start int + end int + wantP50 float64 + wantP95 float64 + wantP99 float64 + wantMean float64 + wantMin int64 + wantMax int64 + }{ + { + values: []int64{}, + start: 1, + end: 11, + wantP50: 5.5, wantP95: 10, wantP99: 10, + wantMin: 1, wantMax: 10, wantMean: 5.5, + }, + { + values: []int64{}, + start: 1, + end: 101, + wantP50: 50.5, wantP95: 95.94999999999999, wantP99: 99.99, + wantMin: 1, wantMax: 100, wantMean: 50.5, + }, + { + values: []int64{1}, + start: 0, + end: 0, + wantP50: 1, wantP95: 1, wantP99: 1, + wantMin: 1, wantMax: 1, wantMean: 1, + }, + { + values: []int64{0}, + start: 0, + end: 0, + wantP50: 0, wantP95: 0, wantP99: 0, + wantMin: 0, wantMax: 0, wantMean: 0, + }, + { + values: []int64{}, + start: 0, + end: 0, + wantP50: 0, wantP95: 0, wantP99: 0, + wantMin: 0, wantMax: 0, wantMean: 0, + }, + { + values: []int64{1, 10}, + start: 0, + end: 0, + wantP50: 5.5, wantP95: 10, wantP99: 10, + wantMin: 1, wantMax: 10, wantMean: 5.5, + }, + } + for i, tt := range tests { + timer := NewResettingTimer() + + for i := tt.start; i < tt.end; i++ { + tt.values = append(tt.values, int64(i)) + } + + for _, v := range tt.values { + timer.Update(time.Duration(v)) + } + snap := timer.Snapshot() + + ps := snap.Percentiles([]float64{0.50, 0.95, 0.99}) + + if have, want := snap.Min(), tt.wantMin; have != want { + t.Fatalf("%d: min: have %d, want %d", i, have, want) + } + if have, want := snap.Max(), tt.wantMax; have != want { + t.Fatalf("%d: max: have %d, want %d", i, have, want) + } + if have, want := snap.Mean(), tt.wantMean; have != want { + t.Fatalf("%d: mean: have %v, want %v", i, have, want) + } + if have, want := ps[0], tt.wantP50; have != want { + t.Errorf("%d: p50: have %v, want %v", i, have, want) + } + if have, want := ps[1], tt.wantP95; have != want { + t.Errorf("%d: p95: have %v, want %v", i, have, want) + } + if have, want := ps[2], tt.wantP99; have != want { + t.Errorf("%d: p99: have %v, want %v", i, have, want) + } + } +} + +func TestResettingTimerWithFivePercentiles(t *testing.T) { + tests := []struct { + values []int64 + start int + end int + wantP05 float64 + wantP20 float64 + wantP50 float64 + wantP95 float64 + wantP99 float64 + wantMean float64 + wantMin int64 + wantMax int64 + }{ + { + values: []int64{}, + start: 1, + end: 11, + wantP05: 1, wantP20: 2.2, wantP50: 5.5, wantP95: 10, wantP99: 10, + wantMin: 1, wantMax: 10, wantMean: 5.5, + }, + { + values: []int64{}, + start: 1, + end: 101, + wantP05: 5.050000000000001, wantP20: 20.200000000000003, wantP50: 50.5, wantP95: 95.94999999999999, wantP99: 99.99, + wantMin: 1, wantMax: 100, wantMean: 50.5, + }, + { + values: []int64{1}, + start: 0, + end: 0, + wantP05: 1, wantP20: 1, wantP50: 1, wantP95: 1, wantP99: 1, + wantMin: 1, wantMax: 1, wantMean: 1, + }, + { + values: []int64{0}, + start: 0, + end: 0, + wantP05: 0, wantP20: 0, wantP50: 0, wantP95: 0, wantP99: 0, + wantMin: 0, wantMax: 0, wantMean: 0, + }, + { + values: []int64{}, + start: 0, + end: 0, + wantP05: 0, wantP20: 0, wantP50: 0, wantP95: 0, wantP99: 0, + wantMin: 0, wantMax: 0, wantMean: 0, + }, + { + values: []int64{1, 10}, + start: 0, + end: 0, + wantP05: 1, wantP20: 1, wantP50: 5.5, wantP95: 10, wantP99: 10, + wantMin: 1, wantMax: 10, wantMean: 5.5, + }, + } + for ind, tt := range tests { + timer := NewResettingTimer() + + for i := tt.start; i < tt.end; i++ { + tt.values = append(tt.values, int64(i)) + } + + for _, v := range tt.values { + timer.Update(time.Duration(v)) + } + + snap := timer.Snapshot() + + ps := snap.Percentiles([]float64{0.05, 0.20, 0.50, 0.95, 0.99}) + + if tt.wantMin != snap.Min() { + t.Errorf("%d: min: got %d, want %d", ind, snap.Min(), tt.wantMin) + } + + if tt.wantMax != snap.Max() { + t.Errorf("%d: max: got %d, want %d", ind, snap.Max(), tt.wantMax) + } + + if tt.wantMean != snap.Mean() { + t.Errorf("%d: mean: got %.2f, want %.2f", ind, snap.Mean(), tt.wantMean) + } + if tt.wantP05 != ps[0] { + t.Errorf("%d: p05: got %v, want %v", ind, ps[0], tt.wantP05) + } + if tt.wantP20 != ps[1] { + t.Errorf("%d: p20: got %v, want %v", ind, ps[1], tt.wantP20) + } + if tt.wantP50 != ps[2] { + t.Errorf("%d: p50: got %v, want %v", ind, ps[2], tt.wantP50) + } + if tt.wantP95 != ps[3] { + t.Errorf("%d: p95: got %v, want %v", ind, ps[3], tt.wantP95) + } + if tt.wantP99 != ps[4] { + t.Errorf("%d: p99: got %v, want %v", ind, ps[4], tt.wantP99) + } + } +} diff --git a/common/metrics/runtimehistogram.go b/common/metrics/runtimehistogram.go new file mode 100644 index 00000000000..efbed498af8 --- /dev/null +++ b/common/metrics/runtimehistogram.go @@ -0,0 +1,298 @@ +package metrics + +import ( + "math" + "runtime/metrics" + "sort" + "sync/atomic" +) + +func getOrRegisterRuntimeHistogram(name string, scale float64, r Registry) *runtimeHistogram { + constructor := func() Histogram { return newRuntimeHistogram(scale) } + return getOrRegister(name, constructor, r).(*runtimeHistogram) +} + +// runtimeHistogram wraps a runtime/metrics histogram. +type runtimeHistogram struct { + v atomic.Pointer[metrics.Float64Histogram] + scaleFactor float64 +} + +func newRuntimeHistogram(scale float64) *runtimeHistogram { + h := &runtimeHistogram{scaleFactor: scale} + h.update(new(metrics.Float64Histogram)) + return h +} + +func RuntimeHistogramFromData(scale float64, hist *metrics.Float64Histogram) *runtimeHistogram { + h := &runtimeHistogram{scaleFactor: scale} + h.update(hist) + return h +} + +func (h *runtimeHistogram) update(mh *metrics.Float64Histogram) { + if mh == nil { + // The update value can be nil if the current Go version doesn't support a + // requested metric. It's just easier to handle nil here than putting + // conditionals everywhere. + return + } + + s := metrics.Float64Histogram{ + Counts: make([]uint64, len(mh.Counts)), + Buckets: make([]float64, len(mh.Buckets)), + } + copy(s.Counts, mh.Counts) + for i, b := range mh.Buckets { + s.Buckets[i] = b * h.scaleFactor + } + h.v.Store(&s) +} + +func (h *runtimeHistogram) Clear() { + panic("runtimeHistogram does not support Clear") +} +func (h *runtimeHistogram) Update(int64) { + panic("runtimeHistogram does not support Update") +} + +// Snapshot returns a non-changing copy of the histogram. +func (h *runtimeHistogram) Snapshot() HistogramSnapshot { + hist := h.v.Load() + return newRuntimeHistogramSnapshot(hist) +} + +type runtimeHistogramSnapshot struct { + internal *metrics.Float64Histogram + calculated bool + // The following fields are (lazily) calculated based on 'internal' + mean float64 + count int64 + min int64 // min is the lowest sample value. + max int64 // max is the highest sample value. + variance float64 +} + +func newRuntimeHistogramSnapshot(h *metrics.Float64Histogram) *runtimeHistogramSnapshot { + return &runtimeHistogramSnapshot{ + internal: h, + } +} + +// calc calculates the values for the snapshot. This method is not threadsafe. +func (h *runtimeHistogramSnapshot) calc() { + h.calculated = true + var ( + count int64 // number of samples + sum float64 // approx sum of all sample values + min int64 + max float64 + ) + if len(h.internal.Counts) == 0 { + return + } + for i, c := range h.internal.Counts { + if c == 0 { + continue + } + if count == 0 { // Set min only first loop iteration + min = int64(math.Floor(h.internal.Buckets[i])) + } + count += int64(c) + sum += h.midpoint(i) * float64(c) + // Set max on every iteration + edge := h.internal.Buckets[i+1] + if math.IsInf(edge, 1) { + edge = h.internal.Buckets[i] + } + if edge > max { + max = edge + } + } + h.min = min + h.max = int64(max) + h.mean = sum / float64(count) + h.count = count +} + +// Count returns the sample count. +func (h *runtimeHistogramSnapshot) Count() int64 { + if !h.calculated { + h.calc() + } + return h.count +} + +// Size returns the size of the sample at the time the snapshot was taken. +func (h *runtimeHistogramSnapshot) Size() int { + return len(h.internal.Counts) +} + +// Mean returns an approximation of the mean. +func (h *runtimeHistogramSnapshot) Mean() float64 { + if !h.calculated { + h.calc() + } + return h.mean +} + +func (h *runtimeHistogramSnapshot) midpoint(bucket int) float64 { + high := h.internal.Buckets[bucket+1] + low := h.internal.Buckets[bucket] + if math.IsInf(high, 1) { + // The edge of the highest bucket can be +Inf, and it's supposed to mean that this + // bucket contains all remaining samples > low. We can't get the middle of an + // infinite range, so just return the lower bound of this bucket instead. + return low + } + if math.IsInf(low, -1) { + // Similarly, we can get -Inf in the left edge of the lowest bucket, + // and it means the bucket contains all remaining values < high. + return high + } + return (low + high) / 2 +} + +// StdDev approximates the standard deviation of the histogram. +func (h *runtimeHistogramSnapshot) StdDev() float64 { + return math.Sqrt(h.Variance()) +} + +// Variance approximates the variance of the histogram. +func (h *runtimeHistogramSnapshot) Variance() float64 { + if len(h.internal.Counts) == 0 { + return 0 + } + if !h.calculated { + h.calc() + } + if h.count <= 1 { + // There is no variance when there are zero or one items. + return 0 + } + // Variance is not calculated in 'calc', because it requires a second iteration. + // Therefore we calculate it lazily in this method, triggered either by + // a direct call to Variance or via StdDev. + if h.variance != 0.0 { + return h.variance + } + var sum float64 + + for i, c := range h.internal.Counts { + midpoint := h.midpoint(i) + d := midpoint - h.mean + sum += float64(c) * (d * d) + } + h.variance = sum / float64(h.count-1) + return h.variance +} + +// Percentile computes the p'th percentile value. +func (h *runtimeHistogramSnapshot) Percentile(p float64) float64 { + threshold := float64(h.Count()) * p + values := [1]float64{threshold} + h.computePercentiles(values[:]) + return values[0] +} + +// Percentiles computes all requested percentile values. +func (h *runtimeHistogramSnapshot) Percentiles(ps []float64) []float64 { + // Compute threshold values. We need these to be sorted + // for the percentile computation, but restore the original + // order later, so keep the indexes as well. + count := float64(h.Count()) + thresholds := make([]float64, len(ps)) + indexes := make([]int, len(ps)) + for i, percentile := range ps { + thresholds[i] = count * max(0, min(1.0, percentile)) + indexes[i] = i + } + sort.Sort(floatsAscendingKeepingIndex{thresholds, indexes}) + + // Now compute. The result is stored back into the thresholds slice. + h.computePercentiles(thresholds) + + // Put the result back into the requested order. + sort.Sort(floatsByIndex{thresholds, indexes}) + return thresholds +} + +func (h *runtimeHistogramSnapshot) computePercentiles(thresh []float64) { + var totalCount float64 + for i, count := range h.internal.Counts { + totalCount += float64(count) + + for len(thresh) > 0 && thresh[0] < totalCount { + thresh[0] = h.internal.Buckets[i] + thresh = thresh[1:] + } + if len(thresh) == 0 { + return + } + } +} + +// Note: runtime/metrics.Float64Histogram is a collection of float64s, but the methods +// below need to return int64 to satisfy the interface. The histogram provided by runtime +// also doesn't keep track of individual samples, so results are approximated. + +// Max returns the highest sample value. +func (h *runtimeHistogramSnapshot) Max() int64 { + if !h.calculated { + h.calc() + } + return h.max +} + +// Min returns the lowest sample value. +func (h *runtimeHistogramSnapshot) Min() int64 { + if !h.calculated { + h.calc() + } + return h.min +} + +// Sum returns the sum of all sample values. +func (h *runtimeHistogramSnapshot) Sum() int64 { + var sum float64 + for i := range h.internal.Counts { + sum += h.internal.Buckets[i] * float64(h.internal.Counts[i]) + } + return int64(math.Ceil(sum)) +} + +type floatsAscendingKeepingIndex struct { + values []float64 + indexes []int +} + +func (s floatsAscendingKeepingIndex) Len() int { + return len(s.values) +} + +func (s floatsAscendingKeepingIndex) Less(i, j int) bool { + return s.values[i] < s.values[j] +} + +func (s floatsAscendingKeepingIndex) Swap(i, j int) { + s.values[i], s.values[j] = s.values[j], s.values[i] + s.indexes[i], s.indexes[j] = s.indexes[j], s.indexes[i] +} + +type floatsByIndex struct { + values []float64 + indexes []int +} + +func (s floatsByIndex) Len() int { + return len(s.values) +} + +func (s floatsByIndex) Less(i, j int) bool { + return s.indexes[i] < s.indexes[j] +} + +func (s floatsByIndex) Swap(i, j int) { + s.values[i], s.values[j] = s.values[j], s.values[i] + s.indexes[i], s.indexes[j] = s.indexes[j], s.indexes[i] +} diff --git a/common/metrics/runtimehistogram_test.go b/common/metrics/runtimehistogram_test.go new file mode 100644 index 00000000000..cf7e36420ae --- /dev/null +++ b/common/metrics/runtimehistogram_test.go @@ -0,0 +1,162 @@ +package metrics + +import ( + "bytes" + "encoding/gob" + "fmt" + "math" + "reflect" + "runtime/metrics" + "testing" + "time" +) + +var _ Histogram = (*runtimeHistogram)(nil) + +type runtimeHistogramTest struct { + h metrics.Float64Histogram + + Count int64 + Min int64 + Max int64 + Sum int64 + Mean float64 + Variance float64 + StdDev float64 + Percentiles []float64 // .5 .8 .9 .99 .995 +} + +// This test checks the results of statistical functions implemented +// by runtimeHistogramSnapshot. +func TestRuntimeHistogramStats(t *testing.T) { + tests := []runtimeHistogramTest{ + 0: { + h: metrics.Float64Histogram{ + Counts: []uint64{}, + Buckets: []float64{}, + }, + Count: 0, + Max: 0, + Min: 0, + Sum: 0, + Mean: 0, + Variance: 0, + StdDev: 0, + Percentiles: []float64{0, 0, 0, 0, 0}, + }, + 1: { + // This checks the case where the highest bucket is +Inf. + h: metrics.Float64Histogram{ + Counts: []uint64{0, 1, 2}, + Buckets: []float64{0, 0.5, 1, math.Inf(1)}, + }, + Count: 3, + Max: 1, + Min: 0, + Sum: 3, + Mean: 0.9166666, + Percentiles: []float64{1, 1, 1, 1, 1}, + Variance: 0.020833, + StdDev: 0.144433, + }, + 2: { + h: metrics.Float64Histogram{ + Counts: []uint64{8, 6, 3, 1}, + Buckets: []float64{12, 16, 18, 24, 25}, + }, + Count: 18, + Max: 25, + Min: 12, + Sum: 270, + Mean: 16.75, + Variance: 10.3015, + StdDev: 3.2096, + Percentiles: []float64{16, 18, 18, 24, 24}, + }, + } + + for i, test := range tests { + t.Run(fmt.Sprint(i), func(t *testing.T) { + s := RuntimeHistogramFromData(1.0, &test.h).Snapshot() + + if v := s.Count(); v != test.Count { + t.Errorf("Count() = %v, want %v", v, test.Count) + } + if v := s.Min(); v != test.Min { + t.Errorf("Min() = %v, want %v", v, test.Min) + } + if v := s.Max(); v != test.Max { + t.Errorf("Max() = %v, want %v", v, test.Max) + } + if v := s.Sum(); v != test.Sum { + t.Errorf("Sum() = %v, want %v", v, test.Sum) + } + if v := s.Mean(); !approxEqual(v, test.Mean, 0.0001) { + t.Errorf("Mean() = %v, want %v", v, test.Mean) + } + if v := s.Variance(); !approxEqual(v, test.Variance, 0.0001) { + t.Errorf("Variance() = %v, want %v", v, test.Variance) + } + if v := s.StdDev(); !approxEqual(v, test.StdDev, 0.0001) { + t.Errorf("StdDev() = %v, want %v", v, test.StdDev) + } + ps := []float64{.5, .8, .9, .99, .995} + if v := s.Percentiles(ps); !reflect.DeepEqual(v, test.Percentiles) { + t.Errorf("Percentiles(%v) = %v, want %v", ps, v, test.Percentiles) + } + }) + } +} + +func approxEqual(x, y, ε float64) bool { + if math.IsInf(x, -1) && math.IsInf(y, -1) { + return true + } + if math.IsInf(x, 1) && math.IsInf(y, 1) { + return true + } + if math.IsNaN(x) && math.IsNaN(y) { + return true + } + return math.Abs(x-y) < ε +} + +// This test verifies that requesting Percentiles in unsorted order +// returns them in the requested order. +func TestRuntimeHistogramStatsPercentileOrder(t *testing.T) { + s := RuntimeHistogramFromData(1.0, &metrics.Float64Histogram{ + Counts: []uint64{1, 1, 1, 1, 1, 1, 1, 1, 1, 1}, + Buckets: []float64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, + }).Snapshot() + result := s.Percentiles([]float64{1, 0.2, 0.5, 0.1, 0.2}) + expected := []float64{10, 2, 5, 1, 2} + if !reflect.DeepEqual(result, expected) { + t.Fatal("wrong result:", result) + } +} + +func BenchmarkRuntimeHistogramSnapshotRead(b *testing.B) { + var sLatency = "7\xff\x81\x03\x01\x01\x10Float64Histogram\x01\xff\x82\x00\x01\x02\x01\x06Counts\x01\xff\x84\x00\x01\aBuckets\x01\xff\x86\x00\x00\x00\x16\xff\x83\x02\x01\x01\b[]uint64\x01\xff\x84\x00\x01\x06\x00\x00\x17\xff\x85\x02\x01\x01\t[]float64\x01\xff\x86\x00\x01\b\x00\x00\xfe\x06T\xff\x82\x01\xff\xa2\x00\xfe\r\xef\x00\x01\x02\x02\x04\x05\x04\b\x15\x17 B?6.L;$!2) \x1a? \x190aH7FY6#\x190\x1d\x14\x10\x1b\r\t\x04\x03\x01\x01\x00\x03\x02\x00\x03\x05\x05\x02\x02\x06\x04\v\x06\n\x15\x18\x13'&.\x12=H/L&\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\xff\xa3\xfe\xf0\xff\x00\xf8\x95\xd6&\xe8\v.q>\xf8\x95\xd6&\xe8\v.\x81>\xf8\xdfA:\xdc\x11ʼn>\xf8\x95\xd6&\xe8\v.\x91>\xf8:\x8c0\xe2\x8ey\x95>\xf8\xdfA:\xdc\x11ř>\xf8\x84\xf7C֔\x10\x9e>\xf8\x95\xd6&\xe8\v.\xa1>\xf8:\x8c0\xe2\x8ey\xa5>\xf8\xdfA:\xdc\x11ũ>\xf8\x84\xf7C֔\x10\xae>\xf8\x95\xd6&\xe8\v.\xb1>\xf8:\x8c0\xe2\x8ey\xb5>\xf8\xdfA:\xdc\x11Ź>\xf8\x84\xf7C֔\x10\xbe>\xf8\x95\xd6&\xe8\v.\xc1>\xf8:\x8c0\xe2\x8ey\xc5>\xf8\xdfA:\xdc\x11\xc5\xc9>\xf8\x84\xf7C֔\x10\xce>\xf8\x95\xd6&\xe8\v.\xd1>\xf8:\x8c0\xe2\x8ey\xd5>\xf8\xdfA:\xdc\x11\xc5\xd9>\xf8\x84\xf7C֔\x10\xde>\xf8\x95\xd6&\xe8\v.\xe1>\xf8:\x8c0\xe2\x8ey\xe5>\xf8\xdfA:\xdc\x11\xc5\xe9>\xf8\x84\xf7C֔\x10\xee>\xf8\x95\xd6&\xe8\v.\xf1>\xf8:\x8c0\xe2\x8ey\xf5>\xf8\xdfA:\xdc\x11\xc5\xf9>\xf8\x84\xf7C֔\x10\xfe>\xf8\x95\xd6&\xe8\v.\x01?\xf8:\x8c0\xe2\x8ey\x05?\xf8\xdfA:\xdc\x11\xc5\t?\xf8\x84\xf7C֔\x10\x0e?\xf8\x95\xd6&\xe8\v.\x11?\xf8:\x8c0\xe2\x8ey\x15?\xf8\xdfA:\xdc\x11\xc5\x19?\xf8\x84\xf7C֔\x10\x1e?\xf8\x95\xd6&\xe8\v.!?\xf8:\x8c0\xe2\x8ey%?\xf8\xdfA:\xdc\x11\xc5)?\xf8\x84\xf7C֔\x10.?\xf8\x95\xd6&\xe8\v.1?\xf8:\x8c0\xe2\x8ey5?\xf8\xdfA:\xdc\x11\xc59?\xf8\x84\xf7C֔\x10>?\xf8\x95\xd6&\xe8\v.A?\xf8:\x8c0\xe2\x8eyE?\xf8\xdfA:\xdc\x11\xc5I?\xf8\x84\xf7C֔\x10N?\xf8\x95\xd6&\xe8\v.Q?\xf8:\x8c0\xe2\x8eyU?\xf8\xdfA:\xdc\x11\xc5Y?\xf8\x84\xf7C֔\x10^?\xf8\x95\xd6&\xe8\v.a?\xf8:\x8c0\xe2\x8eye?\xf8\xdfA:\xdc\x11\xc5i?\xf8\x84\xf7C֔\x10n?\xf8\x95\xd6&\xe8\v.q?\xf8:\x8c0\xe2\x8eyu?\xf8\xdfA:\xdc\x11\xc5y?\xf8\x84\xf7C֔\x10~?\xf8\x95\xd6&\xe8\v.\x81?\xf8:\x8c0\xe2\x8ey\x85?\xf8\xdfA:\xdc\x11ʼn?\xf8\x84\xf7C֔\x10\x8e?\xf8\x95\xd6&\xe8\v.\x91?\xf8:\x8c0\xe2\x8ey\x95?\xf8\xdfA:\xdc\x11ř?\xf8\x84\xf7C֔\x10\x9e?\xf8\x95\xd6&\xe8\v.\xa1?\xf8:\x8c0\xe2\x8ey\xa5?\xf8\xdfA:\xdc\x11ũ?\xf8\x84\xf7C֔\x10\xae?\xf8\x95\xd6&\xe8\v.\xb1?\xf8:\x8c0\xe2\x8ey\xb5?\xf8\xdfA:\xdc\x11Ź?\xf8\x84\xf7C֔\x10\xbe?\xf8\x95\xd6&\xe8\v.\xc1?\xf8:\x8c0\xe2\x8ey\xc5?\xf8\xdfA:\xdc\x11\xc5\xc9?\xf8\x84\xf7C֔\x10\xce?\xf8\x95\xd6&\xe8\v.\xd1?\xf8:\x8c0\xe2\x8ey\xd5?\xf8\xdfA:\xdc\x11\xc5\xd9?\xf8\x84\xf7C֔\x10\xde?\xf8\x95\xd6&\xe8\v.\xe1?\xf8:\x8c0\xe2\x8ey\xe5?\xf8\xdfA:\xdc\x11\xc5\xe9?\xf8\x84\xf7C֔\x10\xee?\xf8\x95\xd6&\xe8\v.\xf1?\xf8:\x8c0\xe2\x8ey\xf5?\xf8\xdfA:\xdc\x11\xc5\xf9?\xf8\x84\xf7C֔\x10\xfe?\xf8\x95\xd6&\xe8\v.\x01@\xf8:\x8c0\xe2\x8ey\x05@\xf8\xdfA:\xdc\x11\xc5\t@\xf8\x84\xf7C֔\x10\x0e@\xf8\x95\xd6&\xe8\v.\x11@\xf8:\x8c0\xe2\x8ey\x15@\xf8\xdfA:\xdc\x11\xc5\x19@\xf8\x84\xf7C֔\x10\x1e@\xf8\x95\xd6&\xe8\v.!@\xf8:\x8c0\xe2\x8ey%@\xf8\xdfA:\xdc\x11\xc5)@\xf8\x84\xf7C֔\x10.@\xf8\x95\xd6&\xe8\v.1@\xf8:\x8c0\xe2\x8ey5@\xf8\xdfA:\xdc\x11\xc59@\xf8\x84\xf7C֔\x10>@\xf8\x95\xd6&\xe8\v.A@\xf8:\x8c0\xe2\x8eyE@\xf8\xdfA:\xdc\x11\xc5I@\xf8\x84\xf7C֔\x10N@\xf8\x95\xd6&\xe8\v.Q@\xf8:\x8c0\xe2\x8eyU@\xf8\xdfA:\xdc\x11\xc5Y@\xf8\x84\xf7C֔\x10^@\xf8\x95\xd6&\xe8\v.a@\xf8:\x8c0\xe2\x8eye@\xf8\xdfA:\xdc\x11\xc5i@\xf8\x84\xf7C֔\x10n@\xf8\x95\xd6&\xe8\v.q@\xf8:\x8c0\xe2\x8eyu@\xf8\xdfA:\xdc\x11\xc5y@\xf8\x84\xf7C֔\x10~@\xf8\x95\xd6&\xe8\v.\x81@\xf8:\x8c0\xe2\x8ey\x85@\xf8\xdfA:\xdc\x11ʼn@\xf8\x84\xf7C֔\x10\x8e@\xf8\x95\xd6&\xe8\v.\x91@\xf8:\x8c0\xe2\x8ey\x95@\xf8\xdfA:\xdc\x11ř@\xf8\x84\xf7C֔\x10\x9e@\xf8\x95\xd6&\xe8\v.\xa1@\xf8:\x8c0\xe2\x8ey\xa5@\xf8\xdfA:\xdc\x11ũ@\xf8\x84\xf7C֔\x10\xae@\xf8\x95\xd6&\xe8\v.\xb1@\xf8:\x8c0\xe2\x8ey\xb5@\xf8\xdfA:\xdc\x11Ź@\xf8\x84\xf7C֔\x10\xbe@\xf8\x95\xd6&\xe8\v.\xc1@\xf8:\x8c0\xe2\x8ey\xc5@\xf8\xdfA:\xdc\x11\xc5\xc9@\xf8\x84\xf7C֔\x10\xce@\xf8\x95\xd6&\xe8\v.\xd1@\xf8:\x8c0\xe2\x8ey\xd5@\xf8\xdfA:\xdc\x11\xc5\xd9@\xf8\x84\xf7C֔\x10\xde@\xf8\x95\xd6&\xe8\v.\xe1@\xf8:\x8c0\xe2\x8ey\xe5@\xf8\xdfA:\xdc\x11\xc5\xe9@\xf8\x84\xf7C֔\x10\xee@\xf8\x95\xd6&\xe8\v.\xf1@\xf8:\x8c0\xe2\x8ey\xf5@\xf8\xdfA:\xdc\x11\xc5\xf9@\xf8\x84\xf7C֔\x10\xfe@\xf8\x95\xd6&\xe8\v.\x01A\xfe\xf0\x7f\x00" + + dserialize := func(data string) *metrics.Float64Histogram { + var res metrics.Float64Histogram + if err := gob.NewDecoder(bytes.NewReader([]byte(data))).Decode(&res); err != nil { + panic(err) + } + return &res + } + latency := RuntimeHistogramFromData(float64(time.Second), dserialize(sLatency)) + b.ResetTimer() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + snap := latency.Snapshot() + // These are the fields that influxdb accesses + _ = snap.Count() + _ = snap.Max() + _ = snap.Mean() + _ = snap.Min() + _ = snap.StdDev() + _ = snap.Variance() + _ = snap.Percentiles([]float64{0.25, 0.5, 0.75, 0.95, 0.99, 0.999, 0.9999}) + } +} diff --git a/common/metrics/sample.go b/common/metrics/sample.go new file mode 100644 index 00000000000..dc8167809fd --- /dev/null +++ b/common/metrics/sample.go @@ -0,0 +1,425 @@ +package metrics + +import ( + "math" + "math/rand" + "slices" + "sync" + "time" +) + +const rescaleThreshold = time.Hour + +// Sample maintains a statistically-significant selection of values from +// a stream. +type Sample interface { + Snapshot() *sampleSnapshot + Clear() + Update(int64) +} + +var ( + _ Sample = (*ExpDecaySample)(nil) + _ Sample = (*UniformSample)(nil) + _ Sample = (*resettingSample)(nil) +) + +// sampleSnapshot is a read-only copy of a Sample. +type sampleSnapshot struct { + count int64 + values []int64 + + max int64 + min int64 + mean float64 + sum int64 + variance float64 +} + +// newSampleSnapshotPrecalculated creates a read-only sampleSnapShot, using +// precalculated sums to avoid iterating the values +func newSampleSnapshotPrecalculated(count int64, values []int64, min, max, sum int64) *sampleSnapshot { + if len(values) == 0 { + return &sampleSnapshot{ + count: count, + values: values, + } + } + return &sampleSnapshot{ + count: count, + values: values, + max: max, + min: min, + mean: float64(sum) / float64(len(values)), + sum: sum, + } +} + +// newSampleSnapshot creates a read-only sampleSnapShot, and calculates some +// numbers. +func newSampleSnapshot(count int64, values []int64) *sampleSnapshot { + var ( + max int64 = math.MinInt64 + min int64 = math.MaxInt64 + sum int64 + ) + for _, v := range values { + sum += v + if v > max { + max = v + } + if v < min { + min = v + } + } + return newSampleSnapshotPrecalculated(count, values, min, max, sum) +} + +// Count returns the count of inputs at the time the snapshot was taken. +func (s *sampleSnapshot) Count() int64 { return s.count } + +// Max returns the maximal value at the time the snapshot was taken. +func (s *sampleSnapshot) Max() int64 { return s.max } + +// Mean returns the mean value at the time the snapshot was taken. +func (s *sampleSnapshot) Mean() float64 { return s.mean } + +// Min returns the minimal value at the time the snapshot was taken. +func (s *sampleSnapshot) Min() int64 { return s.min } + +// Percentile returns an arbitrary percentile of values at the time the +// snapshot was taken. +func (s *sampleSnapshot) Percentile(p float64) float64 { + return SamplePercentile(s.values, p) +} + +// Percentiles returns a slice of arbitrary percentiles of values at the time +// the snapshot was taken. +func (s *sampleSnapshot) Percentiles(ps []float64) []float64 { + return CalculatePercentiles(s.values, ps) +} + +// Size returns the size of the sample at the time the snapshot was taken. +func (s *sampleSnapshot) Size() int { return len(s.values) } + +// StdDev returns the standard deviation of values at the time the snapshot was +// taken. +func (s *sampleSnapshot) StdDev() float64 { + if s.variance == 0.0 { + s.variance = SampleVariance(s.mean, s.values) + } + return math.Sqrt(s.variance) +} + +// Sum returns the sum of values at the time the snapshot was taken. +func (s *sampleSnapshot) Sum() int64 { return s.sum } + +// Values returns a copy of the values in the sample. +func (s *sampleSnapshot) Values() []int64 { + return slices.Clone(s.values) +} + +// Variance returns the variance of values at the time the snapshot was taken. +func (s *sampleSnapshot) Variance() float64 { + if s.variance == 0.0 { + s.variance = SampleVariance(s.mean, s.values) + } + return s.variance +} + +// ExpDecaySample is an exponentially-decaying sample using a forward-decaying +// priority reservoir. See Cormode et al's "Forward Decay: A Practical Time +// Decay Model for Streaming Systems". +// +// +type ExpDecaySample struct { + alpha float64 + count int64 + mutex sync.Mutex + reservoirSize int + t0, t1 time.Time + values *expDecaySampleHeap + rand *rand.Rand +} + +// NewExpDecaySample constructs a new exponentially-decaying sample with the +// given reservoir size and alpha. +func NewExpDecaySample(reservoirSize int, alpha float64) Sample { + s := &ExpDecaySample{ + alpha: alpha, + reservoirSize: reservoirSize, + t0: time.Now(), + values: newExpDecaySampleHeap(reservoirSize), + } + s.t1 = s.t0.Add(rescaleThreshold) + return s +} + +// SetRand sets the random source (useful in tests) +func (s *ExpDecaySample) SetRand(prng *rand.Rand) Sample { + s.rand = prng + return s +} + +// Clear clears all samples. +func (s *ExpDecaySample) Clear() { + s.mutex.Lock() + defer s.mutex.Unlock() + s.count = 0 + s.t0 = time.Now() + s.t1 = s.t0.Add(rescaleThreshold) + s.values.Clear() +} + +// Snapshot returns a read-only copy of the sample. +func (s *ExpDecaySample) Snapshot() *sampleSnapshot { + s.mutex.Lock() + defer s.mutex.Unlock() + var ( + samples = s.values.Values() + values = make([]int64, len(samples)) + max int64 = math.MinInt64 + min int64 = math.MaxInt64 + sum int64 + ) + for i, item := range samples { + v := item.v + values[i] = v + sum += v + if v > max { + max = v + } + if v < min { + min = v + } + } + return newSampleSnapshotPrecalculated(s.count, values, min, max, sum) +} + +// Update samples a new value. +func (s *ExpDecaySample) Update(v int64) { + if !metricsEnabled { + return + } + s.update(time.Now(), v) +} + +// update samples a new value at a particular timestamp. This is a method all +// its own to facilitate testing. +func (s *ExpDecaySample) update(t time.Time, v int64) { + s.mutex.Lock() + defer s.mutex.Unlock() + s.count++ + if s.values.Size() == s.reservoirSize { + s.values.Pop() + } + var f64 float64 + if s.rand != nil { + f64 = s.rand.Float64() + } else { + f64 = rand.Float64() + } + s.values.Push(expDecaySample{ + k: math.Exp(t.Sub(s.t0).Seconds()*s.alpha) / f64, + v: v, + }) + if t.After(s.t1) { + values := s.values.Values() + t0 := s.t0 + s.values.Clear() + s.t0 = t + s.t1 = s.t0.Add(rescaleThreshold) + for _, v := range values { + v.k = v.k * math.Exp(-s.alpha*s.t0.Sub(t0).Seconds()) + s.values.Push(v) + } + } +} + +// SamplePercentile returns an arbitrary percentile of the slice of int64. +func SamplePercentile(values []int64, p float64) float64 { + return CalculatePercentiles(values, []float64{p})[0] +} + +// CalculatePercentiles returns a slice of arbitrary percentiles of the slice of +// int64. This method returns interpolated results, so e.g. if there are only two +// values, [0, 10], a 50% percentile will land between them. +// +// Note: As a side-effect, this method will also sort the slice of values. +// Note2: The input format for percentiles is NOT percent! To express 50%, use 0.5, not 50. +func CalculatePercentiles(values []int64, ps []float64) []float64 { + scores := make([]float64, len(ps)) + size := len(values) + if size == 0 { + return scores + } + slices.Sort(values) + for i, p := range ps { + pos := p * float64(size+1) + + if pos < 1.0 { + scores[i] = float64(values[0]) + } else if pos >= float64(size) { + scores[i] = float64(values[size-1]) + } else { + lower := float64(values[int(pos)-1]) + upper := float64(values[int(pos)]) + scores[i] = lower + (pos-math.Floor(pos))*(upper-lower) + } + } + return scores +} + +// SampleVariance returns the variance of the slice of int64. +func SampleVariance(mean float64, values []int64) float64 { + if len(values) == 0 { + return 0.0 + } + var sum float64 + for _, v := range values { + d := float64(v) - mean + sum += d * d + } + return sum / float64(len(values)) +} + +// UniformSample implements a uniform sample using Vitter's Algorithm R. +// +// +type UniformSample struct { + count int64 + mutex sync.Mutex + reservoirSize int + values []int64 + rand *rand.Rand +} + +// NewUniformSample constructs a new uniform sample with the given reservoir +// size. +func NewUniformSample(reservoirSize int) Sample { + return &UniformSample{ + reservoirSize: reservoirSize, + values: make([]int64, 0, reservoirSize), + } +} + +// SetRand sets the random source (useful in tests) +func (s *UniformSample) SetRand(prng *rand.Rand) Sample { + s.rand = prng + return s +} + +// Clear clears all samples. +func (s *UniformSample) Clear() { + s.mutex.Lock() + defer s.mutex.Unlock() + s.count = 0 + clear(s.values) +} + +// Snapshot returns a read-only copy of the sample. +func (s *UniformSample) Snapshot() *sampleSnapshot { + s.mutex.Lock() + values := slices.Clone(s.values) + count := s.count + s.mutex.Unlock() + return newSampleSnapshot(count, values) +} + +// Update samples a new value. +func (s *UniformSample) Update(v int64) { + if !metricsEnabled { + return + } + s.mutex.Lock() + defer s.mutex.Unlock() + s.count++ + if len(s.values) < s.reservoirSize { + s.values = append(s.values, v) + return + } + var r int64 + if s.rand != nil { + r = s.rand.Int63n(s.count) + } else { + r = rand.Int63n(s.count) + } + if r < int64(len(s.values)) { + s.values[int(r)] = v + } +} + +// expDecaySample represents an individual sample in a heap. +type expDecaySample struct { + k float64 + v int64 +} + +func newExpDecaySampleHeap(reservoirSize int) *expDecaySampleHeap { + return &expDecaySampleHeap{make([]expDecaySample, 0, reservoirSize)} +} + +// expDecaySampleHeap is a min-heap of expDecaySamples. +// The internal implementation is copied from the standard library's container/heap +type expDecaySampleHeap struct { + s []expDecaySample +} + +func (h *expDecaySampleHeap) Clear() { + h.s = h.s[:0] +} + +func (h *expDecaySampleHeap) Push(s expDecaySample) { + n := len(h.s) + h.s = h.s[0 : n+1] + h.s[n] = s + h.up(n) +} + +func (h *expDecaySampleHeap) Pop() expDecaySample { + n := len(h.s) - 1 + h.s[0], h.s[n] = h.s[n], h.s[0] + h.down(0, n) + + n = len(h.s) + s := h.s[n-1] + h.s = h.s[0 : n-1] + return s +} + +func (h *expDecaySampleHeap) Size() int { + return len(h.s) +} + +func (h *expDecaySampleHeap) Values() []expDecaySample { + return h.s +} + +func (h *expDecaySampleHeap) up(j int) { + for { + i := (j - 1) / 2 // parent + if i == j || !(h.s[j].k < h.s[i].k) { + break + } + h.s[i], h.s[j] = h.s[j], h.s[i] + j = i + } +} + +func (h *expDecaySampleHeap) down(i, n int) { + for { + j1 := 2*i + 1 + if j1 >= n || j1 < 0 { // j1 < 0 after int overflow + break + } + j := j1 // left child + if j2 := j1 + 1; j2 < n && !(h.s[j1].k < h.s[j2].k) { + j = j2 // = 2*i + 2 // right child + } + if !(h.s[j].k < h.s[i].k) { + break + } + h.s[i], h.s[j] = h.s[j], h.s[i] + i = j + } +} diff --git a/common/metrics/sample_test.go b/common/metrics/sample_test.go new file mode 100644 index 00000000000..6619eb1e9eb --- /dev/null +++ b/common/metrics/sample_test.go @@ -0,0 +1,324 @@ +package metrics + +import ( + "math" + "math/rand" + "testing" + "time" +) + +const epsilonPercentile = .00000000001 + +// Benchmark{Compute,Copy}{1000,1000000} demonstrate that, even for relatively +// expensive computations like Variance, the cost of copying the Sample, as +// approximated by a make and copy, is much greater than the cost of the +// computation for small samples and only slightly less for large samples. +func BenchmarkCompute1000(b *testing.B) { + s := make([]int64, 1000) + var sum int64 + for i := 0; i < len(s); i++ { + s[i] = int64(i) + sum += int64(i) + } + mean := float64(sum) / float64(len(s)) + b.ResetTimer() + for i := 0; i < b.N; i++ { + SampleVariance(mean, s) + } +} + +func BenchmarkCompute1000000(b *testing.B) { + s := make([]int64, 1000000) + var sum int64 + for i := 0; i < len(s); i++ { + s[i] = int64(i) + sum += int64(i) + } + mean := float64(sum) / float64(len(s)) + b.ResetTimer() + for i := 0; i < b.N; i++ { + SampleVariance(mean, s) + } +} + +func BenchmarkExpDecaySample257(b *testing.B) { + benchmarkSample(b, NewExpDecaySample(257, 0.015)) +} + +func BenchmarkExpDecaySample514(b *testing.B) { + benchmarkSample(b, NewExpDecaySample(514, 0.015)) +} + +func BenchmarkExpDecaySample1028(b *testing.B) { + benchmarkSample(b, NewExpDecaySample(1028, 0.015)) +} + +func BenchmarkUniformSample257(b *testing.B) { + benchmarkSample(b, NewUniformSample(257)) +} + +func BenchmarkUniformSample514(b *testing.B) { + benchmarkSample(b, NewUniformSample(514)) +} + +func BenchmarkUniformSample1028(b *testing.B) { + benchmarkSample(b, NewUniformSample(1028)) +} + +func TestExpDecaySample(t *testing.T) { + for _, tc := range []struct { + reservoirSize int + alpha float64 + updates int + }{ + {100, 0.99, 10}, + {1000, 0.01, 100}, + {100, 0.99, 1000}, + } { + sample := NewExpDecaySample(tc.reservoirSize, tc.alpha) + for i := 0; i < tc.updates; i++ { + sample.Update(int64(i)) + } + snap := sample.Snapshot() + if have, want := int(snap.Count()), tc.updates; have != want { + t.Errorf("unexpected count: have %d want %d", have, want) + } + if have, want := snap.Size(), min(tc.updates, tc.reservoirSize); have != want { + t.Errorf("unexpected size: have %d want %d", have, want) + } + values := snap.values + if have, want := len(values), min(tc.updates, tc.reservoirSize); have != want { + t.Errorf("unexpected values length: have %d want %d", have, want) + } + for _, v := range values { + if v > int64(tc.updates) || v < 0 { + t.Errorf("out of range [0, %d]: %v", tc.updates, v) + } + } + } +} + +// This test makes sure that the sample's priority is not amplified by using +// nanosecond duration since start rather than second duration since start. +// The priority becomes +Inf quickly after starting if this is done, +// effectively freezing the set of samples until a rescale step happens. +func TestExpDecaySampleNanosecondRegression(t *testing.T) { + sw := NewExpDecaySample(1000, 0.99) + for i := 0; i < 1000; i++ { + sw.Update(10) + } + time.Sleep(1 * time.Millisecond) + for i := 0; i < 1000; i++ { + sw.Update(20) + } + v := sw.Snapshot().values + avg := float64(0) + for i := 0; i < len(v); i++ { + avg += float64(v[i]) + } + avg /= float64(len(v)) + if avg > 16 || avg < 14 { + t.Errorf("out of range [14, 16]: %v\n", avg) + } +} + +func TestExpDecaySampleRescale(t *testing.T) { + s := NewExpDecaySample(2, 0.001).(*ExpDecaySample) + s.update(time.Now(), 1) + s.update(time.Now().Add(time.Hour+time.Microsecond), 1) + for _, v := range s.values.Values() { + if v.k == 0.0 { + t.Fatal("v.k == 0.0") + } + } +} + +func TestExpDecaySampleSnapshot(t *testing.T) { + now := time.Now() + s := NewExpDecaySample(100, 0.99).(*ExpDecaySample).SetRand(rand.New(rand.NewSource(1))) + for i := 1; i <= 10000; i++ { + s.(*ExpDecaySample).update(now.Add(time.Duration(i)), int64(i)) + } + snapshot := s.Snapshot() + s.Update(1) + testExpDecaySampleStatistics(t, snapshot) +} + +func TestExpDecaySampleStatistics(t *testing.T) { + now := time.Now() + s := NewExpDecaySample(100, 0.99).(*ExpDecaySample).SetRand(rand.New(rand.NewSource(1))) + for i := 1; i <= 10000; i++ { + s.(*ExpDecaySample).update(now.Add(time.Duration(i)), int64(i)) + } + testExpDecaySampleStatistics(t, s.Snapshot()) +} + +func TestUniformSample(t *testing.T) { + sw := NewUniformSample(100) + for i := 0; i < 1000; i++ { + sw.Update(int64(i)) + } + s := sw.Snapshot() + if size := s.Count(); size != 1000 { + t.Errorf("s.Count(): 1000 != %v\n", size) + } + if size := s.Size(); size != 100 { + t.Errorf("s.Size(): 100 != %v\n", size) + } + values := s.values + + if l := len(values); l != 100 { + t.Errorf("len(s.Values()): 100 != %v\n", l) + } + for _, v := range values { + if v > 1000 || v < 0 { + t.Errorf("out of range [0, 1000]: %v\n", v) + } + } +} + +func TestUniformSampleIncludesTail(t *testing.T) { + sw := NewUniformSample(100) + max := 100 + for i := 0; i < max; i++ { + sw.Update(int64(i)) + } + v := sw.Snapshot().values + sum := 0 + exp := (max - 1) * max / 2 + for i := 0; i < len(v); i++ { + sum += int(v[i]) + } + if exp != sum { + t.Errorf("sum: %v != %v\n", exp, sum) + } +} + +func TestUniformSampleSnapshot(t *testing.T) { + s := NewUniformSample(100).(*UniformSample).SetRand(rand.New(rand.NewSource(1))) + for i := 1; i <= 10000; i++ { + s.Update(int64(i)) + } + snapshot := s.Snapshot() + s.Update(1) + testUniformSampleStatistics(t, snapshot) +} + +func TestUniformSampleStatistics(t *testing.T) { + s := NewUniformSample(100).(*UniformSample).SetRand(rand.New(rand.NewSource(1))) + for i := 1; i <= 10000; i++ { + s.Update(int64(i)) + } + testUniformSampleStatistics(t, s.Snapshot()) +} + +func benchmarkSample(b *testing.B, s Sample) { + for i := 0; i < b.N; i++ { + s.Update(1) + } +} + +func testExpDecaySampleStatistics(t *testing.T, s *sampleSnapshot) { + if sum := s.Sum(); sum != 496598 { + t.Errorf("s.Sum(): 496598 != %v\n", sum) + } + if count := s.Count(); count != 10000 { + t.Errorf("s.Count(): 10000 != %v\n", count) + } + if min := s.Min(); min != 107 { + t.Errorf("s.Min(): 107 != %v\n", min) + } + if max := s.Max(); max != 10000 { + t.Errorf("s.Max(): 10000 != %v\n", max) + } + if mean := s.Mean(); mean != 4965.98 { + t.Errorf("s.Mean(): 4965.98 != %v\n", mean) + } + if stdDev := s.StdDev(); stdDev != 2959.825156930727 { + t.Errorf("s.StdDev(): 2959.825156930727 != %v\n", stdDev) + } + ps := s.Percentiles([]float64{0.5, 0.75, 0.99}) + if ps[0] != 4615 { + t.Errorf("median: 4615 != %v\n", ps[0]) + } + if ps[1] != 7672 { + t.Errorf("75th percentile: 7672 != %v\n", ps[1]) + } + if ps[2] != 9998.99 { + t.Errorf("99th percentile: 9998.99 != %v\n", ps[2]) + } +} + +func testUniformSampleStatistics(t *testing.T, s *sampleSnapshot) { + if count := s.Count(); count != 10000 { + t.Errorf("s.Count(): 10000 != %v\n", count) + } + if min := s.Min(); min != 37 { + t.Errorf("s.Min(): 37 != %v\n", min) + } + if max := s.Max(); max != 9989 { + t.Errorf("s.Max(): 9989 != %v\n", max) + } + if mean := s.Mean(); mean != 4748.14 { + t.Errorf("s.Mean(): 4748.14 != %v\n", mean) + } + if stdDev := s.StdDev(); stdDev != 2826.684117548333 { + t.Errorf("s.StdDev(): 2826.684117548333 != %v\n", stdDev) + } + ps := s.Percentiles([]float64{0.5, 0.75, 0.99}) + if ps[0] != 4599 { + t.Errorf("median: 4599 != %v\n", ps[0]) + } + if ps[1] != 7380.5 { + t.Errorf("75th percentile: 7380.5 != %v\n", ps[1]) + } + if math.Abs(9986.429999999998-ps[2]) > epsilonPercentile { + t.Errorf("99th percentile: 9986.429999999998 != %v\n", ps[2]) + } +} + +// TestUniformSampleConcurrentUpdateCount would expose data race problems with +// concurrent Update and Count calls on Sample when test is called with -race +// argument +func TestUniformSampleConcurrentUpdateCount(t *testing.T) { + if testing.Short() { + t.Skip("skipping in short mode") + } + s := NewUniformSample(100) + for i := 0; i < 100; i++ { + s.Update(int64(i)) + } + quit := make(chan struct{}) + go func() { + t := time.NewTicker(10 * time.Millisecond) + defer t.Stop() + for { + select { + case <-t.C: + s.Update(rand.Int63()) + case <-quit: + t.Stop() + return + } + } + }() + for i := 0; i < 1000; i++ { + s.Snapshot().Count() + time.Sleep(5 * time.Millisecond) + } + quit <- struct{}{} +} + +func BenchmarkCalculatePercentiles(b *testing.B) { + pss := []float64{0.5, 0.75, 0.95, 0.99, 0.999, 0.9999} + var vals []int64 + for i := 0; i < 1000; i++ { + vals = append(vals, int64(rand.Int31())) + } + v := make([]int64, len(vals)) + b.ResetTimer() + for i := 0; i < b.N; i++ { + copy(v, vals) + _ = CalculatePercentiles(v, pss) + } +} diff --git a/common/metrics/syslog.go b/common/metrics/syslog.go new file mode 100644 index 00000000000..b265328f876 --- /dev/null +++ b/common/metrics/syslog.go @@ -0,0 +1,83 @@ +//go:build !windows +// +build !windows + +package metrics + +import ( + "fmt" + "log/syslog" + "time" +) + +// Syslog outputs each metric in the given registry to syslog periodically using +// the given syslogger. +func Syslog(r Registry, d time.Duration, w *syslog.Writer) { + for range time.Tick(d) { + r.Each(func(name string, i interface{}) { + switch metric := i.(type) { + case *Counter: + w.Info(fmt.Sprintf("counter %s: count: %d", name, metric.Snapshot().Count())) + case *CounterFloat64: + w.Info(fmt.Sprintf("counter %s: count: %f", name, metric.Snapshot().Count())) + case *Gauge: + w.Info(fmt.Sprintf("gauge %s: value: %d", name, metric.Snapshot().Value())) + case *GaugeFloat64: + w.Info(fmt.Sprintf("gauge %s: value: %f", name, metric.Snapshot().Value())) + case *GaugeInfo: + w.Info(fmt.Sprintf("gauge %s: value: %s", name, metric.Snapshot().Value())) + case *Healthcheck: + metric.Check() + w.Info(fmt.Sprintf("healthcheck %s: error: %v", name, metric.Error())) + case Histogram: + h := metric.Snapshot() + ps := h.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999}) + w.Info(fmt.Sprintf( + "histogram %s: count: %d min: %d max: %d mean: %.2f stddev: %.2f median: %.2f 75%%: %.2f 95%%: %.2f 99%%: %.2f 99.9%%: %.2f", + name, + h.Count(), + h.Min(), + h.Max(), + h.Mean(), + h.StdDev(), + ps[0], + ps[1], + ps[2], + ps[3], + ps[4], + )) + case *Meter: + m := metric.Snapshot() + w.Info(fmt.Sprintf( + "meter %s: count: %d 1-min: %.2f 5-min: %.2f 15-min: %.2f mean: %.2f", + name, + m.Count(), + m.Rate1(), + m.Rate5(), + m.Rate15(), + m.RateMean(), + )) + case *Timer: + t := metric.Snapshot() + ps := t.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999}) + w.Info(fmt.Sprintf( + "timer %s: count: %d min: %d max: %d mean: %.2f stddev: %.2f median: %.2f 75%%: %.2f 95%%: %.2f 99%%: %.2f 99.9%%: %.2f 1-min: %.2f 5-min: %.2f 15-min: %.2f mean-rate: %.2f", + name, + t.Count(), + t.Min(), + t.Max(), + t.Mean(), + t.StdDev(), + ps[0], + ps[1], + ps[2], + ps[3], + ps[4], + t.Rate1(), + t.Rate5(), + t.Rate15(), + t.RateMean(), + )) + } + }) + } +} diff --git a/common/metrics/timer.go b/common/metrics/timer.go new file mode 100644 index 00000000000..894bdfc3273 --- /dev/null +++ b/common/metrics/timer.go @@ -0,0 +1,149 @@ +package metrics + +import ( + "sync" + "time" +) + +// GetOrRegisterTimer returns an existing Timer or constructs and registers a +// new Timer. +// Be sure to unregister the meter from the registry once it is of no use to +// allow for garbage collection. +func GetOrRegisterTimer(name string, r Registry) *Timer { + return getOrRegister(name, NewTimer, r) +} + +// NewCustomTimer constructs a new Timer from a Histogram and a Meter. +// Be sure to call Stop() once the timer is of no use to allow for garbage collection. +func NewCustomTimer(h Histogram, m *Meter) *Timer { + return &Timer{ + histogram: h, + meter: m, + } +} + +// NewRegisteredTimer constructs and registers a new Timer. +// Be sure to unregister the meter from the registry once it is of no use to +// allow for garbage collection. +func NewRegisteredTimer(name string, r Registry) *Timer { + c := NewTimer() + if nil == r { + r = DefaultRegistry + } + r.Register(name, c) + return c +} + +// NewTimer constructs a new Timer using an exponentially-decaying +// sample with the same reservoir size and alpha as UNIX load averages. +// Be sure to call Stop() once the timer is of no use to allow for garbage collection. +func NewTimer() *Timer { + return &Timer{ + histogram: NewHistogram(NewExpDecaySample(1028, 0.015)), + meter: NewMeter(), + } +} + +// Timer captures the duration and rate of events, using a Histogram and a Meter. +type Timer struct { + histogram Histogram + meter *Meter + mutex sync.Mutex +} + +// Snapshot returns a read-only copy of the timer. +func (t *Timer) Snapshot() *TimerSnapshot { + t.mutex.Lock() + defer t.mutex.Unlock() + return &TimerSnapshot{ + histogram: t.histogram.Snapshot(), + meter: t.meter.Snapshot(), + } +} + +// Stop stops the meter. +func (t *Timer) Stop() { + t.meter.Stop() +} + +// Time record the duration of the execution of the given function. +func (t *Timer) Time(f func()) { + ts := time.Now() + f() + t.Update(time.Since(ts)) +} + +// Update the duration of an event, in nanoseconds. +func (t *Timer) Update(d time.Duration) { + t.mutex.Lock() + defer t.mutex.Unlock() + t.histogram.Update(d.Nanoseconds()) + t.meter.Mark(1) +} + +// UpdateSince update the duration of an event that started at a time and ends now. +// The record uses nanoseconds. +func (t *Timer) UpdateSince(ts time.Time) { + t.Update(time.Since(ts)) +} + +// TimerSnapshot is a read-only copy of another Timer. +type TimerSnapshot struct { + histogram HistogramSnapshot + meter *MeterSnapshot +} + +// Count returns the number of events recorded at the time the snapshot was +// taken. +func (t *TimerSnapshot) Count() int64 { return t.histogram.Count() } + +// Max returns the maximum value at the time the snapshot was taken. +func (t *TimerSnapshot) Max() int64 { return t.histogram.Max() } + +// Size returns the size of the sample at the time the snapshot was taken. +func (t *TimerSnapshot) Size() int { return t.histogram.Size() } + +// Mean returns the mean value at the time the snapshot was taken. +func (t *TimerSnapshot) Mean() float64 { return t.histogram.Mean() } + +// Min returns the minimum value at the time the snapshot was taken. +func (t *TimerSnapshot) Min() int64 { return t.histogram.Min() } + +// Percentile returns an arbitrary percentile of sampled values at the time the +// snapshot was taken. +func (t *TimerSnapshot) Percentile(p float64) float64 { + return t.histogram.Percentile(p) +} + +// Percentiles returns a slice of arbitrary percentiles of sampled values at +// the time the snapshot was taken. +func (t *TimerSnapshot) Percentiles(ps []float64) []float64 { + return t.histogram.Percentiles(ps) +} + +// Rate1 returns the one-minute moving average rate of events per second at the +// time the snapshot was taken. +func (t *TimerSnapshot) Rate1() float64 { return t.meter.Rate1() } + +// Rate5 returns the five-minute moving average rate of events per second at +// the time the snapshot was taken. +func (t *TimerSnapshot) Rate5() float64 { return t.meter.Rate5() } + +// Rate15 returns the fifteen-minute moving average rate of events per second +// at the time the snapshot was taken. +func (t *TimerSnapshot) Rate15() float64 { return t.meter.Rate15() } + +// RateMean returns the meter's mean rate of events per second at the time the +// snapshot was taken. +func (t *TimerSnapshot) RateMean() float64 { return t.meter.RateMean() } + +// StdDev returns the standard deviation of the values at the time the snapshot +// was taken. +func (t *TimerSnapshot) StdDev() float64 { return t.histogram.StdDev() } + +// Sum returns the sum at the time the snapshot was taken. +func (t *TimerSnapshot) Sum() int64 { return t.histogram.Sum() } + +// Variance returns the variance of the values at the time the snapshot was +// taken. +func (t *TimerSnapshot) Variance() float64 { return t.histogram.Variance() } diff --git a/common/metrics/timer_test.go b/common/metrics/timer_test.go new file mode 100644 index 00000000000..f10de16c9c2 --- /dev/null +++ b/common/metrics/timer_test.go @@ -0,0 +1,114 @@ +package metrics + +import ( + "fmt" + "math" + "testing" + "time" +) + +func BenchmarkTimer(b *testing.B) { + tm := NewTimer() + b.ResetTimer() + for i := 0; i < b.N; i++ { + tm.Update(1) + } +} + +func TestGetOrRegisterTimer(t *testing.T) { + r := NewRegistry() + NewRegisteredTimer("foo", r).Update(47) + if tm := GetOrRegisterTimer("foo", r).Snapshot(); tm.Count() != 1 { + t.Fatal(tm) + } +} + +func TestTimerExtremes(t *testing.T) { + tm := NewTimer() + tm.Update(math.MaxInt64) + tm.Update(0) + if stdDev := tm.Snapshot().StdDev(); stdDev != 4.611686018427388e+18 { + t.Errorf("tm.StdDev(): 4.611686018427388e+18 != %v\n", stdDev) + } +} + +func TestTimerStop(t *testing.T) { + l := len(arbiter.meters) + tm := NewTimer() + if l+1 != len(arbiter.meters) { + t.Errorf("arbiter.meters: %d != %d\n", l+1, len(arbiter.meters)) + } + tm.Stop() + if l != len(arbiter.meters) { + t.Errorf("arbiter.meters: %d != %d\n", l, len(arbiter.meters)) + } +} + +func TestTimerFunc(t *testing.T) { + var ( + tm = NewTimer() + testStart = time.Now() + actualTime time.Duration + ) + tm.Time(func() { + time.Sleep(50 * time.Millisecond) + actualTime = time.Since(testStart) + }) + var ( + drift = time.Millisecond * 2 + measured = time.Duration(tm.Snapshot().Max()) + ceil = actualTime + drift + floor = actualTime - drift + ) + if measured > ceil || measured < floor { + t.Errorf("tm.Max(): %v > %v || %v > %v\n", measured, ceil, measured, floor) + } +} + +func TestTimerZero(t *testing.T) { + tm := NewTimer().Snapshot() + if count := tm.Count(); count != 0 { + t.Errorf("tm.Count(): 0 != %v\n", count) + } + if min := tm.Min(); min != 0 { + t.Errorf("tm.Min(): 0 != %v\n", min) + } + if max := tm.Max(); max != 0 { + t.Errorf("tm.Max(): 0 != %v\n", max) + } + if mean := tm.Mean(); mean != 0.0 { + t.Errorf("tm.Mean(): 0.0 != %v\n", mean) + } + if stdDev := tm.StdDev(); stdDev != 0.0 { + t.Errorf("tm.StdDev(): 0.0 != %v\n", stdDev) + } + ps := tm.Percentiles([]float64{0.5, 0.75, 0.99}) + if ps[0] != 0.0 { + t.Errorf("median: 0.0 != %v\n", ps[0]) + } + if ps[1] != 0.0 { + t.Errorf("75th percentile: 0.0 != %v\n", ps[1]) + } + if ps[2] != 0.0 { + t.Errorf("99th percentile: 0.0 != %v\n", ps[2]) + } + if rate1 := tm.Rate1(); rate1 != 0.0 { + t.Errorf("tm.Rate1(): 0.0 != %v\n", rate1) + } + if rate5 := tm.Rate5(); rate5 != 0.0 { + t.Errorf("tm.Rate5(): 0.0 != %v\n", rate5) + } + if rate15 := tm.Rate15(); rate15 != 0.0 { + t.Errorf("tm.Rate15(): 0.0 != %v\n", rate15) + } + if rateMean := tm.RateMean(); rateMean != 0.0 { + t.Errorf("tm.RateMean(): 0.0 != %v\n", rateMean) + } +} + +func ExampleGetOrRegisterTimer() { + m := "account.create.latency" + t := GetOrRegisterTimer(m, nil) + t.Update(47) + fmt.Println(t.Snapshot().Max()) // Output: 47 +} diff --git a/common/metrics/validate.sh b/common/metrics/validate.sh new file mode 100755 index 00000000000..c4ae91e642d --- /dev/null +++ b/common/metrics/validate.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set -e + +# check there are no formatting issues +GOFMT_LINES=`gofmt -l . | wc -l | xargs` +test $GOFMT_LINES -eq 0 || echo "gofmt needs to be run, ${GOFMT_LINES} files have issues" + +# run the tests for the root package +go test -race . diff --git a/common/metrics/writer.go b/common/metrics/writer.go new file mode 100644 index 00000000000..2a41f8e1fe3 --- /dev/null +++ b/common/metrics/writer.go @@ -0,0 +1,99 @@ +package metrics + +import ( + "fmt" + "io" + "slices" + "strings" + "time" +) + +// Write sorts writes each metric in the given registry periodically to the +// given io.Writer. +func Write(r Registry, d time.Duration, w io.Writer) { + for range time.Tick(d) { + WriteOnce(r, w) + } +} + +// WriteOnce sorts and writes metrics in the given registry to the given +// io.Writer. +func WriteOnce(r Registry, w io.Writer) { + var namedMetrics []namedMetric + r.Each(func(name string, i interface{}) { + namedMetrics = append(namedMetrics, namedMetric{name, i}) + }) + slices.SortFunc(namedMetrics, namedMetric.cmp) + for _, namedMetric := range namedMetrics { + switch metric := namedMetric.m.(type) { + case *Counter: + fmt.Fprintf(w, "counter %s\n", namedMetric.name) + fmt.Fprintf(w, " count: %9d\n", metric.Snapshot().Count()) + case *CounterFloat64: + fmt.Fprintf(w, "counter %s\n", namedMetric.name) + fmt.Fprintf(w, " count: %f\n", metric.Snapshot().Count()) + case *Gauge: + fmt.Fprintf(w, "gauge %s\n", namedMetric.name) + fmt.Fprintf(w, " value: %9d\n", metric.Snapshot().Value()) + case *GaugeFloat64: + fmt.Fprintf(w, "gauge %s\n", namedMetric.name) + fmt.Fprintf(w, " value: %f\n", metric.Snapshot().Value()) + case *GaugeInfo: + fmt.Fprintf(w, "gauge %s\n", namedMetric.name) + fmt.Fprintf(w, " value: %s\n", metric.Snapshot().Value().String()) + case *Healthcheck: + metric.Check() + fmt.Fprintf(w, "healthcheck %s\n", namedMetric.name) + fmt.Fprintf(w, " error: %v\n", metric.Error()) + case Histogram: + h := metric.Snapshot() + ps := h.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999}) + fmt.Fprintf(w, "histogram %s\n", namedMetric.name) + fmt.Fprintf(w, " count: %9d\n", h.Count()) + fmt.Fprintf(w, " min: %9d\n", h.Min()) + fmt.Fprintf(w, " max: %9d\n", h.Max()) + fmt.Fprintf(w, " mean: %12.2f\n", h.Mean()) + fmt.Fprintf(w, " stddev: %12.2f\n", h.StdDev()) + fmt.Fprintf(w, " median: %12.2f\n", ps[0]) + fmt.Fprintf(w, " 75%%: %12.2f\n", ps[1]) + fmt.Fprintf(w, " 95%%: %12.2f\n", ps[2]) + fmt.Fprintf(w, " 99%%: %12.2f\n", ps[3]) + fmt.Fprintf(w, " 99.9%%: %12.2f\n", ps[4]) + case *Meter: + m := metric.Snapshot() + fmt.Fprintf(w, "meter %s\n", namedMetric.name) + fmt.Fprintf(w, " count: %9d\n", m.Count()) + fmt.Fprintf(w, " 1-min rate: %12.2f\n", m.Rate1()) + fmt.Fprintf(w, " 5-min rate: %12.2f\n", m.Rate5()) + fmt.Fprintf(w, " 15-min rate: %12.2f\n", m.Rate15()) + fmt.Fprintf(w, " mean rate: %12.2f\n", m.RateMean()) + case *Timer: + t := metric.Snapshot() + ps := t.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999}) + fmt.Fprintf(w, "timer %s\n", namedMetric.name) + fmt.Fprintf(w, " count: %9d\n", t.Count()) + fmt.Fprintf(w, " min: %9d\n", t.Min()) + fmt.Fprintf(w, " max: %9d\n", t.Max()) + fmt.Fprintf(w, " mean: %12.2f\n", t.Mean()) + fmt.Fprintf(w, " stddev: %12.2f\n", t.StdDev()) + fmt.Fprintf(w, " median: %12.2f\n", ps[0]) + fmt.Fprintf(w, " 75%%: %12.2f\n", ps[1]) + fmt.Fprintf(w, " 95%%: %12.2f\n", ps[2]) + fmt.Fprintf(w, " 99%%: %12.2f\n", ps[3]) + fmt.Fprintf(w, " 99.9%%: %12.2f\n", ps[4]) + fmt.Fprintf(w, " 1-min rate: %12.2f\n", t.Rate1()) + fmt.Fprintf(w, " 5-min rate: %12.2f\n", t.Rate5()) + fmt.Fprintf(w, " 15-min rate: %12.2f\n", t.Rate15()) + fmt.Fprintf(w, " mean rate: %12.2f\n", t.RateMean()) + } + } +} + +type namedMetric struct { + name string + m interface{} +} + +func (m namedMetric) cmp(other namedMetric) int { + return strings.Compare(m.name, other.name) +} diff --git a/common/metrics/writer_test.go b/common/metrics/writer_test.go new file mode 100644 index 00000000000..edcfe955abc --- /dev/null +++ b/common/metrics/writer_test.go @@ -0,0 +1,22 @@ +package metrics + +import ( + "slices" + "testing" +) + +func TestMetricsSorting(t *testing.T) { + var namedMetrics = []namedMetric{ + {name: "zzz"}, + {name: "bbb"}, + {name: "fff"}, + {name: "ggg"}, + } + + slices.SortFunc(namedMetrics, namedMetric.cmp) + for i, name := range []string{"bbb", "fff", "ggg", "zzz"} { + if namedMetrics[i].name != name { + t.Fail() + } + } +} diff --git a/execution/rlp/decode.go b/execution/rlp/decode.go index d28990a5211..df81db180a9 100644 --- a/execution/rlp/decode.go +++ b/execution/rlp/decode.go @@ -31,8 +31,7 @@ import ( "strings" "sync" - "github.com/erigontech/erigon/common/log/v3" - + "github.com/erigontech/erigon/execution/rlp/internal/rlpstruct" "github.com/holiman/uint256" ) @@ -43,15 +42,13 @@ import ( var EOL = errors.New("rlp: end of list") var ( - ErrExpectedString = errors.New("rlp: expected String or Byte") - ErrExpectedList = errors.New("rlp: expected List") - ErrCanonInt = errors.New("rlp: non-canonical integer format") - ErrCanonSize = errors.New("rlp: non-canonical size information") - ErrElemTooLarge = errors.New("rlp: element is larger than containing list") - ErrValueTooLarge = errors.New("rlp: value size exceeds available input length") - ErrMoreThanOneValue = errors.New("rlp: input contains more than one value") - ErrWrongTxTypePrefix = errors.New("rlp: only 1-byte tx type prefix is supported") - ErrUnknownTxTypePrefix = errors.New("rlp: unknown tx type prefix") + ErrExpectedString = errors.New("rlp: expected String or Byte") + ErrExpectedList = errors.New("rlp: expected List") + ErrCanonInt = errors.New("rlp: non-canonical integer format") + ErrCanonSize = errors.New("rlp: non-canonical size information") + ErrElemTooLarge = errors.New("rlp: element is larger than containing list") + ErrValueTooLarge = errors.New("rlp: value size exceeds available input length") + ErrMoreThanOneValue = errors.New("rlp: input contains more than one value") // internal errors errNotInList = errors.New("rlp: call of ListEnd outside of any list") @@ -59,34 +56,13 @@ var ( errUintOverflow = errors.New("rlp: uint overflow") errNoPointer = errors.New("rlp: interface given to Decode must be a pointer") errDecodeIntoNil = errors.New("rlp: pointer given to Decode must not be nil") + errUint256Large = errors.New("rlp: value too large for uint256") streamPool = sync.Pool{ - New: func() any { return new(Stream) }, + New: func() interface{} { return new(Stream) }, } ) -func IsInvalidRLPError(err error) bool { - return errors.Is(err, ErrExpectedString) || - errors.Is(err, ErrExpectedList) || - errors.Is(err, ErrCanonInt) || - errors.Is(err, ErrCanonSize) || - errors.Is(err, ErrElemTooLarge) || - errors.Is(err, ErrValueTooLarge) || - errors.Is(err, ErrMoreThanOneValue) || - errors.Is(err, ErrWrongTxTypePrefix) || - errors.Is(err, ErrUnknownTxTypePrefix) || - // internal errors - errors.Is(err, errNotInList) || - errors.Is(err, errNotAtEOL) || - errors.Is(err, errUintOverflow) || - // stream errors - strings.Contains(err.Error(), "rlp: input list has too many elements") || - strings.Contains(err.Error(), "rlp: expected input string or byte") || - strings.Contains(err.Error(), "rlp: expected input list") || - strings.Contains(err.Error(), "rlp: non-canonical size information") || - strings.Contains(err.Error(), "rlp: non-canonical integer (leading zero bytes)") -} - // Decoder is implemented by types that require custom RLP decoding rules or need to decode // into private fields. // @@ -106,11 +82,8 @@ type Decoder interface { // panics cause by huge value sizes. If you need an input limit, use // // NewStream(r, limit).Decode(val) -func Decode(r io.Reader, val any) error { - stream, ok := streamPool.Get().(*Stream) - if !ok { - log.Warn("Failed to type convert to Stream pointer") - } +func Decode(r io.Reader, val interface{}) error { + stream := streamPool.Get().(*Stream) defer streamPool.Put(stream) stream.Reset(r, 0) @@ -119,13 +92,10 @@ func Decode(r io.Reader, val any) error { // DecodeBytes parses RLP data from b into val. Please see package-level documentation for // the decoding rules. The input must contain exactly one value and no trailing data. -func DecodeBytes(b []byte, val any) error { +func DecodeBytes(b []byte, val interface{}) error { r := (*sliceReader)(&b) - stream, ok := streamPool.Get().(*Stream) - if !ok { - log.Warn("Failed to type convert to Stream pointer") - } + stream := streamPool.Get().(*Stream) defer streamPool.Put(stream) stream.Reset(r, uint64(len(b))) @@ -138,22 +108,6 @@ func DecodeBytes(b []byte, val any) error { return nil } -func DecodeBytesPartial(b []byte, val any) error { - r := (*sliceReader)(&b) - - stream, ok := streamPool.Get().(*Stream) - if !ok { - log.Warn("Failed to type convert to Stream pointer") - } - defer streamPool.Put(stream) - - stream.Reset(r, uint64(len(b))) - if err := stream.Decode(val); err != nil { - return err - } - return nil -} - type decodeError struct { msg string typ reflect.Type @@ -172,30 +126,25 @@ func (err *decodeError) Error() string { } func wrapStreamError(err error, typ reflect.Type) error { - switch { - case errors.Is(err, ErrCanonInt): + switch err { + case ErrCanonInt: return &decodeError{msg: "non-canonical integer (leading zero bytes)", typ: typ} - case errors.Is(err, ErrCanonSize): + case ErrCanonSize: return &decodeError{msg: "non-canonical size information", typ: typ} - case errors.Is(err, ErrExpectedList): + case ErrExpectedList: return &decodeError{msg: "expected input list", typ: typ} - case errors.Is(err, ErrExpectedString): + case ErrExpectedString: return &decodeError{msg: "expected input string or byte", typ: typ} - case errors.Is(err, errUintOverflow): + case errUintOverflow: return &decodeError{msg: "input string too long", typ: typ} - case errors.Is(err, errNotAtEOL): + case errNotAtEOL: return &decodeError{msg: "input list has too many elements", typ: typ} } return err } -func WrapStreamError(err error, typ reflect.Type) error { - return wrapStreamError(err, typ) -} - func addErrorContext(err error, ctx string) error { - var decErr *decodeError - if errors.As(err, &decErr) { + if decErr, ok := err.(*decodeError); ok { decErr.ctx = append(decErr.ctx, ctx) } return err @@ -204,30 +153,28 @@ func addErrorContext(err error, ctx string) error { var ( decoderInterface = reflect.TypeFor[Decoder]() bigInt = reflect.TypeFor[big.Int]() - uint256Int = reflect.TypeFor[uint256.Int]() + u256Int = reflect.TypeFor[uint256.Int]() ) -func makeDecoder(typ reflect.Type, tags tags) (dec decoder, err error) { +func makeDecoder(typ reflect.Type, tags rlpstruct.Tags) (dec decoder, err error) { kind := typ.Kind() switch { case typ == rawValueType: return decodeRawValue, nil - case typ.AssignableTo(reflect.PtrTo(bigInt)): + case typ.AssignableTo(reflect.PointerTo(bigInt)): return decodeBigInt, nil case typ.AssignableTo(bigInt): return decodeBigIntNoPtr, nil - case typ.AssignableTo(reflect.PtrTo(uint256Int)): - return decodeUint256, nil - case typ.AssignableTo(uint256Int): - return decodeUint256NoPtr, nil + case typ == reflect.PointerTo(u256Int): + return decodeU256, nil + case typ == u256Int: + return decodeU256NoPtr, nil case kind == reflect.Ptr: return makePtrDecoder(typ, tags) - case reflect.PtrTo(typ).Implements(decoderInterface): + case reflect.PointerTo(typ).Implements(decoderInterface): return decodeDecoder, nil case isUint(kind): return decodeUint, nil - case isInt(kind): - return decodeInt, nil case kind == reflect.Bool: return decodeBool, nil case kind == reflect.String: @@ -262,16 +209,6 @@ func decodeUint(s *Stream, val reflect.Value) error { return nil } -func decodeInt(s *Stream, val reflect.Value) error { - typ := val.Type() - num, err := s.uint(typ.Bits()) - if err != nil { - return wrapStreamError(err, val.Type()) - } - val.SetInt(int64(num)) - return nil -} - func decodeBool(s *Stream, val reflect.Value) error { b, err := s.Bool() if err != nil { @@ -295,50 +232,46 @@ func decodeBigIntNoPtr(s *Stream, val reflect.Value) error { } func decodeBigInt(s *Stream, val reflect.Value) error { - b, err := s.bigIntBytes() - if err != nil { - return wrapStreamError(err, val.Type()) - } - - // Set the integer bytes. i := val.Interface().(*big.Int) if i == nil { i = new(big.Int) val.Set(reflect.ValueOf(i)) } - i.SetBytes(b) - return nil -} - -func decodeUint256NoPtr(s *Stream, val reflect.Value) error { - return decodeUint256(s, val.Addr()) -} -func decodeUint256(s *Stream, val reflect.Value) error { - b, err := s.Uint256Bytes() + err := s.decodeBigInt(i) if err != nil { return wrapStreamError(err, val.Type()) } + return nil +} - // Set the integer bytes. +func decodeU256NoPtr(s *Stream, val reflect.Value) error { + return decodeU256(s, val.Addr()) +} + +func decodeU256(s *Stream, val reflect.Value) error { i := val.Interface().(*uint256.Int) if i == nil { i = new(uint256.Int) val.Set(reflect.ValueOf(i)) } - i.SetBytes(b) + + err := s.ReadUint256(i) + if err != nil { + return wrapStreamError(err, val.Type()) + } return nil } -func makeListDecoder(typ reflect.Type, tag tags) (decoder, error) { +func makeListDecoder(typ reflect.Type, tag rlpstruct.Tags) (decoder, error) { etype := typ.Elem() - if etype.Kind() == reflect.Uint8 && !reflect.PtrTo(etype).Implements(decoderInterface) { + if etype.Kind() == reflect.Uint8 && !reflect.PointerTo(etype).Implements(decoderInterface) { if typ.Kind() == reflect.Array { return decodeByteArray, nil } return decodeByteSlice, nil } - etypeinfo := theTC.infoWhileGenerating(etype, tags{}) + etypeinfo := theTC.infoWhileGenerating(etype, rlpstruct.Tags{}) if etypeinfo.decoderErr != nil { return nil, etypeinfo.decoderErr } @@ -348,7 +281,7 @@ func makeListDecoder(typ reflect.Type, tag tags) (decoder, error) { dec = func(s *Stream, val reflect.Value) error { return decodeListArray(s, val, etypeinfo.decoder) } - case tag.tail: + case tag.Tail: // A slice with "tail" tag can occur as the last field // of a struct and is supposed to swallow all remaining // list elements. The struct decoder already called s.List, @@ -384,7 +317,10 @@ func decodeSliceElems(s *Stream, val reflect.Value, elemdec decoder) error { for ; ; i++ { // grow slice if necessary if i >= val.Cap() { - newcap := max(val.Cap()+val.Cap()/2, 4) + newcap := val.Cap() + val.Cap()/2 + if newcap < 4 { + newcap = 4 + } newv := reflect.MakeSlice(val.Type(), val.Len(), newcap) reflect.Copy(newv, val) val.Set(newv) @@ -393,7 +329,7 @@ func decodeSliceElems(s *Stream, val reflect.Value, elemdec decoder) error { val.SetLen(i + 1) } // decode into element - if err := elemdec(s, val.Index(i)); errors.Is(err, EOL) { + if err := elemdec(s, val.Index(i)); err == EOL { break } else if err != nil { return addErrorContext(err, fmt.Sprint("[", i, "]")) @@ -412,7 +348,7 @@ func decodeListArray(s *Stream, val reflect.Value, elemdec decoder) error { vlen := val.Len() i := 0 for ; i < vlen; i++ { - if err := elemdec(s, val.Index(i)); errors.Is(err, EOL) { + if err := elemdec(s, val.Index(i)); err == EOL { break } else if err != nil { return addErrorContext(err, fmt.Sprint("[", i, "]")) @@ -438,25 +374,23 @@ func decodeByteArray(s *Stream, val reflect.Value) error { if err != nil { return err } - vlen := val.Len() + slice := val.Bytes() switch kind { case Byte: - if vlen == 0 { + if len(slice) == 0 { return &decodeError{msg: "input string too long", typ: val.Type()} - } - if vlen > 1 { + } else if len(slice) > 1 { return &decodeError{msg: "input string too short", typ: val.Type()} } - bv, _ := s.Uint() - val.Index(0).SetUint(bv) + slice[0] = s.byteval + s.kind = -1 case String: - if uint64(vlen) < size { + if uint64(len(slice)) < size { return &decodeError{msg: "input string too long", typ: val.Type()} } - if uint64(vlen) > size { + if uint64(len(slice)) > size { return &decodeError{msg: "input string too short", typ: val.Type()} } - slice := val.Slice(0, vlen).Interface().([]byte) if err := s.readFull(slice); err != nil { return err } @@ -486,7 +420,7 @@ func makeStructDecoder(typ reflect.Type) (decoder, error) { } for i, f := range fields { err := f.info.decoder(s, val.Field(f.index)) - if errors.Is(err, EOL) { + if err == EOL { if f.optional { // The field is optional, so reaching the end of the list before // reaching the last field is acceptable. All remaining undecoded @@ -512,16 +446,16 @@ func zeroFields(structval reflect.Value, fields []field) { } // makePtrDecoder creates a decoder that decodes into the pointer's element type. -func makePtrDecoder(typ reflect.Type, tag tags) (decoder, error) { +func makePtrDecoder(typ reflect.Type, tag rlpstruct.Tags) (decoder, error) { etype := typ.Elem() - etypeinfo := theTC.infoWhileGenerating(etype, tags{}) + etypeinfo := theTC.infoWhileGenerating(etype, rlpstruct.Tags{}) switch { case etypeinfo.decoderErr != nil: return nil, etypeinfo.decoderErr - case !tag.nilOK: + case !tag.NilOK: return makeSimplePtrDecoder(etype, etypeinfo), nil default: - return makeNilPtrDecoder(etype, etypeinfo, tag.nilKind), nil + return makeNilPtrDecoder(etype, etypeinfo, tag), nil } } @@ -542,9 +476,13 @@ func makeSimplePtrDecoder(etype reflect.Type, etypeinfo *typeinfo) decoder { // values are decoded into a value of the element type, just like makePtrDecoder does. // // This decoder is used for pointer-typed struct fields with struct tag "nil". -func makeNilPtrDecoder(etype reflect.Type, etypeinfo *typeinfo, nilKind Kind) decoder { - typ := reflect.PtrTo(etype) +func makeNilPtrDecoder(etype reflect.Type, etypeinfo *typeinfo, ts rlpstruct.Tags) decoder { + typ := reflect.PointerTo(etype) nilPtr := reflect.Zero(typ) + + // Determine the value kind that results in nil pointer. + nilKind := typeNilKind(etype, ts) + return func(s *Stream, val reflect.Value) (err error) { kind, size, err := s.Kind() if err != nil { @@ -608,7 +546,7 @@ func decodeDecoder(s *Stream, val reflect.Value) error { } // Kind represents the kind of value contained in an RLP stream. -type Kind int +type Kind int8 const ( Byte Kind = iota @@ -651,22 +589,16 @@ type ByteReader interface { type Stream struct { r ByteReader - // number of bytes remaining to be read from r. - remaining uint64 - limited bool - - // auxiliary buffer for integer decoding - uintbuf [32]byte - - kind Kind // kind of value ahead - size uint64 // size of value ahead - byteval byte // value of single byte in type tag - kinderr error // error from last readKind - stack []listpos + remaining uint64 // number of bytes remaining to be read from r + size uint64 // size of value ahead + kinderr error // error from last readKind + stack []uint64 // list sizes + uintbuf [32]byte // auxiliary buffer for integer decoding + kind Kind // kind of value ahead + byteval byte // value of single byte in type tag + limited bool // true if input limit is in effect } -type listpos struct{ pos, size uint64 } - // NewStream creates a new decoding stream reading from r. // // If r implements the ByteReader interface, Stream will @@ -700,22 +632,6 @@ func NewListStream(r io.Reader, len uint64) *Stream { return s } -func NewStreamFromPool(r io.Reader, inputLimit uint64) (stream *Stream, done func()) { - stream, ok := streamPool.Get().(*Stream) - if !ok { - log.Warn("Failed to get stream from pool") - } - stream.Reset(r, inputLimit) - return stream, func() { - streamPool.Put(stream) - } -} - -// Remaining returns number of bytes remaining to be read -func (s *Stream) Remaining() uint64 { - return s.remaining -} - // Bytes reads an RLP string and returns its contents as a byte slice. // If the input does not contain an RLP string, the returned // error will be ErrExpectedString. @@ -742,6 +658,8 @@ func (s *Stream) Bytes() ([]byte, error) { } } +// ReadBytes decodes the next RLP value and stores the result in b. +// The value size must match len(b) exactly. func (s *Stream) ReadBytes(b []byte) error { kind, size, err := s.Kind() if err != nil { @@ -781,8 +699,8 @@ func (s *Stream) Raw() ([]byte, error) { s.kind = -1 // rearm Kind return []byte{s.byteval}, nil } - // the original header has already been read and is no longer - // available. read content and put a new header in front of it. + // The original header has already been read and is no longer + // available. Read content and put a new header in front of it. start := headsize(size) buf := make([]byte, uint64(start)+size) if err := s.readFull(buf[start:]); err != nil { @@ -799,10 +717,31 @@ func (s *Stream) Raw() ([]byte, error) { // Uint reads an RLP string of up to 8 bytes and returns its contents // as an unsigned integer. If the input does not contain an RLP string, the // returned error will be ErrExpectedString. +// +// Deprecated: use s.Uint64 instead. func (s *Stream) Uint() (uint64, error) { return s.uint(64) } +func (s *Stream) Uint64() (uint64, error) { + return s.uint(64) +} + +func (s *Stream) Uint32() (uint32, error) { + i, err := s.uint(32) + return uint32(i), err +} + +func (s *Stream) Uint16() (uint16, error) { + i, err := s.uint(16) + return uint16(i), err +} + +func (s *Stream) Uint8() (uint8, error) { + i, err := s.uint(8) + return uint8(i), err +} + func (s *Stream) uint(maxbits int) (uint64, error) { kind, size, err := s.Kind() if err != nil { @@ -821,7 +760,7 @@ func (s *Stream) uint(maxbits int) (uint64, error) { } v, err := s.readUint(byte(size)) switch { - case errors.Is(err, ErrCanonSize): + case err == ErrCanonSize: // Adjust error because we're not reading a size right now. return 0, ErrCanonInt case err != nil: @@ -836,29 +775,87 @@ func (s *Stream) uint(maxbits int) (uint64, error) { } } -// Deprecated: Uint256Bytes generates unecessary garbage by -// returning a heap buffer which is immediately converted -// into an array and disguared. Use Uint256() instead -// which processed the buffer internally so it doesnt leak -func (s *Stream) Uint256Bytes() ([]byte, error) { - b, err := s.bigIntBytes() +// Bool reads an RLP string of up to 1 byte and returns its contents +// as a boolean. If the input does not contain an RLP string, the +// returned error will be ErrExpectedString. +func (s *Stream) Bool() (bool, error) { + num, err := s.uint(8) if err != nil { - return nil, err + return false, err + } + switch num { + case 0: + return false, nil + case 1: + return true, nil + default: + return false, fmt.Errorf("rlp: invalid boolean value: %d", num) + } +} + +// List starts decoding an RLP list. If the input does not contain a +// list, the returned error will be ErrExpectedList. When the list's +// end has been reached, any Stream operation will return EOL. +func (s *Stream) List() (size uint64, err error) { + kind, size, err := s.Kind() + if err != nil { + return 0, err } - if len(b) > 32 { - return nil, errUintOverflow + if kind != List { + return 0, ErrExpectedList } - return b, nil + + // Remove size of inner list from outer list before pushing the new size + // onto the stack. This ensures that the remaining outer list size will + // be correct after the matching call to ListEnd. + if inList, limit := s.listLimit(); inList { + s.stack[len(s.stack)-1] = limit - size + } + s.stack = append(s.stack, size) + s.kind = -1 + s.size = 0 + return size, nil } -func (s *Stream) Uint256() (i uint256.Int, err error) { +// ListEnd returns to the enclosing list. +// The input reader must be positioned at the end of a list. +func (s *Stream) ListEnd() error { + // Ensure that no more data is remaining in the current list. + if inList, listLimit := s.listLimit(); !inList { + return errNotInList + } else if listLimit > 0 { + return errNotAtEOL + } + s.stack = s.stack[:len(s.stack)-1] // pop + s.kind = -1 + s.size = 0 + return nil +} + +// MoreDataInList reports whether the current list context contains +// more data to be read. +func (s *Stream) MoreDataInList() bool { + _, listLimit := s.listLimit() + return listLimit > 0 +} + +// BigInt decodes an arbitrary-size integer value. +func (s *Stream) BigInt() (*big.Int, error) { + i := new(big.Int) + if err := s.decodeBigInt(i); err != nil { + return nil, err + } + return i, nil +} + +func (s *Stream) decodeBigInt(dst *big.Int) error { var buffer []byte kind, size, err := s.Kind() switch { case err != nil: - return i, err + return err case kind == List: - return i, ErrExpectedString + return ErrExpectedString case kind == Byte: buffer = s.uintbuf[:1] buffer[0] = s.byteval @@ -866,45 +863,43 @@ func (s *Stream) Uint256() (i uint256.Int, err error) { case size == 0: // Avoid zero-length read. s.kind = -1 - case size > 32: - return i, ErrElemTooLarge case size <= uint64(len(s.uintbuf)): // For integers smaller than s.uintbuf, allocating a buffer // can be avoided. buffer = s.uintbuf[:size] if err := s.readFull(buffer); err != nil { - return i, err + return err } // Reject inputs where single byte encoding should have been used. if size == 1 && buffer[0] < 128 { - return i, ErrCanonSize + return ErrCanonSize } default: // For large integers, a temporary buffer is needed. buffer = make([]byte, size) if err := s.readFull(buffer); err != nil { - return i, err + return err } } // Reject leading zero bytes. if len(buffer) > 0 && buffer[0] == 0 { - return i, ErrCanonInt + return ErrCanonInt } - - i.SetBytes(buffer) - - return i, nil + // Set the integer bytes. + dst.SetBytes(buffer) + return nil } -func (s *Stream) bigIntBytes() ([]byte, error) { +// ReadUint256 decodes the next value as a uint256. +func (s *Stream) ReadUint256(dst *uint256.Int) error { var buffer []byte kind, size, err := s.Kind() switch { case err != nil: - return nil, err + return err case kind == List: - return nil, ErrExpectedString + return ErrExpectedString case kind == Byte: buffer = s.uintbuf[:1] buffer[0] = s.byteval @@ -913,95 +908,32 @@ func (s *Stream) bigIntBytes() ([]byte, error) { // Avoid zero-length read. s.kind = -1 case size <= uint64(len(s.uintbuf)): - // For integers smaller than s.uintbuf, allocating a buffer - // can be avoided. + // All possible uint256 values fit into s.uintbuf. buffer = s.uintbuf[:size] if err := s.readFull(buffer); err != nil { - return nil, err + return err } // Reject inputs where single byte encoding should have been used. if size == 1 && buffer[0] < 128 { - return nil, ErrCanonSize + return ErrCanonSize } default: - // For large integers, a temporary buffer is needed. - buffer = make([]byte, size) - if err := s.readFull(buffer); err != nil { - return nil, err - } + return errUint256Large } // Reject leading zero bytes. if len(buffer) > 0 && buffer[0] == 0 { - return nil, ErrCanonInt - } - - return buffer, nil -} - -// Bool reads an RLP string of up to 1 byte and returns its contents -// as a boolean. If the input does not contain an RLP string, the -// returned error will be ErrExpectedString. -func (s *Stream) Bool() (bool, error) { - num, err := s.uint(8) - if err != nil { - return false, err - } - switch num { - case 0: - return false, nil - case 1: - return true, nil - default: - return false, fmt.Errorf("rlp: invalid boolean value: %d", num) - } -} - -// List starts decoding an RLP list. If the input does not contain a -// list, the returned error will be ErrExpectedList. When the list's -// end has been reached, any Stream operation will return EOL. -func (s *Stream) List() (size uint64, err error) { - kind, size, err := s.Kind() - if err != nil { - return 0, err - } - if kind != List { - return 0, ErrExpectedList - } - s.NewList(size) - return size, nil -} - -// NewList starts decoding an RLP list, but without reading its prefix -func (s *Stream) NewList(size uint64) { - s.stack = append(s.stack, listpos{0, size}) - s.kind = -1 - s.size = 0 -} - -// ListEnd returns to the enclosing list. -// The input reader must be positioned at the end of a list. -func (s *Stream) ListEnd() error { - if len(s.stack) == 0 { - return errNotInList - } - tos := s.stack[len(s.stack)-1] - if tos.pos != tos.size { - return errNotAtEOL + return ErrCanonInt } - s.stack = s.stack[:len(s.stack)-1] // pop - if len(s.stack) > 0 { - s.stack[len(s.stack)-1].pos += tos.size - } - s.kind = -1 - s.size = 0 + // Set the integer bytes. + dst.SetBytes(buffer) return nil } // Decode decodes a value and stores the result in the value pointed // to by val. Please see the documentation for the Decode function // to learn about the decoding rules. -func (s *Stream) Decode(val any) error { +func (s *Stream) Decode(val interface{}) error { if val == nil { return errDecodeIntoNil } @@ -1013,15 +945,14 @@ func (s *Stream) Decode(val any) error { if rval.IsNil() { return errDecodeIntoNil } - dcd, err := cachedDecoder(rtyp.Elem()) + decoder, err := cachedDecoder(rtyp.Elem()) if err != nil { return err } - err = dcd(s, rval.Elem()) - var decErr *decodeError - if errors.As(err, &decErr) && len(decErr.ctx) > 0 { - // add decode target type to error so context has more meaning + err = decoder(s, rval.Elem()) + if decErr, ok := err.(*decodeError); ok && len(decErr.ctx) > 0 { + // Add decode target type to error so context has more meaning. decErr.ctx = append(decErr.ctx, fmt.Sprint("(", rtyp.Elem(), ")")) } return err @@ -1044,6 +975,9 @@ func (s *Stream) Reset(r io.Reader, inputLimit uint64) { case *bytes.Reader: s.remaining = uint64(br.Len()) s.limited = true + case *bytes.Buffer: + s.remaining = uint64(br.Len()) + s.limited = true case *strings.Reader: s.remaining = uint64(br.Len()) s.limited = true @@ -1062,8 +996,8 @@ func (s *Stream) Reset(r io.Reader, inputLimit uint64) { s.size = 0 s.kind = -1 s.kinderr = nil - s.uintbuf = [32]byte{} s.byteval = 0 + s.uintbuf = [32]byte{} } // Kind returns the kind and size of the next value in the @@ -1078,35 +1012,29 @@ func (s *Stream) Reset(r io.Reader, inputLimit uint64) { // the value. Subsequent calls to Kind (until the value is decoded) // will not advance the input reader and return cached information. func (s *Stream) Kind() (kind Kind, size uint64, err error) { - var tos *listpos - if len(s.stack) > 0 { - tos = &s.stack[len(s.stack)-1] - } - if s.kind < 0 { - s.kinderr = nil - // Don't read further if we're at the end of the - // innermost list. - if tos != nil && tos.pos == tos.size { - return 0, 0, EOL - } - s.kind, s.size, s.kinderr = s.readKind() - if s.kinderr == nil { - if tos == nil { - // At toplevel, check that the value is smaller - // than the remaining input length. - if s.limited && s.size > s.remaining { - s.kinderr = ErrValueTooLarge - } - } else { - // Inside a list, check that the value doesn't overflow the list. - if s.size > tos.size-tos.pos { - s.kinderr = ErrElemTooLarge - } - } + if s.kind >= 0 { + return s.kind, s.size, s.kinderr + } + + // Check for end of list. This needs to be done here because readKind + // checks against the list size, and would return the wrong error. + inList, listLimit := s.listLimit() + if inList && listLimit == 0 { + return 0, 0, EOL + } + // Read the actual size tag. + s.kind, s.size, s.kinderr = s.readKind() + if s.kinderr == nil { + // Check the data size of the value ahead against input limits. This + // is done here because many decoders require allocating an input + // buffer matching the value size. Checking it here protects those + // decoders from inputs declaring very large value size. + if inList && s.size > listLimit { + s.kinderr = ErrElemTooLarge + } else if s.limited && s.size > s.remaining { + s.kinderr = ErrValueTooLarge } } - // Note: this might return a sticky error generated - // by an earlier call to readKind. return s.kind, s.size, s.kinderr } @@ -1116,10 +1044,10 @@ func (s *Stream) readKind() (kind Kind, size uint64, err error) { if len(s.stack) == 0 { // At toplevel, Adjust the error to actual EOF. io.EOF is // used by callers to determine when to stop decoding. - switch { - case errors.Is(err, io.ErrUnexpectedEOF): + switch err { + case io.ErrUnexpectedEOF: err = io.EOF - case errors.Is(err, ErrValueTooLarge): + case ErrValueTooLarge: err = io.EOF } } @@ -1127,44 +1055,42 @@ func (s *Stream) readKind() (kind Kind, size uint64, err error) { } s.byteval = 0 switch { - case b < 0x80: //128 + case b < 0x80: // For a single byte whose value is in the [0x00, 0x7F] range, that byte // is its own RLP encoding. s.byteval = b return Byte, 0, nil - case b < 0xB8: //184 - // Otherwise, if a string is 0-55 bytes long, - // the RLP encoding consists of a single byte with value 0x80 plus the - // length of the string followed by the string. The range of the first - // byte is thus [0x80, 0xB7]. + case b < 0xB8: + // Otherwise, if a string is 0-55 bytes long, the RLP encoding consists + // of a single byte with value 0x80 plus the length of the string + // followed by the string. The range of the first byte is thus [0x80, 0xB7]. return String, uint64(b - 0x80), nil - case b < 0xC0: //192 - // If a string is more than 55 bytes long, the - // RLP encoding consists of a single byte with value 0xB7 plus the length - // of the length of the string in binary form, followed by the length of - // the string, followed by the string. For example, a length-1024 string - // would be encoded as 0xB90400 followed by the string. The range of - // the first byte is thus [0xB8, 0xBF]. + case b < 0xC0: + // If a string is more than 55 bytes long, the RLP encoding consists of a + // single byte with value 0xB7 plus the length of the length of the + // string in binary form, followed by the length of the string, followed + // by the string. For example, a length-1024 string would be encoded as + // 0xB90400 followed by the string. The range of the first byte is thus + // [0xB8, 0xBF]. size, err = s.readUint(b - 0xB7) if err == nil && size < 56 { err = ErrCanonSize } return String, size, err - case b < 0xF8: //248 - // If the total payload of a list - // (i.e. the combined length of all its items) is 0-55 bytes long, the - // RLP encoding consists of a single byte with value 0xC0 plus the length - // of the list followed by the concatenation of the RLP encodings of the - // items. The range of the first byte is thus [0xC0, 0xF7]. + case b < 0xF8: + // If the total payload of a list (i.e. the combined length of all its + // items) is 0-55 bytes long, the RLP encoding consists of a single byte + // with value 0xC0 plus the length of the list followed by the + // concatenation of the RLP encodings of the items. The range of the + // first byte is thus [0xC0, 0xF7]. return List, uint64(b - 0xC0), nil default: - // If the total payload of a list is more than 55 bytes long, - // the RLP encoding consists of a single byte with value 0xF7 - // plus the length of the length of the payload in binary - // form, followed by the length of the payload, followed by - // the concatenation of the RLP encodings of the items. The - // range of the first byte is thus [0xF8, 0xFF]. - size, err = s.readUint(b - 0xF7) //247 + // If the total payload of a list is more than 55 bytes long, the RLP + // encoding consists of a single byte with value 0xF7 plus the length of + // the length of the payload in binary form, followed by the length of + // the payload, followed by the concatenation of the RLP encodings of + // the items. The range of the first byte is thus [0xF8, 0xFF]. + size, err = s.readUint(b - 0xF7) if err == nil && size < 56 { err = ErrCanonSize } @@ -1192,20 +1118,21 @@ func (s *Stream) readUint(size byte) (uint64, error) { // The error needs to be adjusted to become ErrCanonInt in this case. return 0, ErrCanonSize } - return binary.BigEndian.Uint64(buffer), nil + return binary.BigEndian.Uint64(buffer[:]), nil } } +// readFull reads into buf from the underlying stream. func (s *Stream) readFull(buf []byte) (err error) { - if e := s.willRead(uint64(len(buf))); e != nil { - return e + if err := s.willRead(uint64(len(buf))); err != nil { + return err } var nn, n int for n < len(buf) && err == nil { nn, err = s.r.Read(buf[n:]) n += nn } - if errors.Is(err, io.EOF) { + if err == io.EOF { if n < len(buf) { err = io.ErrUnexpectedEOF } else { @@ -1217,27 +1144,28 @@ func (s *Stream) readFull(buf []byte) (err error) { return err } +// readByte reads a single byte from the underlying stream. func (s *Stream) readByte() (byte, error) { if err := s.willRead(1); err != nil { return 0, err } b, err := s.r.ReadByte() - if errors.Is(err, io.EOF) { + if err == io.EOF { err = io.ErrUnexpectedEOF } return b, err } +// willRead is called before any read from the underlying stream. It checks +// n against size limits, and updates the limits if n doesn't overflow them. func (s *Stream) willRead(n uint64) error { s.kind = -1 // rearm Kind - if len(s.stack) > 0 { - // check list overflow - tos := s.stack[len(s.stack)-1] - if n > tos.size-tos.pos { + if inList, limit := s.listLimit(); inList { + if n > limit { return ErrElemTooLarge } - s.stack[len(s.stack)-1].pos += n + s.stack[len(s.stack)-1] = limit - n } if s.limited { if n > s.remaining { @@ -1248,20 +1176,12 @@ func (s *Stream) willRead(n uint64) error { return nil } -// MoreDataInList reports whether the current list context contains -// more data to be read. -func (s *Stream) MoreDataInList() bool { - _, listLimit := s.listLimit() - return listLimit > 0 -} - // listLimit returns the amount of data remaining in the innermost list. func (s *Stream) listLimit() (inList bool, limit uint64) { if len(s.stack) == 0 { return false, 0 } - a := s.stack[len(s.stack)-1] - return true, a.size - a.pos + return true, s.stack[len(s.stack)-1] } type sliceReader []byte diff --git a/execution/rlp/encbuffer.go b/execution/rlp/encbuffer.go index f58c04d1bb1..d3556fac214 100644 --- a/execution/rlp/encbuffer.go +++ b/execution/rlp/encbuffer.go @@ -20,143 +20,203 @@ package rlp import ( + "encoding/binary" "io" + "math/big" "reflect" "sync" + + "github.com/erigontech/erigon/common/math" + "github.com/holiman/uint256" ) type encBuffer struct { - str []byte // string data, contains everything except list headers - lheads []listhead // all list headers - lhsize int // sum of sizes of all encoded list headers - sizebuf [9]byte // auxiliary buffer for uint encoding - bufvalue reflect.Value // used in writeByteArrayCopy + str []byte // string data, contains everything except list headers + lheads []listhead // all list headers + lhsize int // sum of sizes of all encoded list headers + sizebuf [9]byte // auxiliary buffer for uint encoding } -// encbufs are pooled. +// The global encBuffer pool. var encBufferPool = sync.Pool{ - New: func() any { - var bytes []byte - return &encBuffer{bufvalue: reflect.ValueOf(&bytes).Elem()} - }, + New: func() interface{} { return new(encBuffer) }, } -func (w *encBuffer) reset() { - w.lhsize = 0 - w.str = w.str[:0] - w.lheads = w.lheads[:0] +func getEncBuffer() *encBuffer { + buf := encBufferPool.Get().(*encBuffer) + buf.reset() + return buf } -// encBuffer implements io.Writer so it can be passed it into EncodeRLP. -func (w *encBuffer) Write(b []byte) (int, error) { - w.str = append(w.str, b...) - return len(b), nil +func (buf *encBuffer) reset() { + buf.lhsize = 0 + buf.str = buf.str[:0] + buf.lheads = buf.lheads[:0] } -func (w *encBuffer) encode(val any) error { - rval := reflect.ValueOf(val) - writer, err := cachedWriter(rval.Type()) - if err != nil { - return err +// size returns the length of the encoded data. +func (buf *encBuffer) size() int { + return len(buf.str) + buf.lhsize +} + +// makeBytes creates the encoder output. +func (buf *encBuffer) makeBytes() []byte { + out := make([]byte, buf.size()) + buf.copyTo(out) + return out +} + +func (buf *encBuffer) copyTo(dst []byte) { + strpos := 0 + pos := 0 + for _, head := range buf.lheads { + // write string data before header + n := copy(dst[pos:], buf.str[strpos:head.offset]) + pos += n + strpos += n + // write the header + enc := head.encode(dst[pos:]) + pos += len(enc) } - return writer(rval, w) + // copy string data after the last list header + copy(dst[pos:], buf.str[strpos:]) } -func (w *encBuffer) encodeStringHeader(size int) { - if size < 56 { - w.str = append(w.str, EmptyStringCode+byte(size)) - } else { - sizesize := putint(w.sizebuf[1:], uint64(size)) - w.sizebuf[0] = 0xB7 + byte(sizesize) - w.str = append(w.str, w.sizebuf[:sizesize+1]...) +// writeTo writes the encoder output to w. +func (buf *encBuffer) writeTo(w io.Writer) (err error) { + strpos := 0 + for _, head := range buf.lheads { + // write string data before header + if head.offset-strpos > 0 { + n, err := w.Write(buf.str[strpos:head.offset]) + strpos += n + if err != nil { + return err + } + } + // write the header + enc := head.encode(buf.sizebuf[:]) + if _, err = w.Write(enc); err != nil { + return err + } + } + if strpos < len(buf.str) { + // write string data after the last list header + _, err = w.Write(buf.str[strpos:]) } + return err } -func (w *encBuffer) encodeString(b []byte) { - if len(b) == 1 && b[0] <= 0x7F { - // fits single byte, no string header - w.str = append(w.str, b[0]) +// Write implements io.Writer and appends b directly to the output. +func (buf *encBuffer) Write(b []byte) (int, error) { + buf.str = append(buf.str, b...) + return len(b), nil +} + +// writeBool writes b as the integer 0 (false) or 1 (true). +func (buf *encBuffer) writeBool(b bool) { + if b { + buf.str = append(buf.str, 0x01) } else { - w.encodeStringHeader(len(b)) - w.str = append(w.str, b...) + buf.str = append(buf.str, 0x80) } } -func (w *encBuffer) encodeUint(i uint64) { +func (buf *encBuffer) writeUint64(i uint64) { if i == 0 { - w.str = append(w.str, 0x80) + buf.str = append(buf.str, 0x80) } else if i < 128 { // fits single byte - w.str = append(w.str, byte(i)) + buf.str = append(buf.str, byte(i)) + } else { + s := putint(buf.sizebuf[1:], i) + buf.sizebuf[0] = 0x80 + byte(s) + buf.str = append(buf.str, buf.sizebuf[:s+1]...) + } +} + +func (buf *encBuffer) writeBytes(b []byte) { + if len(b) == 1 && b[0] <= 0x7F { + // fits single byte, no string header + buf.str = append(buf.str, b[0]) } else { - s := putint(w.sizebuf[1:], i) - w.sizebuf[0] = 0x80 + byte(s) - w.str = append(w.str, w.sizebuf[:s+1]...) + buf.encodeStringHeader(len(b)) + buf.str = append(buf.str, b...) } } -// list adds a new list header to the header stack. It returns the index -// of the header. The caller must call listEnd with this index after encoding -// the content of the list. -func (w *encBuffer) list() int { - w.lheads = append(w.lheads, listhead{offset: len(w.str), size: w.lhsize}) - return len(w.lheads) - 1 +func (buf *encBuffer) writeString(s string) { + buf.writeBytes([]byte(s)) } -func (w *encBuffer) listEnd(index int) { - lh := &w.lheads[index] - lh.size = w.size() - lh.offset - lh.size - if lh.size < 56 { - w.lhsize++ // length encoded into kind tag - } else { - w.lhsize += 1 + intsize(uint64(lh.size)) +// writeBigInt writes i as an integer. +func (buf *encBuffer) writeBigInt(i *big.Int) { + bitlen := i.BitLen() + if bitlen <= 64 { + buf.writeUint64(i.Uint64()) + return } + // Integer is larger than 64 bits, encode from i.Bits(). + // The minimal byte length is bitlen rounded up to the next + // multiple of 8, divided by 8. + length := ((bitlen + 7) & -8) >> 3 + buf.encodeStringHeader(length) + buf.str = append(buf.str, make([]byte, length)...) + bytesBuf := buf.str[len(buf.str)-length:] + math.ReadBits(i, bytesBuf) } -func (w *encBuffer) size() int { - return len(w.str) + w.lhsize +// writeUint256 writes z as an integer. +func (buf *encBuffer) writeUint256(z *uint256.Int) { + bitlen := z.BitLen() + if bitlen <= 64 { + buf.writeUint64(z.Uint64()) + return + } + nBytes := byte((bitlen + 7) / 8) + var b [33]byte + binary.BigEndian.PutUint64(b[1:9], z[3]) + binary.BigEndian.PutUint64(b[9:17], z[2]) + binary.BigEndian.PutUint64(b[17:25], z[1]) + binary.BigEndian.PutUint64(b[25:33], z[0]) + b[32-nBytes] = 0x80 + nBytes + buf.str = append(buf.str, b[32-nBytes:]...) } -func (w *encBuffer) toBytes() []byte { - out := make([]byte, w.size()) - strpos := 0 - pos := 0 - for _, head := range w.lheads { - // write string data before header - n := copy(out[pos:], w.str[strpos:head.offset]) - pos += n - strpos += n - // write the header - enc := head.encode(out[pos:]) - pos += len(enc) +// list adds a new list header to the header stack. It returns the index of the header. +// Call listEnd with this index after encoding the content of the list. +func (buf *encBuffer) list() int { + buf.lheads = append(buf.lheads, listhead{offset: len(buf.str), size: buf.lhsize}) + return len(buf.lheads) - 1 +} + +func (buf *encBuffer) listEnd(index int) { + lh := &buf.lheads[index] + lh.size = buf.size() - lh.offset - lh.size + if lh.size < 56 { + buf.lhsize++ // length encoded into kind tag + } else { + buf.lhsize += 1 + intsize(uint64(lh.size)) } - // copy string data after the last list header - copy(out[pos:], w.str[strpos:]) - return out } -func (w *encBuffer) toWriter(out io.Writer) (err error) { - strpos := 0 - for _, head := range w.lheads { - // write string data before header - if head.offset-strpos > 0 { - n, nErr := out.Write(w.str[strpos:head.offset]) - strpos += n - if nErr != nil { - return nErr - } - } - // write the header - enc := head.encode(w.sizebuf[:]) - if _, wErr := out.Write(enc); wErr != nil { - return wErr - } +func (buf *encBuffer) encode(val interface{}) error { + rval := reflect.ValueOf(val) + writer, err := cachedWriter(rval.Type()) + if err != nil { + return err } - if strpos < len(w.str) { - // write string data after the last list header - _, err = out.Write(w.str[strpos:]) + return writer(rval, buf) +} + +func (buf *encBuffer) encodeStringHeader(size int) { + if size < 56 { + buf.str = append(buf.str, 0x80+byte(size)) + } else { + sizesize := putint(buf.sizebuf[1:], uint64(size)) + buf.sizebuf[0] = 0xB7 + byte(sizesize) + buf.str = append(buf.str, buf.sizebuf[:sizesize+1]...) } - return err } // encReader is the io.Reader returned by EncodeToReader. @@ -225,3 +285,133 @@ func (r *encReader) next() []byte { return nil } } + +func encBufferFromWriter(w io.Writer) *encBuffer { + switch w := w.(type) { + case EncoderBuffer: + return w.buf + case *EncoderBuffer: + return w.buf + case *encBuffer: + return w + default: + return nil + } +} + +// EncoderBuffer is a buffer for incremental encoding. +// +// The zero value is NOT ready for use. To get a usable buffer, +// create it using NewEncoderBuffer or call Reset. +type EncoderBuffer struct { + buf *encBuffer + dst io.Writer + + ownBuffer bool +} + +// NewEncoderBuffer creates an encoder buffer. +func NewEncoderBuffer(dst io.Writer) EncoderBuffer { + var w EncoderBuffer + w.Reset(dst) + return w +} + +// Reset truncates the buffer and sets the output destination. +func (w *EncoderBuffer) Reset(dst io.Writer) { + if w.buf != nil && !w.ownBuffer { + panic("can't Reset derived EncoderBuffer") + } + + // If the destination writer has an *encBuffer, use it. + // Note that w.ownBuffer is left false here. + if dst != nil { + if outer := encBufferFromWriter(dst); outer != nil { + *w = EncoderBuffer{outer, nil, false} + return + } + } + + // Get a fresh buffer. + if w.buf == nil { + w.buf = encBufferPool.Get().(*encBuffer) + w.ownBuffer = true + } + w.buf.reset() + w.dst = dst +} + +// Flush writes encoded RLP data to the output writer. This can only be called once. +// If you want to re-use the buffer after Flush, you must call Reset. +func (w *EncoderBuffer) Flush() error { + var err error + if w.dst != nil { + err = w.buf.writeTo(w.dst) + } + // Release the internal buffer. + if w.ownBuffer { + encBufferPool.Put(w.buf) + } + *w = EncoderBuffer{} + return err +} + +// ToBytes returns the encoded bytes. +func (w *EncoderBuffer) ToBytes() []byte { + return w.buf.makeBytes() +} + +// AppendToBytes appends the encoded bytes to dst. +func (w *EncoderBuffer) AppendToBytes(dst []byte) []byte { + size := w.buf.size() + out := append(dst, make([]byte, size)...) + w.buf.copyTo(out[len(dst):]) + return out +} + +// Write appends b directly to the encoder output. +func (w EncoderBuffer) Write(b []byte) (int, error) { + return w.buf.Write(b) +} + +// WriteBool writes b as the integer 0 (false) or 1 (true). +func (w EncoderBuffer) WriteBool(b bool) { + w.buf.writeBool(b) +} + +// WriteUint64 encodes an unsigned integer. +func (w EncoderBuffer) WriteUint64(i uint64) { + w.buf.writeUint64(i) +} + +// WriteBigInt encodes a big.Int as an RLP string. +// Note: Unlike with Encode, the sign of i is ignored. +func (w EncoderBuffer) WriteBigInt(i *big.Int) { + w.buf.writeBigInt(i) +} + +// WriteUint256 encodes uint256.Int as an RLP string. +func (w EncoderBuffer) WriteUint256(i *uint256.Int) { + w.buf.writeUint256(i) +} + +// WriteBytes encodes b as an RLP string. +func (w EncoderBuffer) WriteBytes(b []byte) { + w.buf.writeBytes(b) +} + +// WriteString encodes s as an RLP string. +func (w EncoderBuffer) WriteString(s string) { + w.buf.writeString(s) +} + +// List starts a list. It returns an internal index. Call EndList with +// this index after encoding the content to finish the list. +func (w EncoderBuffer) List() int { + return w.buf.list() +} + +// ListEnd finishes the given list. +func (w EncoderBuffer) ListEnd(index int) { + w.buf.listEnd(index) +} diff --git a/execution/rlp/encbuffer_example_test.go b/execution/rlp/encbuffer_example_test.go new file mode 100644 index 00000000000..ee15d82a77b --- /dev/null +++ b/execution/rlp/encbuffer_example_test.go @@ -0,0 +1,45 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rlp_test + +import ( + "bytes" + "fmt" + + "github.com/ethereum/go-ethereum/rlp" +) + +func ExampleEncoderBuffer() { + var w bytes.Buffer + + // Encode [4, [5, 6]] to w. + buf := rlp.NewEncoderBuffer(&w) + l1 := buf.List() + buf.WriteUint64(4) + l2 := buf.List() + buf.WriteUint64(5) + buf.WriteUint64(6) + buf.ListEnd(l2) + buf.ListEnd(l1) + + if err := buf.Flush(); err != nil { + panic(err) + } + fmt.Printf("%X\n", w.Bytes()) + // Output: + // C404C20506 +} diff --git a/execution/rlp/encode.go b/execution/rlp/encode.go index 34e36b04421..bf52f792c68 100644 --- a/execution/rlp/encode.go +++ b/execution/rlp/encode.go @@ -29,26 +29,22 @@ import ( "reflect" "github.com/erigontech/erigon/common" + "github.com/erigontech/erigon/execution/rlp/internal/rlpstruct" "github.com/holiman/uint256" ) -// https://ethereum.org/en/developers/docs/data-structures-and-encoding/rlp/ -const ( - // EmptyStringCode is the RLP code for empty strings. - EmptyStringCode = 0x80 - // EmptyListCode is the RLP code for empty lists. - EmptyListCode = 0xC0 -) - -var ErrNegativeBigInt = errors.New("rlp: cannot encode negative big.Int") - var ( // Common encoded values. // These are useful when implementing EncodeRLP. - EmptyString = []byte{EmptyStringCode} - EmptyList = []byte{EmptyListCode} + + // EmptyString is the encoding of an empty string. + EmptyString = []byte{0x80} + // EmptyList is the encoding of an empty list. + EmptyList = []byte{0xC0} ) +var ErrNegativeBigInt = errors.New("rlp: cannot encode negative big.Int") + // Encoder is implemented by types that require custom // encoding rules or want to encode private fields. type Encoder interface { @@ -68,43 +64,30 @@ type Encoder interface { // buffered. // // Please see package-level documentation of encoding rules. -func Encode(w io.Writer, val any) error { - if outer, ok := w.(*encBuffer); ok { - // Encode was called by some type's EncodeRLP. - // Avoid copying by writing to the outer encBuffer directly. - return outer.encode(val) - } - eb := encBufferPool.Get().(*encBuffer) - defer encBufferPool.Put(eb) - eb.reset() - if err := eb.encode(val); err != nil { - return err +func Encode(w io.Writer, val interface{}) error { + // Optimization: reuse *encBuffer when called by EncodeRLP. + if buf := encBufferFromWriter(w); buf != nil { + return buf.encode(val) } - return eb.toWriter(w) -} -func Write(w io.Writer, val []byte) error { - if outer, ok := w.(*encBuffer); ok { - // Encode was called by some type's EncodeRLP. - // Avoid copying by writing to the outer encBuffer directly. - _, err := outer.Write(val) + buf := getEncBuffer() + defer encBufferPool.Put(buf) + if err := buf.encode(val); err != nil { return err } - - _, err := w.Write(val) - return err + return buf.writeTo(w) } // EncodeToBytes returns the RLP encoding of val. // Please see package-level documentation for the encoding rules. -func EncodeToBytes(val any) ([]byte, error) { - eb := encBufferPool.Get().(*encBuffer) - defer encBufferPool.Put(eb) - eb.reset() - if err := eb.encode(val); err != nil { +func EncodeToBytes(val interface{}) ([]byte, error) { + buf := getEncBuffer() + defer encBufferPool.Put(buf) + + if err := buf.encode(val); err != nil { return nil, err } - return eb.toBytes(), nil + return buf.makeBytes(), nil } // EncodeToReader returns a reader from which the RLP encoding of val @@ -112,13 +95,16 @@ func EncodeToBytes(val any) ([]byte, error) { // data. // // Please see the documentation of Encode for the encoding rules. -func EncodeToReader(val any) (size int, r io.Reader, err error) { - eb := encBufferPool.Get().(*encBuffer) - eb.reset() - if err := eb.encode(val); err != nil { +func EncodeToReader(val interface{}) (size int, r io.Reader, err error) { + buf := getEncBuffer() + if err := buf.encode(val); err != nil { + encBufferPool.Put(buf) return 0, nil, err } - return eb.size(), &encReader{buf: eb}, nil + // Note: can't put the reader back into the pool here + // because it is held by encReader. The reader puts it + // back when it has been fully consumed. + return buf.size(), &encReader{buf: buf}, nil } type listhead struct { @@ -156,27 +142,25 @@ func puthead(buf []byte, smalltag, largetag byte, size uint64) int { var encoderInterface = reflect.TypeFor[Encoder]() // makeWriter creates a writer function for the given type. -func makeWriter(typ reflect.Type, ts tags) (writer, error) { +func makeWriter(typ reflect.Type, ts rlpstruct.Tags) (writer, error) { kind := typ.Kind() switch { case typ == rawValueType: return writeRawValue, nil - case typ.AssignableTo(reflect.PtrTo(bigInt)): + case typ.AssignableTo(reflect.PointerTo(bigInt)): return writeBigIntPtr, nil case typ.AssignableTo(bigInt): return writeBigIntNoPtr, nil - case typ.AssignableTo(reflect.PtrTo(uint256Int)): - return writeUint256Ptr, nil - case typ.AssignableTo(uint256Int): - return writeUint256NoPtr, nil + case typ == reflect.PointerTo(u256Int): + return writeU256IntPtr, nil + case typ == u256Int: + return writeU256IntNoPtr, nil case kind == reflect.Ptr: return makePtrWriter(typ, ts) - case reflect.PtrTo(typ).Implements(encoderInterface): + case reflect.PointerTo(typ).Implements(encoderInterface): return makeEncoderWriter(typ), nil case isUint(kind): return writeUint, nil - case isInt(kind): - return writeInt, nil case kind == reflect.Bool: return writeBool, nil case kind == reflect.String: @@ -202,121 +186,78 @@ func writeRawValue(val reflect.Value, w *encBuffer) error { } func writeUint(val reflect.Value, w *encBuffer) error { - w.encodeUint(val.Uint()) - return nil -} - -func writeInt(val reflect.Value, w *encBuffer) error { - i := val.Int() - if i < 0 { - return fmt.Errorf("rlp: type %T -ve values are not RLP-serializable", val) - } - w.encodeUint(uint64(i)) + w.writeUint64(val.Uint()) return nil } func writeBool(val reflect.Value, w *encBuffer) error { - if val.Bool() { - w.str = append(w.str, 0x01) - } else { - w.str = append(w.str, EmptyStringCode) - } + w.writeBool(val.Bool()) return nil } func writeBigIntPtr(val reflect.Value, w *encBuffer) error { ptr := val.Interface().(*big.Int) if ptr == nil { - w.str = append(w.str, EmptyStringCode) + w.str = append(w.str, 0x80) return nil } - return writeBigInt(ptr, w) + if ptr.Sign() == -1 { + return ErrNegativeBigInt + } + w.writeBigInt(ptr) + return nil } func writeBigIntNoPtr(val reflect.Value, w *encBuffer) error { i := val.Interface().(big.Int) - return writeBigInt(&i, w) -} - -// wordBytes is the number of bytes in a big.Word -const wordBytes = (32 << (uint64(^big.Word(0)) >> 63)) / 8 - -func writeBigInt(i *big.Int, w *encBuffer) error { if i.Sign() == -1 { return ErrNegativeBigInt } - bitlen := i.BitLen() - if bitlen <= 64 { - w.encodeUint(i.Uint64()) - return nil - } - // Integer is larger than 64 bits, encode from i.Bits(). - // The minimal byte length is bitlen rounded up to the next - // multiple of 8, divided by 8. - length := ((bitlen + 7) & -8) >> 3 - w.encodeStringHeader(length) - w.str = append(w.str, make([]byte, length)...) - index := length - buf := w.str[len(w.str)-length:] - for _, d := range i.Bits() { - for j := 0; j < wordBytes && index > 0; j++ { - index-- - buf[index] = byte(d) - d >>= 8 - } - } + w.writeBigInt(&i) return nil } -func writeUint256Ptr(val reflect.Value, w *encBuffer) error { +func writeU256IntPtr(val reflect.Value, w *encBuffer) error { ptr := val.Interface().(*uint256.Int) if ptr == nil { - w.str = append(w.str, EmptyStringCode) + w.str = append(w.str, 0x80) return nil } - return writeUint256(ptr, w) + w.writeUint256(ptr) + return nil } -func writeUint256NoPtr(val reflect.Value, w *encBuffer) error { +func writeU256IntNoPtr(val reflect.Value, w *encBuffer) error { i := val.Interface().(uint256.Int) - return writeUint256(&i, w) -} - -func writeUint256(i *uint256.Int, w *encBuffer) error { - if i.IsZero() { - w.str = append(w.str, EmptyStringCode) - } else if i.LtUint64(0x80) { - w.str = append(w.str, byte(i.Uint64())) - } else { - n := i.ByteLen() - w.str = append(w.str, EmptyStringCode+byte(n)) - pos := len(w.str) - w.str = append(w.str, make([]byte, n)...) - i.WriteToSlice(w.str[pos:]) - } + w.writeUint256(&i) return nil } func writeBytes(val reflect.Value, w *encBuffer) error { - w.encodeString(val.Bytes()) + w.writeBytes(val.Bytes()) return nil } -var byteType = reflect.TypeFor[byte]() - func makeByteArrayWriter(typ reflect.Type) writer { - length := typ.Len() - if length == 0 { + switch typ.Len() { + case 0: return writeLengthZeroByteArray - } else if length == 1 { + case 1: return writeLengthOneByteArray - } - if typ.Elem() != byteType { - return writeNamedByteArray - } - return func(val reflect.Value, w *encBuffer) error { - writeByteArrayCopy(length, val, w) - return nil + default: + return func(val reflect.Value, w *encBuffer) error { + if !val.CanAddr() { + // Getting the byte slice of val requires it to be addressable. Make it + // addressable by copying. + copy := reflect.New(val.Type()).Elem() + copy.Set(val) + val = copy + } + slice := val.Bytes() + w.encodeStringHeader(len(slice)) + w.str = append(w.str, slice...) + return nil + } } } @@ -335,32 +276,6 @@ func writeLengthOneByteArray(val reflect.Value, w *encBuffer) error { return nil } -// writeByteArrayCopy encodes byte arrays using reflect.Copy. This is -// the fast path for [N]byte where N > 1. -func writeByteArrayCopy(length int, val reflect.Value, w *encBuffer) { - w.encodeStringHeader(length) - offset := len(w.str) - w.str = append(w.str, make([]byte, length)...) - w.bufvalue.SetBytes(w.str[offset:]) - reflect.Copy(w.bufvalue, val) -} - -// writeNamedByteArray encodes byte arrays with named element type. -// This exists because reflect.Copy can't be used with such types. -func writeNamedByteArray(val reflect.Value, w *encBuffer) error { - if !val.CanAddr() { - // Slice requires the value to be addressable. - // Make it addressable by copying. - copy := reflect.New(val.Type()).Elem() - copy.Set(val) - val = copy - } - size := val.Len() - slice := val.Slice(0, size).Bytes() - w.encodeString(slice) - return nil -} - func writeString(val reflect.Value, w *encBuffer) error { s := val.String() if len(s) == 1 && s[0] <= 0x7f { @@ -378,35 +293,55 @@ func writeInterface(val reflect.Value, w *encBuffer) error { // Write empty list. This is consistent with the previous RLP // encoder that we had and should therefore avoid any // problems. - w.str = append(w.str, EmptyListCode) + w.str = append(w.str, 0xC0) return nil } eval := val.Elem() - wtr, wErr := cachedWriter(eval.Type()) - if wErr != nil { - return wErr + writer, err := cachedWriter(eval.Type()) + if err != nil { + return err } - return wtr(eval, w) + return writer(eval, w) } -func makeSliceWriter(typ reflect.Type, ts tags) (writer, error) { - etypeinfo := theTC.infoWhileGenerating(typ.Elem(), tags{}) +func makeSliceWriter(typ reflect.Type, ts rlpstruct.Tags) (writer, error) { + etypeinfo := theTC.infoWhileGenerating(typ.Elem(), rlpstruct.Tags{}) if etypeinfo.writerErr != nil { return nil, etypeinfo.writerErr } - writer := func(val reflect.Value, w *encBuffer) error { - if !ts.tail { - defer w.listEnd(w.list()) + + var wfn writer + if ts.Tail { + // This is for struct tail slices. + // w.list is not called for them. + wfn = func(val reflect.Value, w *encBuffer) error { + vlen := val.Len() + for i := 0; i < vlen; i++ { + if err := etypeinfo.writer(val.Index(i), w); err != nil { + return err + } + } + return nil } - vlen := val.Len() - for i := 0; i < vlen; i++ { - if err := etypeinfo.writer(val.Index(i), w); err != nil { - return err + } else { + // This is for regular slices and arrays. + wfn = func(val reflect.Value, w *encBuffer) error { + vlen := val.Len() + if vlen == 0 { + w.str = append(w.str, 0xC0) + return nil } + listOffset := w.list() + for i := 0; i < vlen; i++ { + if err := etypeinfo.writer(val.Index(i), w); err != nil { + return err + } + } + w.listEnd(listOffset) + return nil } - return nil } - return writer, nil + return wfn, nil } func makeStructWriter(typ reflect.Type) (writer, error) { @@ -457,29 +392,23 @@ func makeStructWriter(typ reflect.Type) (writer, error) { return writer, nil } -func makePtrWriter(typ reflect.Type, ts tags) (writer, error) { - etypeinfo := theTC.infoWhileGenerating(typ.Elem(), tags{}) +func makePtrWriter(typ reflect.Type, ts rlpstruct.Tags) (writer, error) { + nilEncoding := byte(0xC0) + if typeNilKind(typ.Elem(), ts) == String { + nilEncoding = 0x80 + } + + etypeinfo := theTC.infoWhileGenerating(typ.Elem(), rlpstruct.Tags{}) if etypeinfo.writerErr != nil { return nil, etypeinfo.writerErr } - // Determine how to encode nil pointers. - var nilKind Kind - if ts.nilOK { - nilKind = ts.nilKind // use struct tag if provided - } else { - nilKind = defaultNilKind(typ.Elem()) - } writer := func(val reflect.Value, w *encBuffer) error { - if val.IsNil() { - if nilKind == String { - w.str = append(w.str, EmptyStringCode) - } else { - w.listEnd(w.list()) - } - return nil + if ev := val.Elem(); ev.IsValid() { + return etypeinfo.writer(ev, w) } - return etypeinfo.writer(val.Elem(), w) + w.str = append(w.str, nilEncoding) + return nil } return writer, nil } diff --git a/execution/rlp/internal/rlpstruct/rlpstruct.go b/execution/rlp/internal/rlpstruct/rlpstruct.go index ca6adec0957..99c37173166 100644 --- a/execution/rlp/internal/rlpstruct/rlpstruct.go +++ b/execution/rlp/internal/rlpstruct/rlpstruct.go @@ -1,4 +1,4 @@ -// Copyright 2017 The go-ethereum Authors +// Copyright 2022 The go-ethereum Authors // (original work) // Copyright 2024 The Erigon Authors // (modifications) @@ -151,7 +151,7 @@ func parseTag(field Field, lastPublic int) (Tags, error) { name := field.Name tag := reflect.StructTag(field.Tag) var ts Tags - for t := range strings.SplitSeq(tag.Get("rlp"), ",") { + for _, t := range strings.Split(tag.Get("rlp"), ",") { switch t = strings.TrimSpace(t); t { case "": // empty tag is allowed for some reason diff --git a/execution/rlp/raw.go b/execution/rlp/raw.go index 0ef83528573..8994f97d136 100644 --- a/execution/rlp/raw.go +++ b/execution/rlp/raw.go @@ -31,13 +31,46 @@ type RawValue []byte var rawValueType = reflect.TypeFor[RawValue]() +// StringSize returns the encoded size of a string. +func StringSize(s string) uint64 { + switch n := len(s); n { + case 0: + return 1 + case 1: + if s[0] <= 0x7f { + return 1 + } else { + return 2 + } + default: + return uint64(headsize(uint64(n)) + n) + } +} + +// BytesSize returns the encoded size of a byte slice. +func BytesSize(b []byte) uint64 { + switch n := len(b); n { + case 0: + return 1 + case 1: + if b[0] <= 0x7f { + return 1 + } else { + return 2 + } + default: + return uint64(headsize(uint64(n)) + n) + } +} + // ListSize returns the encoded size of an RLP list with the given // content size. func ListSize(contentSize uint64) uint64 { return uint64(headsize(contentSize)) + contentSize } -// IntSize returns the encoded size of the integer x. +// IntSize returns the encoded size of the integer x. Note: The return type of this +// function is 'int' for backwards-compatibility reasons. The result is always positive. func IntSize(x uint64) int { if x < 0x80 { return 1 @@ -75,18 +108,20 @@ func SplitUint64(b []byte) (x uint64, rest []byte, err error) { if err != nil { return 0, b, err } - switch { - case len(content) == 0: + switch n := len(content); n { + case 0: return 0, rest, nil - case len(content) == 1: + case 1: if content[0] == 0 { return 0, b, ErrCanonInt } return uint64(content[0]), rest, nil - case len(content) > 8: - return 0, b, errUintOverflow default: - x, err = readSize(content, byte(len(content))) + if n > 8 { + return 0, b, errUintOverflow + } + + x, err = readSize(content, byte(n)) if err != nil { return 0, b, ErrCanonInt } @@ -120,6 +155,45 @@ func CountValues(b []byte) (int, error) { return i, nil } +// SplitListValues extracts the raw elements from the list RLP-encoding blob. +// +// Note: the returned slice must not be modified, as it shares the same +// backing array as the original slice. It's acceptable to deep-copy the elements +// out if necessary, but let's stick with this approach for less allocation +// overhead. +func SplitListValues(b []byte) ([][]byte, error) { + b, _, err := SplitList(b) + if err != nil { + return nil, err + } + n, err := CountValues(b) + if err != nil { + return nil, err + } + var elements = make([][]byte, 0, n) + + for len(b) > 0 { + _, tagsize, size, err := readKind(b) + if err != nil { + return nil, err + } + elements = append(elements, b[:tagsize+size]) + b = b[tagsize+size:] + } + return elements, nil +} + +// MergeListValues takes a list of raw elements and rlp-encodes them as list. +func MergeListValues(elems [][]byte) ([]byte, error) { + w := NewEncoderBuffer(nil) + offset := w.List() + for _, elem := range elems { + w.Write(elem) + } + w.ListEnd(offset) + return w.ToBytes(), nil +} + func readKind(buf []byte) (k Kind, tagsize, contentsize uint64, err error) { if len(buf) == 0 { return 0, 0, 0, io.ErrUnexpectedEOF diff --git a/execution/rlp/raw_test.go b/execution/rlp/raw_test.go index c5d71c3dcdc..6627307b908 100644 --- a/execution/rlp/raw_test.go +++ b/execution/rlp/raw_test.go @@ -58,21 +58,41 @@ func TestCountValues(t *testing.T) { if count != test.count { t.Errorf("test %d: count mismatch, got %d want %d\ninput: %s", i, count, test.count, test.input) } - if !reflect.DeepEqual(err, test.err) { + if !errors.Is(err, test.err) { t.Errorf("test %d: err mismatch, got %q want %q\ninput: %s", i, err, test.err, test.input) } } } -func TestSplitTypes(t *testing.T) { - if _, _, err := SplitString(unhex("C100")); !errors.Is(err, ErrExpectedString) { - t.Errorf("SplitString returned %q, want %q", err, ErrExpectedString) - } - if _, _, err := SplitList(unhex("01")); !errors.Is(err, ErrExpectedList) { - t.Errorf("SplitString returned %q, want %q", err, ErrExpectedList) +func TestSplitString(t *testing.T) { + for i, test := range []string{ + "C0", + "C100", + "C3010203", + "C88363617483646F67", + "F8384C6F72656D20697073756D20646F6C6F722073697420616D65742C20636F6E7365637465747572206164697069736963696E6720656C6974", + } { + if _, _, err := SplitString(unhex(test)); !errors.Is(err, ErrExpectedString) { + t.Errorf("test %d: error mismatch: have %q, want %q", i, err, ErrExpectedString) + } } - if _, _, err := SplitList(unhex("81FF")); !errors.Is(err, ErrExpectedList) { - t.Errorf("SplitString returned %q, want %q", err, ErrExpectedList) +} + +func TestSplitList(t *testing.T) { + for i, test := range []string{ + "80", + "00", + "01", + "8180", + "81FF", + "820400", + "83636174", + "83646F67", + "B8384C6F72656D20697073756D20646F6C6F722073697420616D65742C20636F6E7365637465747572206164697069736963696E6720656C6974", + } { + if _, _, err := SplitList(unhex(test)); !errors.Is(err, ErrExpectedList) { + t.Errorf("test %d: error mismatch: have %q, want %q", i, err, ErrExpectedList) + } } } @@ -113,7 +133,7 @@ func TestSplitUint64(t *testing.T) { if !bytes.Equal(rest, unhex(test.rest)) { t.Errorf("test %d: rest mismatch: got %x, want %s (input %q)", i, rest, test.rest, test.input) } - if !errors.Is(err, test.err) { + if err != test.err { t.Errorf("test %d: error mismatch: got %q, want %q", i, err, test.err) } } @@ -197,7 +217,7 @@ func TestSplit(t *testing.T) { if !bytes.Equal(rest, unhex(test.rest)) { t.Errorf("test %d: rest mismatch: got %x, want %s", i, rest, test.rest) } - if !errors.Is(err, test.err) { + if err != test.err { t.Errorf("test %d: error mismatch: got %q, want %q", i, err, test.err) } } @@ -235,7 +255,7 @@ func TestReadSize(t *testing.T) { for _, test := range tests { size, err := readSize(unhex(test.input), test.slen) - if !errors.Is(err, test.err) { + if err != test.err { t.Errorf("readSize(%s, %d): error mismatch: got %q, want %q", test.input, test.slen, err, test.err) continue } @@ -287,3 +307,302 @@ func TestAppendUint64Random(t *testing.T) { t.Fatal(err) } } + +func TestBytesSize(t *testing.T) { + tests := []struct { + v []byte + size uint64 + }{ + {v: []byte{}, size: 1}, + {v: []byte{0x1}, size: 1}, + {v: []byte{0x7E}, size: 1}, + {v: []byte{0x7F}, size: 1}, + {v: []byte{0x80}, size: 2}, + {v: []byte{0xFF}, size: 2}, + {v: []byte{0xFF, 0xF0}, size: 3}, + {v: make([]byte, 55), size: 56}, + {v: make([]byte, 56), size: 58}, + } + + for _, test := range tests { + s := BytesSize(test.v) + if s != test.size { + t.Errorf("BytesSize(%#x) -> %d, want %d", test.v, s, test.size) + } + s = StringSize(string(test.v)) + if s != test.size { + t.Errorf("StringSize(%#x) -> %d, want %d", test.v, s, test.size) + } + // Sanity check: + enc, _ := EncodeToBytes(test.v) + if uint64(len(enc)) != test.size { + t.Errorf("len(EncodeToBytes(%#x)) -> %d, test says %d", test.v, len(enc), test.size) + } + } +} + +func TestSplitListValues(t *testing.T) { + tests := []struct { + name string + input string // hex-encoded RLP list + want []string // hex-encoded expected elements + wantErr error + }{ + { + name: "empty list", + input: "C0", + want: []string{}, + }, + { + name: "single byte element", + input: "C101", + want: []string{"01"}, + }, + { + name: "single empty string", + input: "C180", + want: []string{"80"}, + }, + { + name: "two byte elements", + input: "C20102", + want: []string{"01", "02"}, + }, + { + name: "three elements", + input: "C3010203", + want: []string{"01", "02", "03"}, + }, + { + name: "mixed size elements", + input: "C80182020283030303", + want: []string{"01", "820202", "83030303"}, + }, + { + name: "string elements", + input: "C88363617483646F67", + want: []string{"83636174", "83646F67"}, // cat,dog + }, + { + name: "nested list element", + input: "C4C3010203", // [[1,2,3]] + want: []string{"C3010203"}, // [1,2,3] + }, + { + name: "multiple nested lists", + input: "C6C20102C20304", // [[1,2],[3,4]] + want: []string{"C20102", "C20304"}, // [1,2], [3,4] + }, + { + name: "large list", + input: "C6010203040506", + want: []string{"01", "02", "03", "04", "05", "06"}, + }, + { + name: "list with empty strings", + input: "C3808080", + want: []string{"80", "80", "80"}, + }, + // Error cases + { + name: "single byte", + input: "01", + wantErr: ErrExpectedList, + }, + { + name: "string", + input: "83636174", + wantErr: ErrExpectedList, + }, + { + name: "empty input", + input: "", + wantErr: io.ErrUnexpectedEOF, + }, + { + name: "invalid list - value too large", + input: "C60102030405", + wantErr: ErrValueTooLarge, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := SplitListValues(unhex(tt.input)) + if !errors.Is(err, tt.wantErr) { + t.Errorf("SplitListValues() error = %v, wantErr %v", err, tt.wantErr) + return + } + if err != nil { + return + } + if len(got) != len(tt.want) { + t.Errorf("SplitListValues() got %d elements, want %d", len(got), len(tt.want)) + return + } + for i, elem := range got { + want := unhex(tt.want[i]) + if !bytes.Equal(elem, want) { + t.Errorf("SplitListValues() element[%d] = %x, want %x", i, elem, want) + } + } + }) + } +} + +func TestMergeListValues(t *testing.T) { + tests := []struct { + name string + elems []string // hex-encoded RLP elements + want string // hex-encoded expected result + wantErr error + }{ + { + name: "empty list", + elems: []string{}, + want: "C0", + }, + { + name: "single byte element", + elems: []string{"01"}, + want: "C101", + }, + { + name: "single empty string", + elems: []string{"80"}, + want: "C180", + }, + { + name: "two byte elements", + elems: []string{"01", "02"}, + want: "C20102", + }, + { + name: "three elements", + elems: []string{"01", "02", "03"}, + want: "C3010203", + }, + { + name: "mixed size elements", + elems: []string{"01", "820202", "83030303"}, + want: "C80182020283030303", + }, + { + name: "string elements", + elems: []string{"83636174", "83646F67"}, // cat, dog + want: "C88363617483646F67", + }, + { + name: "nested list element", + elems: []string{"C20102", "03"}, // [[1, 2], 3] + want: "C4C2010203", + }, + { + name: "multiple nested lists", + elems: []string{"C20102", "C3030405"}, // [[1,2],[3,4,5]], + want: "C7C20102C3030405", + }, + { + name: "large list", + elems: []string{"01", "02", "03", "04", "05", "06"}, + want: "C6010203040506", + }, + { + name: "list with empty strings", + elems: []string{"80", "80", "80"}, + want: "C3808080", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + elems := make([][]byte, len(tt.elems)) + for i, s := range tt.elems { + elems[i] = unhex(s) + } + got, err := MergeListValues(elems) + if !errors.Is(err, tt.wantErr) { + t.Errorf("MergeListValues() error = %v, wantErr %v", err, tt.wantErr) + return + } + if err != nil { + return + } + want := unhex(tt.want) + if !bytes.Equal(got, want) { + t.Errorf("MergeListValues() = %x, want %x", got, want) + } + }) + } +} + +func TestSplitMergeList(t *testing.T) { + tests := []struct { + name string + input string // hex-encoded RLP list + }{ + { + name: "empty list", + input: "C0", + }, + { + name: "single byte element", + input: "C101", + }, + { + name: "two byte elements", + input: "C20102", + }, + { + name: "three elements", + input: "C3010203", + }, + { + name: "mixed size elements", + input: "C80182020283030303", + }, + { + name: "string elements", + input: "C88363617483646F67", // [cat, dog] + }, + { + name: "nested list element", + input: "C4C2010203", // [[1,2],3] + }, + { + name: "multiple nested lists", + input: "C6C20102C20304", // [[1,2],[3,4]] + }, + { + name: "large list", + input: "C6010203040506", // [1,2,3,4,5,6] + }, + { + name: "list with empty strings", + input: "C3808080", // ["", "", ""] + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + original := unhex(tt.input) + + // Split the list + elements, err := SplitListValues(original) + if err != nil { + t.Fatalf("SplitListValues() error = %v", err) + } + + // Merge back + merged, err := MergeListValues(elements) + if err != nil { + t.Fatalf("MergeListValues() error = %v", err) + } + + // The merged result should match the original + if !bytes.Equal(merged, original) { + t.Errorf("Round trip failed: original = %x, merged = %x", original, merged) + } + }) + } +} diff --git a/execution/rlp/typecache.go b/execution/rlp/typecache.go index 630d7d23b82..c03a69efc61 100644 --- a/execution/rlp/typecache.go +++ b/execution/rlp/typecache.go @@ -23,11 +23,13 @@ import ( "fmt" "maps" "reflect" - "strings" "sync" "sync/atomic" + + "github.com/erigontech/erigon/execution/rlp/internal/rlpstruct" ) +// typeinfo is an entry in the type cache. type typeinfo struct { decoder decoder decoderErr error // error from makeDecoder @@ -35,46 +37,17 @@ type typeinfo struct { writerErr error // error from makeWriter } -// tags represents struct tags. -type tags struct { - // rlp:"nil" controls whether empty input results in a nil pointer. - // nilKind is the kind of empty value allowed for the field. - nilKind Kind - nilOK bool - - // rlp:"optional" allows for a field to be missing in the input list. - // If this is set, all subsequent fields must also be optional. - optional bool - - // rlp:"tail" controls whether this field swallows additional list elements. It can - // only be set for the last field, which must be of slice type. - tail bool - - // rlp:"-" ignores fields. - ignored bool -} - // typekey is the key of a type in typeCache. It includes the struct tags because // they might generate a different decoder. type typekey struct { reflect.Type - tags + rlpstruct.Tags } type decoder func(*Stream, reflect.Value) error type writer func(reflect.Value, *encBuffer) error -func cachedDecoder(typ reflect.Type) (decoder, error) { - info := theTC.info(typ) - return info.decoder, info.decoderErr -} - -func cachedWriter(typ reflect.Type) (writer, error) { - info := theTC.info(typ) - return info.writer, info.writerErr -} - var theTC = newTypeCache() type typeCache struct { @@ -91,6 +64,16 @@ func newTypeCache() *typeCache { return c } +func cachedDecoder(typ reflect.Type) (decoder, error) { + info := theTC.info(typ) + return info.decoder, info.decoderErr +} + +func cachedWriter(typ reflect.Type) (writer, error) { + info := theTC.info(typ) + return info.writer, info.writerErr +} + func (c *typeCache) info(typ reflect.Type) *typeinfo { key := typekey{Type: typ} if info := c.cur.Load().(map[typekey]*typeinfo)[key]; info != nil { @@ -98,10 +81,10 @@ func (c *typeCache) info(typ reflect.Type) *typeinfo { } // Not in the cache, need to generate info for this type. - return c.generate(typ, tags{}) + return c.generate(typ, rlpstruct.Tags{}) } -func (c *typeCache) generate(typ reflect.Type, tags tags) *typeinfo { +func (c *typeCache) generate(typ reflect.Type, tags rlpstruct.Tags) *typeinfo { c.mu.Lock() defer c.mu.Unlock() @@ -111,8 +94,7 @@ func (c *typeCache) generate(typ reflect.Type, tags tags) *typeinfo { } // Copy cur to next. - c.next = make(map[typekey]*typeinfo, len(cur)+1) - maps.Copy(c.next, cur) + c.next = maps.Clone(cur) // Generate. info := c.infoWhileGenerating(typ, tags) @@ -123,7 +105,7 @@ func (c *typeCache) generate(typ reflect.Type, tags tags) *typeinfo { return info } -func (c *typeCache) infoWhileGenerating(typ reflect.Type, tags tags) *typeinfo { +func (c *typeCache) infoWhileGenerating(typ reflect.Type, tags rlpstruct.Tags) *typeinfo { key := typekey{typ, tags} if info := c.next[key]; info != nil { return info @@ -145,35 +127,40 @@ type field struct { // structFields resolves the typeinfo of all public fields in a struct type. func structFields(typ reflect.Type) (fields []field, err error) { - var ( - lastPublic = lastPublicField(typ) - anyOptional = false - ) + // Convert fields to rlpstruct.Field. + var allStructFields []rlpstruct.Field for i := 0; i < typ.NumField(); i++ { - if f := typ.Field(i); f.PkgPath == "" { // exported - tags, err := parseStructTag(typ, i, lastPublic) - if err != nil { - return nil, err - } - - // Skip rlp:"-" fields. - if tags.ignored { - continue - } - // If any field has the "optional" tag, subsequent fields must also have it. - if tags.optional || tags.tail { - anyOptional = true - } else if anyOptional { - return nil, fmt.Errorf(`rlp: struct field %v.%s needs "optional" tag`, typ, f.Name) - } - info := theTC.infoWhileGenerating(f.Type, tags) - fields = append(fields, field{i, info, tags.optional}) + rf := typ.Field(i) + allStructFields = append(allStructFields, rlpstruct.Field{ + Name: rf.Name, + Index: i, + Exported: rf.PkgPath == "", + Tag: string(rf.Tag), + Type: *rtypeToStructType(rf.Type, nil), + }) + } + + // Filter/validate fields. + structFields, structTags, err := rlpstruct.ProcessFields(allStructFields) + if err != nil { + if tagErr, ok := err.(rlpstruct.TagError); ok { + tagErr.StructType = typ.String() + return nil, tagErr } + return nil, err + } + + // Resolve typeinfo. + for i, sf := range structFields { + typ := typ.Field(sf.Index).Type + tags := structTags[i] + info := theTC.infoWhileGenerating(typ, tags) + fields = append(fields, field{sf.Index, info, tags.Optional}) } return fields, nil } -// anyOptionalFields returns the index of the first field with "optional" tag. +// firstOptionalField returns the index of the first field with "optional" tag. func firstOptionalField(fields []field) int { for i, f := range fields { if f.optional { @@ -193,96 +180,62 @@ func (e structFieldError) Error() string { return fmt.Sprintf("%v (struct field %v.%s)", e.err, e.typ, e.typ.Field(e.field).Name) } -type structTagError struct { - typ reflect.Type - field, tag, err string +func (i *typeinfo) generate(typ reflect.Type, tags rlpstruct.Tags) { + i.decoder, i.decoderErr = makeDecoder(typ, tags) + i.writer, i.writerErr = makeWriter(typ, tags) } -func (e structTagError) Error() string { - return fmt.Sprintf("rlp: invalid struct tag %q for %v.%s (%s)", e.tag, e.typ, e.field, e.err) -} +// rtypeToStructType converts typ to rlpstruct.Type. +func rtypeToStructType(typ reflect.Type, rec map[reflect.Type]*rlpstruct.Type) *rlpstruct.Type { + k := typ.Kind() + if k == reflect.Invalid { + panic("invalid kind") + } -func parseStructTag(typ reflect.Type, fi, lastPublic int) (tags, error) { - f := typ.Field(fi) - var ts tags - for t := range strings.SplitSeq(f.Tag.Get("rlp"), ",") { - switch t = strings.TrimSpace(t); t { - case "": - case "-": - ts.ignored = true - case "nil", "nilString", "nilList": - ts.nilOK = true - if f.Type.Kind() != reflect.Ptr { - return ts, structTagError{typ, f.Name, t, "field is not a pointer"} - } - switch t { - case "nil": - ts.nilKind = defaultNilKind(f.Type.Elem()) - case "nilString": - ts.nilKind = String - case "nilList": - ts.nilKind = List - } - case "optional": - ts.optional = true - if ts.tail { - return ts, structTagError{typ, f.Name, t, `also has "tail" tag`} - } - case "tail": - ts.tail = true - if fi != lastPublic { - return ts, structTagError{typ, f.Name, t, "must be on last field"} - } - if ts.optional { - return ts, structTagError{typ, f.Name, t, `also has "optional" tag`} - } - if f.Type.Kind() != reflect.Slice { - return ts, structTagError{typ, f.Name, t, "field type is not slice"} - } - default: - return ts, fmt.Errorf("rlp: unknown struct tag %q on %v.%s", t, typ, f.Name) - } + if prev := rec[typ]; prev != nil { + return prev // short-circuit for recursive types + } + if rec == nil { + rec = make(map[reflect.Type]*rlpstruct.Type) } - return ts, nil -} -func lastPublicField(typ reflect.Type) int { - last := 0 - for i := 0; i < typ.NumField(); i++ { - if typ.Field(i).PkgPath == "" { - last = i - } + t := &rlpstruct.Type{ + Name: typ.String(), + Kind: k, + IsEncoder: typ.Implements(encoderInterface), + IsDecoder: typ.Implements(decoderInterface), + } + rec[typ] = t + if k == reflect.Array || k == reflect.Slice || k == reflect.Ptr { + t.Elem = rtypeToStructType(typ.Elem(), rec) } - return last + return t } -func (i *typeinfo) generate(typ reflect.Type, tags tags) { - i.decoder, i.decoderErr = makeDecoder(typ, tags) - i.writer, i.writerErr = makeWriter(typ, tags) -} +// typeNilKind gives the RLP value kind for nil pointers to 'typ'. +func typeNilKind(typ reflect.Type, tags rlpstruct.Tags) Kind { + styp := rtypeToStructType(typ, nil) -// defaultNilKind determines whether a nil pointer to typ encodes/decodes -// as an empty string or empty list. -func defaultNilKind(typ reflect.Type) Kind { - k := typ.Kind() - if isInt(k) || isUint(k) || k == reflect.String || k == reflect.Bool || isByteArray(typ) { + var nk rlpstruct.NilKind + if tags.NilOK { + nk = tags.NilKind + } else { + nk = styp.DefaultNilValue() + } + switch nk { + case rlpstruct.NilKindString: return String + case rlpstruct.NilKindList: + return List + default: + panic("invalid nil kind value") } - return List } func isUint(k reflect.Kind) bool { return k >= reflect.Uint && k <= reflect.Uintptr } -func isInt(k reflect.Kind) bool { - return k >= reflect.Int && k <= reflect.Int8 -} - func isByte(typ reflect.Type) bool { return typ.Kind() == reflect.Uint8 && !typ.Implements(encoderInterface) } - -func isByteArray(typ reflect.Type) bool { - return (typ.Kind() == reflect.Slice || typ.Kind() == reflect.Array) && isByte(typ.Elem()) -} diff --git a/go.mod b/go.mod index 44728377a16..56305a1076b 100644 --- a/go.mod +++ b/go.mod @@ -286,6 +286,7 @@ require ( github.com/rivo/uniseg v0.4.7 // indirect github.com/rogpeppe/go-internal v1.14.1 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/shirou/gopsutil v3.21.11+incompatible // indirect github.com/shopspring/decimal v1.2.0 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/sosodev/duration v1.3.1 // indirect diff --git a/go.sum b/go.sum index 425356edcba..36a76b4dcad 100644 --- a/go.sum +++ b/go.sum @@ -877,6 +877,8 @@ github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5P github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= +github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= +github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shirou/gopsutil/v4 v4.25.10 h1:at8lk/5T1OgtuCp+AwrDofFRjnvosn0nkN2OLQ6g8tA= github.com/shirou/gopsutil/v4 v4.25.10/go.mod h1:+kSwyC8DRUD9XXEHCAFjK+0nuArFJM0lva+StQAcskM= github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= diff --git a/p2p/discover/common.go b/p2p/discover/common.go index 6098d5dcf82..c3de9e9fb22 100644 --- a/p2p/discover/common.go +++ b/p2p/discover/common.go @@ -1,33 +1,33 @@ // Copyright 2019 The go-ethereum Authors -// (original work) -// Copyright 2024 The Erigon Authors -// (modifications) -// This file is part of Erigon. +// This file is part of the go-ethereum library. // -// Erigon is free software: you can redistribute it and/or modify +// The go-ethereum library is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // -// Erigon is distributed in the hope that it will be useful, +// The go-ethereum library is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Lesser General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License -// along with Erigon. If not, see . +// along with the go-ethereum library. If not, see . package discover import ( - "context" "crypto/ecdsa" + crand "crypto/rand" + "encoding/binary" + "math/rand" "net" + "net/netip" + "sync" "time" - "github.com/erigontech/erigon/common/crypto" - "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/common/mclock" + "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/p2p/enode" "github.com/erigontech/erigon/p2p/enr" "github.com/erigontech/erigon/p2p/netutil" @@ -35,8 +35,8 @@ import ( // UDPConn is a network connection on which discovery can operate. type UDPConn interface { - ReadFromUDP(b []byte) (n int, addr *net.UDPAddr, err error) - WriteToUDP(b []byte, addr *net.UDPAddr) (n int, err error) + ReadFromUDPAddrPort(b []byte) (n int, addr netip.AddrPort, err error) + WriteToUDPAddrPort(b []byte, addr netip.AddrPort) (n int, err error) Close() error LocalAddr() net.Addr } @@ -46,23 +46,39 @@ type Config struct { // These settings are required and configure the UDP listener: PrivateKey *ecdsa.PrivateKey - // These settings are optional: - NetRestrict *netutil.Netlist // network whitelist - Bootnodes []*enode.Node // list of bootstrap nodes - Unhandled chan<- ReadPacket // unhandled packets are sent on this channel - Log log.Logger // if set, log messages go here - ValidSchemes enr.IdentityScheme // allowed identity schemes - Clock mclock.Clock - ReplyTimeout time.Duration + // All remaining settings are optional. - PingBackDelay time.Duration + // Packet handling configuration: + NetRestrict *netutil.Netlist // list of allowed IP networks + Unhandled chan<- ReadPacket // unhandled packets are sent on this channel + V5RespTimeout time.Duration // timeout for v5 queries - PrivateKeyGenerator func() (*ecdsa.PrivateKey, error) + // Node table configuration: + Bootnodes []*enode.Node // list of bootstrap nodes + PingInterval time.Duration // speed of node liveness check + RefreshInterval time.Duration // used in bucket refresh + NoFindnodeLivenessCheck bool // turns off validation of table nodes in FINDNODE handler - TableRevalidateInterval time.Duration + // The options below are useful in very specific cases, like in unit tests. + V5ProtocolID *[6]byte + Log log.Logger // if set, log messages go here + ValidSchemes enr.IdentityScheme // allowed identity schemes + Clock mclock.Clock } -func (cfg Config) withDefaults(defaultReplyTimeout time.Duration) Config { +func (cfg Config) withDefaults() Config { + // Node table configuration: + if cfg.PingInterval == 0 { + cfg.PingInterval = 3 * time.Second + } + if cfg.RefreshInterval == 0 { + cfg.RefreshInterval = 30 * time.Minute + } + if cfg.V5RespTimeout == 0 { + cfg.V5RespTimeout = 700 * time.Millisecond + } + + // Debug/test settings: if cfg.Log == nil { cfg.Log = log.Root() } @@ -72,34 +88,58 @@ func (cfg Config) withDefaults(defaultReplyTimeout time.Duration) Config { if cfg.Clock == nil { cfg.Clock = mclock.System{} } - if cfg.ReplyTimeout == 0 { - cfg.ReplyTimeout = defaultReplyTimeout - } - if cfg.PingBackDelay == 0 { - cfg.PingBackDelay = respTimeout - } - if cfg.PrivateKeyGenerator == nil { - cfg.PrivateKeyGenerator = crypto.GenerateKey - } - if cfg.TableRevalidateInterval == 0 { - cfg.TableRevalidateInterval = revalidateInterval - } return cfg } // ListenUDP starts listening for discovery packets on the given UDP socket. -func ListenUDP(ctx context.Context, protocol string, c UDPConn, ln *enode.LocalNode, cfg Config) (*UDPv4, error) { - return ListenV4(ctx, protocol, c, ln, cfg) +func ListenUDP(c UDPConn, ln *enode.LocalNode, cfg Config) (*UDPv4, error) { + return ListenV4(c, ln, cfg) } // ReadPacket is a packet that couldn't be handled. Those packets are sent to the unhandled // channel if configured. type ReadPacket struct { Data []byte - Addr *net.UDPAddr + Addr netip.AddrPort +} + +type randomSource interface { + Intn(int) int + Int63n(int64) int64 + Shuffle(int, func(int, int)) +} + +// reseedingRandom is a random number generator that tracks when it was last re-seeded. +type reseedingRandom struct { + mu sync.Mutex + cur *rand.Rand +} + +func (r *reseedingRandom) seed() { + var b [8]byte + crand.Read(b[:]) + seed := binary.BigEndian.Uint64(b[:]) + new := rand.New(rand.NewSource(int64(seed))) + + r.mu.Lock() + r.cur = new + r.mu.Unlock() +} + +func (r *reseedingRandom) Intn(n int) int { + r.mu.Lock() + defer r.mu.Unlock() + return r.cur.Intn(n) +} + +func (r *reseedingRandom) Int63n(n int64) int64 { + r.mu.Lock() + defer r.mu.Unlock() + return r.cur.Int63n(n) } -type UnhandledPacket struct { - ReadPacket - Reason error +func (r *reseedingRandom) Shuffle(n int, swap func(i, j int)) { + r.mu.Lock() + defer r.mu.Unlock() + r.cur.Shuffle(n, swap) } diff --git a/p2p/discover/lookup.go b/p2p/discover/lookup.go index 866a663dafc..692f35b0764 100644 --- a/p2p/discover/lookup.go +++ b/p2p/discover/lookup.go @@ -1,21 +1,18 @@ // Copyright 2019 The go-ethereum Authors -// (original work) -// Copyright 2024 The Erigon Authors -// (modifications) -// This file is part of Erigon. +// This file is part of the go-ethereum library. // -// Erigon is free software: you can redistribute it and/or modify +// The go-ethereum library is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // -// Erigon is distributed in the hope that it will be useful, +// The go-ethereum library is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Lesser General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License -// along with Erigon. If not, see . +// along with the go-ethereum library. If not, see . package discover @@ -30,19 +27,19 @@ import ( // lookup performs a network search for nodes close to the given target. It approaches the // target by querying nodes that are closer to it on each iteration. The given target does // not need to be an actual node identifier. +// lookup on an empty table will return immediately with no nodes. type lookup struct { tab *Table - queryfunc func(*node) ([]*node, error) - replyCh chan []*node + queryfunc queryFunc + replyCh chan []*enode.Node cancelCh <-chan struct{} asked, seen map[enode.ID]bool result nodesByDistance - replyBuffer []*node + replyBuffer []*enode.Node queries int - noSlowdown bool } -type queryFunc func(*node) ([]*node, error) +type queryFunc func(*enode.Node) ([]*enode.Node, error) func newLookup(ctx context.Context, tab *Table, target enode.ID, q queryFunc) *lookup { it := &lookup{ @@ -51,15 +48,17 @@ func newLookup(ctx context.Context, tab *Table, target enode.ID, q queryFunc) *l asked: make(map[enode.ID]bool), seen: make(map[enode.ID]bool), result: nodesByDistance{target: target}, - replyCh: make(chan []*node, alpha), + replyCh: make(chan []*enode.Node, alpha), cancelCh: ctx.Done(), - queries: -1, } - it.noSlowdown = isDisabledLookupSlowdown(ctx) - // Don't query further if we hit ourself. // Unlikely to happen often in practice. it.asked[tab.self().ID()] = true + it.seen[tab.self().ID()] = true + + // Initialize the lookup with nodes from table. + closest := it.tab.findnodeByID(it.result.target, bucketSize, false) + it.addNodes(closest.entries) return it } @@ -67,7 +66,11 @@ func newLookup(ctx context.Context, tab *Table, target enode.ID, q queryFunc) *l func (it *lookup) run() []*enode.Node { for it.advance() { } - return unwrapNodes(it.result.entries) + return it.result.entries +} + +func (it *lookup) empty() bool { + return len(it.replyBuffer) == 0 } // advance advances the lookup until any new nodes have been found. @@ -76,16 +79,9 @@ func (it *lookup) advance() bool { for it.startQueries() { select { case nodes := <-it.replyCh: - it.replyBuffer = it.replyBuffer[:0] - for _, n := range nodes { - if n != nil && !it.seen[n.ID()] { - it.seen[n.ID()] = true - it.result.push(n, bucketSize) - it.replyBuffer = append(it.replyBuffer, n) - } - } it.queries-- - if len(it.replyBuffer) > 0 { + it.addNodes(nodes) + if !it.empty() { return true } case <-it.cancelCh: @@ -95,6 +91,17 @@ func (it *lookup) advance() bool { return false } +func (it *lookup) addNodes(nodes []*enode.Node) { + it.replyBuffer = it.replyBuffer[:0] + for _, n := range nodes { + if n != nil && !it.seen[n.ID()] { + it.seen[n.ID()] = true + it.result.push(n, bucketSize) + it.replyBuffer = append(it.replyBuffer, n) + } + } +} + func (it *lookup) shutdown() { for it.queries > 0 { <-it.replyCh @@ -109,20 +116,6 @@ func (it *lookup) startQueries() bool { return false } - // The first query returns nodes from the local table. - if it.queries == -1 { - closest := it.tab.findnodeByID(it.result.target, bucketSize, false) - // Avoid finishing the lookup too quickly if table is empty. It'd be better to wait - // for the table to fill in this case, but there is no good mechanism for that - // yet. - if len(closest.entries) == 0 { - it.slowdown() - } - it.queries = 1 - it.replyCh <- closest.entries - return true - } - // Ask the closest nodes that we haven't asked yet. for i := 0; i < len(it.result.entries) && it.queries < alpha; i++ { n := it.result.entries[i] @@ -136,73 +129,31 @@ func (it *lookup) startQueries() bool { return it.queries > 0 } -type ctxKey int - -const ( - ckNoSlowdown ctxKey = iota -) - -func disableLookupSlowdown(ctx context.Context) context.Context { - return context.WithValue(ctx, ckNoSlowdown, true) -} - -func isDisabledLookupSlowdown(ctx context.Context) bool { - return ctx.Value(ckNoSlowdown) != nil -} - -func (it *lookup) slowdown() { - if it.noSlowdown { - return - } - sleep := time.NewTimer(1 * time.Second) - defer sleep.Stop() - select { - case <-sleep.C: - case <-it.tab.closeReq: - } -} - -func (it *lookup) query(n *node, reply chan<- []*node) { - fails := it.tab.db.FindFails(n.ID(), n.IP()) +func (it *lookup) query(n *enode.Node, reply chan<- []*enode.Node) { r, err := it.queryfunc(n) - - if errors.Is(err, errClosed) { - // Avoid recording failures on shutdown. - reply <- nil - return - } else if len(r) == 0 { - fails++ - it.tab.db.UpdateFindFails(n.ID(), n.IP(), fails) - // Remove the node from the local table if it fails to return anything useful too - // many times, but only if there are enough other nodes in the bucket. - dropped := false - if fails >= maxFindnodeFailures && it.tab.bucketLen(n.ID()) >= bucketSize/2 { - dropped = true - it.tab.delete(n) + if !errors.Is(err, errClosed) { // avoid recording failures on shutdown. + success := len(r) > 0 + it.tab.trackRequest(n, success, r) + if err != nil { + it.tab.log.Trace("FINDNODE failed", "id", n.ID(), "err", err) } - it.tab.log.Trace("FINDNODE failed", "id", n.ID(), "failcount", fails, "dropped", dropped, "err", err) - } else if fails > 0 { - // Reset failure counter because it counts _consecutive_ failures. - it.tab.db.UpdateFindFails(n.ID(), n.IP(), 0) - } - - // Grab as many nodes as possible. Some of them might not be alive anymore, but we'll - // just remove those again during revalidation. - for _, n := range r { - it.tab.addSeenNode(n) } - reply <- r } // lookupIterator performs lookup operations and iterates over all seen nodes. // When a lookup finishes, a new one is created through nextLookup. +// LookupIterator waits for table initialization and triggers a table refresh +// when necessary. + type lookupIterator struct { - buffer []*node - nextLookup lookupFunc - ctx context.Context - cancel func() - lookup *lookup + buffer []*enode.Node + nextLookup lookupFunc + ctx context.Context + cancel func() + lookup *lookup + tabRefreshing <-chan struct{} + lastLookup time.Time } type lookupFunc func(ctx context.Context) *lookup @@ -217,7 +168,7 @@ func (it *lookupIterator) Node() *enode.Node { if len(it.buffer) == 0 { return nil } - return unwrapNode(it.buffer[0]) + return it.buffer[0] } // Next moves to the next node. @@ -226,6 +177,7 @@ func (it *lookupIterator) Next() bool { if len(it.buffer) > 0 { it.buffer = it.buffer[1:] } + // Advance the lookup to refill the buffer. for len(it.buffer) == 0 { if it.ctx.Err() != nil { @@ -234,18 +186,78 @@ func (it *lookupIterator) Next() bool { return false } if it.lookup == nil { + // Ensure enough time has passed between lookup creations. + it.slowdown() + it.lookup = it.nextLookup(it.ctx) + if it.lookup.empty() { + // If the lookup is empty right after creation, it means the local table + // is in a degraded state, and we need to wait for it to fill again. + it.lookupFailed(it.lookup.tab, 1*time.Minute) + it.lookup = nil + continue + } + // Yield the initial nodes from the iterator before advancing the lookup. + it.buffer = it.lookup.replyBuffer continue } - if !it.lookup.advance() { + + newNodes := it.lookup.advance() + it.buffer = it.lookup.replyBuffer + if !newNodes { it.lookup = nil - continue } - it.buffer = it.lookup.replyBuffer } return true } +// lookupFailed handles failed lookup attempts. This can be called when the table has +// exited, or when it runs out of nodes. +func (it *lookupIterator) lookupFailed(tab *Table, timeout time.Duration) { + tout, cancel := context.WithTimeout(it.ctx, timeout) + defer cancel() + + // Wait for Table initialization to complete, in case it is still in progress. + select { + case <-tab.initDone: + case <-tout.Done(): + return + } + + // Wait for ongoing refresh operation, or trigger one. + if it.tabRefreshing == nil { + it.tabRefreshing = tab.refresh() + } + select { + case <-it.tabRefreshing: + it.tabRefreshing = nil + case <-tout.Done(): + return + } + + // Wait for the table to fill. + tab.waitForNodes(tout, 1) +} + +// slowdown applies a delay between creating lookups. This exists to prevent hot-spinning +// in some test environments where lookups don't yield any results. +func (it *lookupIterator) slowdown() { + const minInterval = 1 * time.Second + + now := time.Now() + diff := now.Sub(it.lastLookup) + it.lastLookup = now + if diff > minInterval { + return + } + wait := time.NewTimer(diff) + defer wait.Stop() + select { + case <-wait.C: + case <-it.ctx.Done(): + } +} + // Close ends the iterator. func (it *lookupIterator) Close() { it.cancel() diff --git a/p2p/discover/lookup_util_test.go b/p2p/discover/lookup_util_test.go deleted file mode 100644 index b16f4d3bd98..00000000000 --- a/p2p/discover/lookup_util_test.go +++ /dev/null @@ -1,198 +0,0 @@ -// Copyright 2019 The go-ethereum Authors -// (original work) -// Copyright 2024 The Erigon Authors -// (modifications) -// This file is part of Erigon. -// -// Erigon is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// Erigon is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with Erigon. If not, see . - -package discover - -import ( - "crypto/ecdsa" - "net" - "slices" - "sort" - "testing" - - "github.com/erigontech/erigon/p2p/discover/v4wire" - "github.com/erigontech/erigon/p2p/enode" - "github.com/erigontech/erigon/p2p/enr" -) - -// This is the test network for the Lookup test. -// The nodes were obtained by running lookupTestnet.mine with a random NodeID as target. -var lookupTestnet = &preminedTestnet{ - target: hexEncPubkey("5d485bdcbe9bc89314a10ae9231e429d33853e3a8fa2af39f5f827370a2e4185e344ace5d16237491dad41f278f1d3785210d29ace76cd627b9147ee340b1125"), - dists: [257][]*ecdsa.PrivateKey{ - 251: { - hexEncPrivkey("29738ba0c1a4397d6a65f292eee07f02df8e58d41594ba2be3cf84ce0fc58169"), - hexEncPrivkey("511b1686e4e58a917f7f848e9bf5539d206a68f5ad6b54b552c2399fe7d174ae"), - hexEncPrivkey("d09e5eaeec0fd596236faed210e55ef45112409a5aa7f3276d26646080dcfaeb"), - hexEncPrivkey("c1e20dbbf0d530e50573bd0a260b32ec15eb9190032b4633d44834afc8afe578"), - hexEncPrivkey("ed5f38f5702d92d306143e5d9154fb21819777da39af325ea359f453d179e80b"), - }, - 252: { - hexEncPrivkey("1c9b1cafbec00848d2c174b858219914b42a7d5c9359b1ca03fd650e8239ae94"), - hexEncPrivkey("e0e1e8db4a6f13c1ffdd3e96b72fa7012293ced187c9dcdcb9ba2af37a46fa10"), - hexEncPrivkey("3d53823e0a0295cb09f3e11d16c1b44d07dd37cec6f739b8df3a590189fe9fb9"), - }, - 253: { - hexEncPrivkey("2d0511ae9bf590166597eeab86b6f27b1ab761761eaea8965487b162f8703847"), - hexEncPrivkey("6cfbd7b8503073fc3dbdb746a7c672571648d3bd15197ccf7f7fef3d904f53a2"), - hexEncPrivkey("a30599b12827b69120633f15b98a7f6bc9fc2e9a0fd6ae2ebb767c0e64d743ab"), - hexEncPrivkey("14a98db9b46a831d67eff29f3b85b1b485bb12ae9796aea98d91be3dc78d8a91"), - hexEncPrivkey("2369ff1fc1ff8ca7d20b17e2673adc3365c3674377f21c5d9dafaff21fe12e24"), - hexEncPrivkey("9ae91101d6b5048607f41ec0f690ef5d09507928aded2410aabd9237aa2727d7"), - hexEncPrivkey("05e3c59090a3fd1ae697c09c574a36fcf9bedd0afa8fe3946f21117319ca4973"), - hexEncPrivkey("06f31c5ea632658f718a91a1b1b9ae4b7549d7b3bc61cbc2be5f4a439039f3ad"), - }, - 254: { - hexEncPrivkey("dec742079ec00ff4ec1284d7905bc3de2366f67a0769431fd16f80fd68c58a7c"), - hexEncPrivkey("ff02c8861fa12fbd129d2a95ea663492ef9c1e51de19dcfbbfe1c59894a28d2b"), - hexEncPrivkey("4dded9e4eefcbce4262be4fd9e8a773670ab0b5f448f286ec97dfc8cf681444a"), - hexEncPrivkey("750d931e2a8baa2c9268cb46b7cd851f4198018bed22f4dceb09dd334a2395f6"), - hexEncPrivkey("ce1435a956a98ffec484cd11489c4f165cf1606819ab6b521cee440f0c677e9e"), - hexEncPrivkey("996e7f8d1638be92d7328b4770f47e5420fc4bafecb4324fd33b1f5d9f403a75"), - hexEncPrivkey("ebdc44e77a6cc0eb622e58cf3bb903c3da4c91ca75b447b0168505d8fc308b9c"), - hexEncPrivkey("46bd1eddcf6431bea66fc19ebc45df191c1c7d6ed552dcdc7392885009c322f0"), - }, - 255: { - hexEncPrivkey("da8645f90826e57228d9ea72aff84500060ad111a5d62e4af831ed8e4b5acfb8"), - hexEncPrivkey("3c944c5d9af51d4c1d43f5d0f3a1a7ef65d5e82744d669b58b5fed242941a566"), - hexEncPrivkey("5ebcde76f1d579eebf6e43b0ffe9157e65ffaa391175d5b9aa988f47df3e33da"), - hexEncPrivkey("97f78253a7d1d796e4eaabce721febcc4550dd68fb11cc818378ba807a2cb7de"), - hexEncPrivkey("a38cd7dc9b4079d1c0406afd0fdb1165c285f2c44f946eca96fc67772c988c7d"), - hexEncPrivkey("d64cbb3ffdf712c372b7a22a176308ef8f91861398d5dbaf326fd89c6eaeef1c"), - hexEncPrivkey("d269609743ef29d6446e3355ec647e38d919c82a4eb5837e442efd7f4218944f"), - hexEncPrivkey("d8f7bcc4a530efde1d143717007179e0d9ace405ddaaf151c4d863753b7fd64c"), - }, - 256: { - hexEncPrivkey("8c5b422155d33ea8e9d46f71d1ad3e7b24cb40051413ffa1a81cff613d243ba9"), - hexEncPrivkey("937b1af801def4e8f5a3a8bd225a8bcff1db764e41d3e177f2e9376e8dd87233"), - hexEncPrivkey("120260dce739b6f71f171da6f65bc361b5fad51db74cf02d3e973347819a6518"), - hexEncPrivkey("1fa56cf25d4b46c2bf94e82355aa631717b63190785ac6bae545a88aadc304a9"), - hexEncPrivkey("3c38c503c0376f9b4adcbe935d5f4b890391741c764f61b03cd4d0d42deae002"), - hexEncPrivkey("3a54af3e9fa162bc8623cdf3e5d9b70bf30ade1d54cc3abea8659aba6cff471f"), - hexEncPrivkey("6799a02ea1999aefdcbcc4d3ff9544478be7365a328d0d0f37c26bd95ade0cda"), - hexEncPrivkey("e24a7bc9051058f918646b0f6e3d16884b2a55a15553b89bab910d55ebc36116"), - }, - }, -} - -type preminedTestnet struct { - target enode.PubkeyEncoded - dists [hashBits + 1][]*ecdsa.PrivateKey -} - -func (tn *preminedTestnet) len() int { - n := 0 - for _, keys := range tn.dists { - n += len(keys) - } - return n -} - -func (tn *preminedTestnet) nodes() []*enode.Node { - result := make([]*enode.Node, 0, tn.len()) - for dist, keys := range tn.dists { - for index := range keys { - result = append(result, tn.node(dist, index)) - } - } - sortByID(result) - return result -} - -func (tn *preminedTestnet) node(dist, index int) *enode.Node { - key := tn.dists[dist][index] - rec := new(enr.Record) - rec.Set(enr.IP{127, byte(dist >> 8), byte(dist), byte(index)}) - rec.Set(enr.UDP(5000)) - enode.SignV4(rec, key) - n, _ := enode.New(enode.ValidSchemes, rec) - return n -} - -func (tn *preminedTestnet) nodeByAddr(addr *net.UDPAddr) (*enode.Node, *ecdsa.PrivateKey) { - dist := int(addr.IP[1])<<8 + int(addr.IP[2]) - index := int(addr.IP[3]) - key := tn.dists[dist][index] - return tn.node(dist, index), key -} - -func (tn *preminedTestnet) nodesAtDistance(dist int) []v4wire.Node { - result := make([]v4wire.Node, len(tn.dists[dist])) - for i := range result { - result[i] = nodeToRPC(wrapNode(tn.node(dist, i))) - } - return result -} - -func (tn *preminedTestnet) neighborsAtDistances(base *enode.Node, distances []uint, elems int) []*enode.Node { - var result []*enode.Node - for d := range lookupTestnet.dists { - for i := range lookupTestnet.dists[d] { - n := lookupTestnet.node(d, i) - d := enode.LogDist(base.ID(), n.ID()) - if slices.Contains(distances, uint(d)) { - result = append(result, n) - if len(result) >= elems { - return result - } - } - } - } - return result -} - -func (tn *preminedTestnet) closest(n int) (nodes []*enode.Node) { - for d := range tn.dists { - for i := range tn.dists[d] { - nodes = append(nodes, tn.node(d, i)) - } - } - sort.Slice(nodes, func(i, j int) bool { - return enode.DistCmp(tn.target.ID(), nodes[i].ID(), nodes[j].ID()) < 0 - }) - return nodes[:n] -} - -func (tn *preminedTestnet) privateKeys() []*ecdsa.PrivateKey { - var keys []*ecdsa.PrivateKey - for d := range tn.dists { - keys = append(keys, tn.dists[d]...) - } - return keys -} - -// checkLookupResults verifies that the results of a lookup are the closest nodes to -// the testnet's target. -func checkLookupResults(t *testing.T, tn *preminedTestnet, results []*enode.Node) { - t.Helper() - t.Logf("results:") - for _, e := range results { - t.Logf(" ld=%d, %x", enode.LogDist(tn.target.ID(), e.ID()), e.ID().Bytes()) - } - if hasDuplicates(wrapNodes(results)) { - t.Errorf("result set contains duplicate entries") - } - if !sortedByDistanceTo(tn.target.ID(), wrapNodes(results)) { - t.Errorf("result set not sorted by distance to target") - } - wantNodes := tn.closest(len(results)) - if err := checkNodesEqual(results, wantNodes); err != nil { - t.Error(err) - } -} diff --git a/p2p/discover/metrics.go b/p2p/discover/metrics.go new file mode 100644 index 00000000000..ed802430ea0 --- /dev/null +++ b/p2p/discover/metrics.go @@ -0,0 +1,82 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package discover + +import ( + "fmt" + "net" + "net/netip" + + "github.com/erigontech/erigon/common/metrics" +) + +const ( + moduleName = "discover" + // ingressMeterName is the prefix of the per-packet inbound metrics. + ingressMeterName = moduleName + "/ingress" + + // egressMeterName is the prefix of the per-packet outbound metrics. + egressMeterName = moduleName + "/egress" +) + +var ( + bucketsCounter []*metrics.Counter + ingressTrafficMeter = metrics.NewRegisteredMeter(ingressMeterName, nil) + egressTrafficMeter = metrics.NewRegisteredMeter(egressMeterName, nil) +) + +func init() { + for i := 0; i < nBuckets; i++ { + bucketsCounter = append(bucketsCounter, metrics.NewRegisteredCounter(fmt.Sprintf("%s/bucket/%d/count", moduleName, i), nil)) + } +} + +// meteredUdpConn is a wrapper around a net.UDPConn that meters both the +// inbound and outbound network traffic. +type meteredUdpConn struct { + udpConn UDPConn +} + +func newMeteredConn(conn UDPConn) UDPConn { + // Short circuit if metrics are disabled + if !metrics.Enabled() { + return conn + } + return &meteredUdpConn{udpConn: conn} +} + +func (c *meteredUdpConn) Close() error { + return c.udpConn.Close() +} + +func (c *meteredUdpConn) LocalAddr() net.Addr { + return c.udpConn.LocalAddr() +} + +// ReadFromUDPAddrPort delegates a network read to the underlying connection, bumping the udp ingress traffic meter along the way. +func (c *meteredUdpConn) ReadFromUDPAddrPort(b []byte) (n int, addr netip.AddrPort, err error) { + n, addr, err = c.udpConn.ReadFromUDPAddrPort(b) + ingressTrafficMeter.Mark(int64(n)) + return n, addr, err +} + +// WriteToUDPAddrPort delegates a network write to the underlying connection, bumping the udp egress traffic meter along the way. +func (c *meteredUdpConn) WriteToUDPAddrPort(b []byte, addr netip.AddrPort) (n int, err error) { + n, err = c.udpConn.WriteToUDPAddrPort(b, addr) + egressTrafficMeter.Mark(int64(n)) + return n, err +} diff --git a/p2p/discover/node.go b/p2p/discover/node.go index fc2ba511e41..7a3f59cf42c 100644 --- a/p2p/discover/node.go +++ b/p2p/discover/node.go @@ -1,67 +1,100 @@ // Copyright 2015 The go-ethereum Authors -// (original work) -// Copyright 2024 The Erigon Authors -// (modifications) -// This file is part of Erigon. +// This file is part of the go-ethereum library. // -// Erigon is free software: you can redistribute it and/or modify +// The go-ethereum library is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // -// Erigon is distributed in the hope that it will be useful, +// The go-ethereum library is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Lesser General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License -// along with Erigon. If not, see . +// along with the go-ethereum library. If not, see . package discover import ( - "net" + "slices" + "sort" "time" "github.com/erigontech/erigon/p2p/enode" ) -// node represents a host on the network. -// The fields of Node may not be modified. -type node struct { - enode.Node - addedAt time.Time // time when the node was added to the table - livenessChecks uint // how often liveness was checked +type BucketNode struct { + Node *enode.Node `json:"node"` + AddedToTable time.Time `json:"addedToTable"` + AddedToBucket time.Time `json:"addedToBucket"` + Checks int `json:"checks"` + Live bool `json:"live"` } -func wrapNode(n *enode.Node) *node { - return &node{Node: *n} +// tableNode is an entry in Table. +type tableNode struct { + *enode.Node + revalList *revalidationList + addedToTable time.Time // first time node was added to bucket or replacement list + addedToBucket time.Time // time it was added in the actual bucket + livenessChecks uint // how often liveness was checked + isValidatedLive bool // true if existence of node is considered validated right now } -func wrapNodes(ns []*enode.Node) []*node { - result := make([]*node, len(ns)) +func unwrapNodes(ns []*tableNode) []*enode.Node { + result := make([]*enode.Node, len(ns)) for i, n := range ns { - result[i] = wrapNode(n) + result[i] = n.Node } return result } -func unwrapNode(n *node) *enode.Node { - return &n.Node +func (n *tableNode) String() string { + return n.Node.String() } -func unwrapNodes(ns []*node) []*enode.Node { - result := make([]*enode.Node, len(ns)) - for i, n := range ns { - result[i] = unwrapNode(n) +// nodesByDistance is a list of nodes, ordered by distance to target. +type nodesByDistance struct { + entries []*enode.Node + target enode.ID +} + +// push adds the given node to the list, keeping the total size below maxElems. +func (h *nodesByDistance) push(n *enode.Node, maxElems int) { + ix := sort.Search(len(h.entries), func(i int) bool { + return enode.DistCmp(h.target, h.entries[i].ID(), n.ID()) > 0 + }) + + end := len(h.entries) + if len(h.entries) < maxElems { + h.entries = append(h.entries, n) + } + if ix < end { + // Slide existing entries down to make room. + // This will overwrite the entry we just appended. + copy(h.entries[ix+1:], h.entries[ix:]) + h.entries[ix] = n } - return result } -func (n *node) addr() *net.UDPAddr { - return &net.UDPAddr{IP: n.IP(), Port: n.UDP()} +type nodeType interface { + ID() enode.ID } -func (n *node) String() string { - return n.Node.String() +// containsID reports whether ns contains a node with the given ID. +func containsID[N nodeType](ns []N, id enode.ID) bool { + for _, n := range ns { + if n.ID() == id { + return true + } + } + return false +} + +// deleteNode removes a node from the list. +func deleteNode[N nodeType](list []N, id enode.ID) []N { + return slices.DeleteFunc(list, func(n N) bool { + return n.ID() == id + }) } diff --git a/p2p/discover/ntp.go b/p2p/discover/ntp.go index 961d29a8d55..d8b4f184688 100644 --- a/p2p/discover/ntp.go +++ b/p2p/discover/ntp.go @@ -1,21 +1,18 @@ // Copyright 2016 The go-ethereum Authors -// (original work) -// Copyright 2024 The Erigon Authors -// (modifications) -// This file is part of Erigon. +// This file is part of the go-ethereum library. // -// Erigon is free software: you can redistribute it and/or modify +// The go-ethereum library is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // -// Erigon is distributed in the hope that it will be useful, +// The go-ethereum library is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Lesser General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License -// along with Erigon. If not, see . +// along with the go-ethereum library. If not, see . // Contains the NTP time drift detection via the SNTP protocol: // https://tools.ietf.org/html/rfc4330 @@ -25,10 +22,9 @@ package discover import ( "fmt" "net" - "sort" + "slices" "time" - "github.com/erigontech/erigon/common/dbg" "github.com/erigontech/erigon/common/log/v3" ) @@ -37,18 +33,9 @@ const ( ntpChecks = 3 // Number of measurements to do against the NTP server ) -// durationSlice attaches the methods of sort.Interface to []time.Duration, -// sorting in increasing order. -type durationSlice []time.Duration - -func (s durationSlice) Len() int { return len(s) } -func (s durationSlice) Less(i, j int) bool { return s[i] < s[j] } -func (s durationSlice) Swap(i, j int) { s[i], s[j] = s[j], s[i] } - // checkClockDrift queries an NTP server for clock drifts and warns the user if // one large enough is detected. func checkClockDrift() { - defer dbg.LogPanic() drift, err := sntpDrift(ntpChecks) if err != nil { return @@ -57,7 +44,7 @@ func checkClockDrift() { log.Warn(fmt.Sprintf("System clock seems off by %v, which can prevent network connectivity", drift)) log.Warn("Please enable network time synchronisation in system settings.") } else { - log.Trace("NTP sanity check done", "drift", drift) + log.Debug("NTP sanity check done", "drift", drift) } } @@ -108,13 +95,13 @@ func sntpDrift(measurements int) (time.Duration, error) { nanosec := sec*1e9 + (frac*1e9)>>32 - t := time.Date(1900, 1, 1, 0, 0, 0, 0, time.UTC).Add(time.Duration(nanosec)).Local() //nolint:gosmopolitan + t := time.Date(1900, 1, 1, 0, 0, 0, 0, time.UTC).Add(time.Duration(nanosec)).Local() // Calculate the drift based on an assumed answer time of RRT/2 drifts = append(drifts, sent.Sub(t)+elapsed/2) } - // Calculate average drif (drop two extremities to avoid outliers) - sort.Sort(durationSlice(drifts)) + // Calculate average drift (drop two extremities to avoid outliers) + slices.Sort(drifts) drift := time.Duration(0) for i := 1; i < len(drifts)-1; i++ { diff --git a/p2p/discover/table.go b/p2p/discover/table.go index a66849a639f..feaa752649d 100644 --- a/p2p/discover/table.go +++ b/p2p/discover/table.go @@ -1,21 +1,18 @@ // Copyright 2015 The go-ethereum Authors -// (original work) -// Copyright 2024 The Erigon Authors -// (modifications) -// This file is part of Erigon. +// This file is part of the go-ethereum library. // -// Erigon is free software: you can redistribute it and/or modify +// The go-ethereum library is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // -// Erigon is distributed in the hope that it will be useful, +// The go-ethereum library is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Lesser General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License -// along with Erigon. If not, see . +// along with the go-ethereum library. If not, see . // Package discover implements the Node Discovery Protocol. // @@ -26,20 +23,19 @@ package discover import ( - crand "crypto/rand" - "encoding/binary" + "context" "fmt" - mrand "math/rand" - "net" - "sort" + "net/netip" + "slices" "sync" - "sync/atomic" "time" "github.com/erigontech/erigon/common" - "github.com/erigontech/erigon/common/dbg" "github.com/erigontech/erigon/common/log/v3" + "github.com/erigontech/erigon/common/mclock" + "github.com/erigontech/erigon/common/metrics" "github.com/erigontech/erigon/p2p/enode" + "github.com/erigontech/erigon/p2p/event" "github.com/erigontech/erigon/p2p/netutil" ) @@ -58,42 +54,39 @@ const ( bucketIPLimit, bucketSubnet = 2, 24 // at most 2 addresses from the same /24 tableIPLimit, tableSubnet = 10, 24 - minRefreshInterval = 30 * time.Second - refreshInterval = 30 * time.Minute - revalidateInterval = 5 * time.Second - maintenanceInterval = 60 * time.Second - seedMinTableTime = 5 * time.Minute - seedCount = 30 - seedMaxAge = 5 * 24 * time.Hour + seedCount = 30 + seedMaxAge = 5 * 24 * time.Hour ) // Table is the 'node table', a Kademlia-like index of neighbor nodes. The table keeps // itself up-to-date by verifying the liveness of neighbors and requesting their node // records when announcements of a new record version are received. type Table struct { - mutex sync.Mutex // protects buckets, bucket content, nursery, rand - buckets [nBuckets]*bucket // index of known nodes by distance - nursery []*node // bootstrap nodes - rand *mrand.Rand // source of randomness, periodically reseeded - ips netutil.DistinctNetSet - - revalidateInterval time.Duration + mutex sync.Mutex // protects buckets, bucket content, nursery, rand + buckets [nBuckets]*bucket // index of known nodes by distance + nursery []*enode.Node // bootstrap nodes + rand reseedingRandom // source of randomness, periodically reseeded + ips netutil.DistinctNetSet + revalidation tableRevalidation - log log.Logger - db *enode.DB // database of known nodes - net transport - refreshReq chan chan struct{} - initDone chan struct{} - closeReq chan struct{} - closed chan struct{} + db *enode.DB // database of known nodes + net transport + cfg Config + log log.Logger - nodeAddedHook func(*node) // for testing + // loop channels + refreshReq chan chan struct{} + revalResponseCh chan revalidationResponse + addNodeCh chan addNodeOp + addNodeHandled chan bool + trackRequestCh chan trackRequestOp + initDone chan struct{} + closeReq chan struct{} + closed chan struct{} - // diagnostics - errors map[string]uint - dbseeds atomic.Int32 - revalidates atomic.Int32 - protocol string + nodeFeed event.FeedOf[*enode.Node] + nodeAddedHook func(*bucket, *tableNode) + nodeRemovedHook func(*bucket, *tableNode) } // transport is implemented by the UDP transports. @@ -103,92 +96,87 @@ type transport interface { lookupRandom() []*enode.Node lookupSelf() []*enode.Node ping(*enode.Node) (seq uint64, err error) - Version() string - Errors() map[string]uint - LenUnsolicited() int } // bucket contains nodes, ordered by their last activity. the entry // that was most recently active is the first element in entries. type bucket struct { - entries []*node // live entries, sorted by time of last contact - replacements []*node // recently seen nodes to be used if revalidation fails + entries []*tableNode // live entries, sorted by time of last contact + replacements []*tableNode // recently seen nodes to be used if revalidation fails ips netutil.DistinctNetSet + index int +} + +type addNodeOp struct { + node *enode.Node + isInbound bool + forceSetLive bool // for tests +} + +type trackRequestOp struct { + node *enode.Node + foundNodes []*enode.Node + success bool } -func newTable( - t transport, - protocol string, - db *enode.DB, - bootnodes []*enode.Node, - revalidateInterval time.Duration, - logger log.Logger, -) (*Table, error) { +func newTable(t transport, db *enode.DB, cfg Config) (*Table, error) { + cfg = cfg.withDefaults() tab := &Table{ - net: t, - db: db, - refreshReq: make(chan chan struct{}), - initDone: make(chan struct{}), - closeReq: make(chan struct{}), - closed: make(chan struct{}), - rand: mrand.New(mrand.NewSource(0)), // nolint: gosec - ips: netutil.DistinctNetSet{Subnet: tableSubnet, Limit: tableIPLimit}, - errors: map[string]uint{}, - revalidateInterval: revalidateInterval, - protocol: protocol, - log: logger, - } - if err := tab.setFallbackNodes(bootnodes); err != nil { - return nil, err + net: t, + db: db, + cfg: cfg, + log: cfg.Log, + refreshReq: make(chan chan struct{}), + revalResponseCh: make(chan revalidationResponse), + addNodeCh: make(chan addNodeOp), + addNodeHandled: make(chan bool), + trackRequestCh: make(chan trackRequestOp), + initDone: make(chan struct{}), + closeReq: make(chan struct{}), + closed: make(chan struct{}), + ips: netutil.DistinctNetSet{Subnet: tableSubnet, Limit: tableIPLimit}, } for i := range tab.buckets { tab.buckets[i] = &bucket{ - ips: netutil.DistinctNetSet{Subnet: bucketSubnet, Limit: bucketIPLimit}, + index: i, + ips: netutil.DistinctNetSet{Subnet: bucketSubnet, Limit: bucketIPLimit}, } } - tab.seedRand() - tab.loadSeedNodes() + tab.rand.seed() + tab.revalidation.init(&cfg) - return tab, nil -} - -func (tab *Table) self() *enode.Node { - return tab.net.Self() -} - -func (tab *Table) seedRand() { - var b [8]byte - if _, err := crand.Read(b[:]); err != nil { - panic("crypto/rand failed in seedRand: " + err.Error()) + // initial table content + if err := tab.setFallbackNodes(cfg.Bootnodes); err != nil { + return nil, err } + tab.loadSeedNodes() - tab.mutex.Lock() - defer tab.mutex.Unlock() - tab.rand.Seed(int64(binary.BigEndian.Uint64(b[:]))) + return tab, nil } -// ReadRandomNodes fills the given slice with random nodes from the table. The results -// are guaranteed to be unique for a single invocation, no node will appear twice. -func (tab *Table) ReadRandomNodes(buf []*enode.Node) (n int) { - if !tab.isInitDone() { - return 0 - } - +// Nodes returns all nodes contained in the table. +func (tab *Table) Nodes() [][]BucketNode { tab.mutex.Lock() defer tab.mutex.Unlock() - nodes := make([]*enode.Node, 0, len(&tab.buckets)) - for _, b := range &tab.buckets { - for _, n := range b.entries { - nodes = append(nodes, unwrapNode(n)) + nodes := make([][]BucketNode, len(tab.buckets)) + for i, b := range &tab.buckets { + nodes[i] = make([]BucketNode, len(b.entries)) + for j, n := range b.entries { + nodes[i][j] = BucketNode{ + Node: n.Node, + Checks: int(n.livenessChecks), + Live: n.isValidatedLive, + AddedToTable: n.addedToTable, + AddedToBucket: n.addedToBucket, + } } } - // Shuffle. - for i := 0; i < len(nodes); i++ { - j := tab.rand.Intn(len(nodes)) - nodes[i], nodes[j] = nodes[j], nodes[i] - } - return copy(buf, nodes) + return nodes +} + +func (tab *Table) self() *enode.Node { + return tab.net.Self() } // getNode returns the node with the given ID or nil if it isn't in the table. @@ -199,7 +187,7 @@ func (tab *Table) getNode(id enode.ID) *enode.Node { b := tab.bucket(id) for _, e := range b.entries { if e.ID() == id { - return unwrapNode(e) + return e.Node } } return nil @@ -215,12 +203,18 @@ func (tab *Table) close() { // are used to connect to the network if the table is empty and there // are no known nodes in the database. func (tab *Table) setFallbackNodes(nodes []*enode.Node) error { + nursery := make([]*enode.Node, 0, len(nodes)) for _, n := range nodes { if err := n.ValidateComplete(); err != nil { - return fmt.Errorf("bad bootstrap node %q: %w", n, err) + return fmt.Errorf("bad bootstrap node %q: %v", n, err) + } + if tab.cfg.NetRestrict != nil && !tab.cfg.NetRestrict.ContainsAddr(n.IPAddr()) { + tab.log.Error("Bootstrap node filtered by netrestrict", "id", n.ID(), "ip", n.IPAddr()) + continue } + nursery = append(nursery, n) } - tab.nursery = wrapNodes(nodes) + tab.nursery = nursery return nil } @@ -244,97 +238,173 @@ func (tab *Table) refresh() <-chan struct{} { return done } -// loop schedules runs of doRefresh, doRevalidate and copyLiveNodes. +// findnodeByID returns the n nodes in the table that are closest to the given id. +// This is used by the FINDNODE/v4 handler. +// +// The preferLive parameter says whether the caller wants liveness-checked results. If +// preferLive is true and the table contains any verified nodes, the result will not +// contain unverified nodes. However, if there are no verified nodes at all, the result +// will contain unverified nodes. +func (tab *Table) findnodeByID(target enode.ID, nresults int, preferLive bool) *nodesByDistance { + tab.mutex.Lock() + defer tab.mutex.Unlock() + + // Scan all buckets. There might be a better way to do this, but there aren't that many + // buckets, so this solution should be fine. The worst-case complexity of this loop + // is O(tab.len() * nresults). + nodes := &nodesByDistance{target: target} + liveNodes := &nodesByDistance{target: target} + for _, b := range &tab.buckets { + for _, n := range b.entries { + nodes.push(n.Node, nresults) + if preferLive && n.isValidatedLive { + liveNodes.push(n.Node, nresults) + } + } + } + + if preferLive && len(liveNodes.entries) > 0 { + return liveNodes + } + return nodes +} + +// appendBucketNodes adds nodes at the given distance to the result slice. +// This is used by the FINDNODE/v5 handler. +func (tab *Table) appendBucketNodes(dist uint, result []*enode.Node, checkLive bool) []*enode.Node { + if dist > 256 { + return result + } + if dist == 0 { + return append(result, tab.self()) + } + + tab.mutex.Lock() + for _, n := range tab.bucketAtDistance(int(dist)).entries { + if !checkLive || n.isValidatedLive { + result = append(result, n.Node) + } + } + tab.mutex.Unlock() + + // Shuffle result to avoid always returning same nodes in FINDNODE/v5. + tab.rand.Shuffle(len(result), func(i, j int) { + result[i], result[j] = result[j], result[i] + }) + return result +} + +// len returns the number of nodes in the table. +func (tab *Table) len() (n int) { + tab.mutex.Lock() + defer tab.mutex.Unlock() + + for _, b := range &tab.buckets { + n += len(b.entries) + } + return n +} + +// addFoundNode adds a node which may not be live. If the bucket has space available, +// adding the node succeeds immediately. Otherwise, the node is added to the replacements +// list. +// +// The caller must not hold tab.mutex. +func (tab *Table) addFoundNode(n *enode.Node, forceSetLive bool) bool { + op := addNodeOp{node: n, isInbound: false, forceSetLive: forceSetLive} + select { + case tab.addNodeCh <- op: + return <-tab.addNodeHandled + case <-tab.closeReq: + return false + } +} + +// addInboundNode adds a node from an inbound contact. If the bucket has no space, the +// node is added to the replacements list. +// +// There is an additional safety measure: if the table is still initializing the node is +// not added. This prevents an attack where the table could be filled by just sending ping +// repeatedly. +// +// The caller must not hold tab.mutex. +func (tab *Table) addInboundNode(n *enode.Node) bool { + op := addNodeOp{node: n, isInbound: true} + select { + case tab.addNodeCh <- op: + return <-tab.addNodeHandled + case <-tab.closeReq: + return false + } +} + +func (tab *Table) trackRequest(n *enode.Node, success bool, foundNodes []*enode.Node) { + op := trackRequestOp{n, foundNodes, success} + select { + case tab.trackRequestCh <- op: + case <-tab.closeReq: + } +} + +// loop is the main loop of Table. func (tab *Table) loop() { var ( - revalidate = time.NewTimer(tab.revalidateInterval) - refresh = time.NewTicker(refreshInterval) - tableMainenance = time.NewTicker(maintenanceInterval) + refresh = time.NewTimer(tab.nextRefreshTime()) refreshDone = make(chan struct{}) // where doRefresh reports completion - revalidateDone chan struct{} // where doRevalidate reports completion waiting = []chan struct{}{tab.initDone} // holds waiting callers while doRefresh runs + revalTimer = mclock.NewAlarm(tab.cfg.Clock) + reseedRandTimer = time.NewTicker(10 * time.Minute) ) - defer dbg.LogPanic() defer refresh.Stop() - defer revalidate.Stop() - defer tableMainenance.Stop() + defer revalTimer.Stop() + defer reseedRandTimer.Stop() // Start initial refresh. go tab.doRefresh(refreshDone) - var minRefreshTimer *time.Timer - var minRefreshTimerActive atomic.Bool - - defer func() { - if minRefreshTimer != nil { - minRefreshTimer.Stop() - } - }() - loop: for { + nextTime := tab.revalidation.run(tab, tab.cfg.Clock.Now()) + revalTimer.Schedule(nextTime) + select { + case <-reseedRandTimer.C: + tab.rand.seed() + + case <-revalTimer.C(): + + case r := <-tab.revalResponseCh: + tab.revalidation.handleResponse(tab, r) + + case op := <-tab.addNodeCh: + tab.mutex.Lock() + ok := tab.handleAddNode(op) + tab.mutex.Unlock() + tab.addNodeHandled <- ok + + case op := <-tab.trackRequestCh: + tab.handleTrackRequest(op) + case <-refresh.C: - tab.seedRand() if refreshDone == nil { refreshDone = make(chan struct{}) go tab.doRefresh(refreshDone) } + case req := <-tab.refreshReq: waiting = append(waiting, req) if refreshDone == nil { refreshDone = make(chan struct{}) go tab.doRefresh(refreshDone) } + case <-refreshDone: for _, ch := range waiting { close(ch) } waiting, refreshDone = nil, nil - case <-revalidate.C: - if revalidateDone == nil { - revalidateDone = make(chan struct{}) - go tab.doRevalidate(revalidateDone) - } - case <-revalidateDone: - revalidate.Reset(tab.revalidateInterval) - if tab.live() == 0 && len(waiting) == 0 && minRefreshTimerActive.CompareAndSwap(false, true) { - minRefreshTimer = time.AfterFunc(minRefreshInterval, func() { - defer minRefreshTimerActive.Store(false) - tab.net.lookupRandom() - tab.refresh() - }) - } - revalidateDone = nil - case <-tableMainenance.C: - live := tab.live() - - vals := []interface{}{"protocol", tab.protocol, "version", tab.net.Version(), - "len", tab.len(), "live", tab.live(), "unsol", tab.net.LenUnsolicited(), "ips", tab.ips.Len(), "db", tab.dbseeds.Load(), "reval", tab.revalidates.Load()} - - func() { - tab.mutex.Lock() - defer tab.mutex.Unlock() - - for err, count := range tab.errors { - vals = append(vals, err, count) - } - - for err, count := range tab.net.Errors() { - vals = append(vals, err, count) - } - }() - - tab.log.Debug("[p2p] Discovery table", vals...) - - if live != 0 { - if revalidateDone == nil { - revalidateDone = make(chan struct{}) - go tab.doRevalidate(revalidateDone) - } - } else { - go tab.copyLiveNodes() - } + refresh.Reset(tab.nextRefreshTime()) + case <-tab.closeReq: break loop } @@ -346,16 +416,12 @@ loop: for _, ch := range waiting { close(ch) } - if revalidateDone != nil { - <-revalidateDone - } close(tab.closed) } // doRefresh performs a lookup for a random target to keep buckets full. seed nodes are // inserted if the table is empty (initial bootstrap or discarded faulty peers). func (tab *Table) doRefresh(done chan struct{}) { - defer dbg.LogPanic() defer close(done) // Load nodes from the database and insert @@ -378,175 +444,24 @@ func (tab *Table) doRefresh(done chan struct{}) { } func (tab *Table) loadSeedNodes() { - dbseeds := tab.db.QuerySeeds(seedCount, seedMaxAge) - tab.dbseeds.Store(int32(len(dbseeds))) - - seeds := wrapNodes(dbseeds) - tab.log.Debug("QuerySeeds read nodes from the node DB", "count", len(seeds)) + seeds := tab.db.QuerySeeds(seedCount, seedMaxAge) seeds = append(seeds, tab.nursery...) for i := range seeds { seed := seeds[i] - age := log.Lazy{Fn: func() interface{} { return time.Since(tab.db.LastPongReceived(seed.ID(), seed.IP())) }} - tab.log.Trace("Found seed node in database", "id", seed.ID(), "addr", seed.addr(), "age", age) - tab.addSeenNode(seed) - } -} - -// doRevalidate checks that the last node in a random bucket is still live and replaces or -// deletes the node if it isn't. -func (tab *Table) doRevalidate(done chan<- struct{}) { - defer dbg.LogPanic() - defer func() { done <- struct{}{} }() - - tab.revalidates.Add(1) - - last, bi := tab.nodeToRevalidate() - if last == nil { - // No non-empty bucket found. - return - } - - // Ping the selected node and wait for a pong. - remoteSeq, rErr := tab.net.ping(unwrapNode(last)) - - // Also fetch record if the node replied and returned a higher sequence number. - if rErr == nil { - if last.Seq() < remoteSeq { - if n, err := tab.net.RequestENR(unwrapNode(last)); err != nil { - rErr = err - tab.log.Trace("ENR request failed", "id", last.ID(), "addr", last.addr(), "err", err) - } else { - last = &node{Node: *n, addedAt: last.addedAt, livenessChecks: last.livenessChecks} - } + if tab.log.Enabled(context.Background(), log.LvlTrace) { + age := time.Since(tab.db.LastPongReceived(seed.ID(), seed.IPAddr())) + addr, _ := seed.UDPEndpoint() + tab.log.Trace("Found seed node in database", "id", seed.ID(), "addr", addr, "age", age) } - } - - tab.mutex.Lock() - defer tab.mutex.Unlock() - b := tab.buckets[bi] - if rErr == nil { - // The node responded, move it to the front. - last.livenessChecks++ - tab.log.Trace("Revalidated node", "b", bi, "id", last.ID(), "checks", last.livenessChecks) - tab.bumpInBucket(b, last) - return - } else { - tab.addError(rErr) - } - - // No reply received, pick a replacement or delete the node if there aren't - // any replacements. - if r := tab.replace(b, last); r != nil { - tab.log.Trace("Replaced dead node", "b", bi, "id", last.ID(), "ip", last.IP(), "checks", last.livenessChecks, "r", r.ID(), "rip", r.IP()) - } else { - tab.log.Trace("Removed dead node", "b", bi, "id", last.ID(), "ip", last.IP(), "checks", last.livenessChecks) + tab.mutex.Lock() + tab.handleAddNode(addNodeOp{node: seed, isInbound: false}) + tab.mutex.Unlock() } } -// nodeToRevalidate returns the last node in a random, non-empty bucket. -func (tab *Table) nodeToRevalidate() (n *node, bi int) { - tab.mutex.Lock() - defer tab.mutex.Unlock() - - for _, bi = range tab.rand.Perm(len(tab.buckets)) { - b := tab.buckets[bi] - if len(b.entries) > 0 { - last := b.entries[len(b.entries)-1] - return last, bi - } - } - return nil, 0 -} - -// copyLiveNodes adds nodes from the table to the database if they have been in the table -// longer than seedMinTableTime. -func (tab *Table) copyLiveNodes() { - tab.mutex.Lock() - defer dbg.LogPanic() - defer tab.mutex.Unlock() - - now := time.Now() - for _, b := range &tab.buckets { - for _, n := range b.entries { - if n.livenessChecks > 0 && now.Sub(n.addedAt) >= seedMinTableTime { - tab.db.UpdateNode(unwrapNode(n)) - } - } - } -} - -// findnodeByID returns the n nodes in the table that are closest to the given id. -// This is used by the FINDNODE/v4 handler. -// -// The preferLive parameter says whether the caller wants liveness-checked results. If -// preferLive is true and the table contains any verified nodes, the result will not -// contain unverified nodes. However, if there are no verified nodes at all, the result -// will contain unverified nodes. -func (tab *Table) findnodeByID(target enode.ID, nresults int, preferLive bool) (nodes nodesByDistance) { - nodes.target = target - tab.mutex.Lock() - defer tab.mutex.Unlock() - - // Scan all buckets. There might be a better way to do this, but there aren't that many - // buckets, so this solution should be fine. The worst-case complexity of this loop - // is O(tab.len() * nresults). - if preferLive { - for _, b := range &tab.buckets { - for _, n := range b.entries { - if n.livenessChecks > 0 { - nodes.push(n, nresults) - } - } - } - if len(nodes.entries) > 0 { - return - } - } - for _, b := range &tab.buckets { - for _, n := range b.entries { - nodes.push(n, nresults) - } - } - return -} - -// len returns the number of nodes in the table. -func (tab *Table) len() (n int) { - tab.mutex.Lock() - defer tab.mutex.Unlock() - - for _, b := range &tab.buckets { - n += len(b.entries) - } - return n -} - -func (tab *Table) live() (n int) { - tab.mutex.Lock() - defer tab.mutex.Unlock() - - for _, b := range &tab.buckets { - for _, e := range b.entries { - if e.livenessChecks > 0 { - n++ - } - } - } - - return n -} - -func (tab *Table) addError(err error) { - str := err.Error() - tab.errors[str] = tab.errors[str] + 1 -} - -// bucketLen returns the number of nodes in the bucket for the given ID. -func (tab *Table) bucketLen(id enode.ID) int { - tab.mutex.Lock() - defer tab.mutex.Unlock() - - return len(tab.bucket(id).entries) +func (tab *Table) nextRefreshTime() time.Duration { + half := tab.cfg.RefreshInterval / 2 + return half + time.Duration(tab.rand.Int63n(int64(half))) } // bucket returns the bucket for the given node ID hash. @@ -562,196 +477,219 @@ func (tab *Table) bucketAtDistance(d int) *bucket { return tab.buckets[d-bucketMinDistance-1] } -// addSeenNode adds a node which may or may not be live to the end of a bucket. If the -// bucket has space available, adding the node succeeds immediately. Otherwise, the node is -// added to the replacements list. -// -// The caller must not hold tab.mutex. -func (tab *Table) addSeenNode(n *node) { - if n.ID() == tab.self().ID() { - return - } - - tab.mutex.Lock() - defer tab.mutex.Unlock() - - b := tab.bucket(n.ID()) - if contains(b.entries, n.ID()) { - // Already in bucket, don't add. - return +func (tab *Table) addIP(b *bucket, ip netip.Addr) bool { + if !ip.IsValid() || ip.IsUnspecified() { + return false // Nodes without IP cannot be added. } - if len(b.entries) >= bucketSize { - // Bucket full, maybe add as replacement. - tab.addReplacement(b, n) - return + if netutil.AddrIsLAN(ip) { + return true } - if !tab.addIP(b, n.IP()) { - // Can't add: IP limit reached. - return + if !tab.ips.AddAddr(ip) { + tab.log.Debug("IP exceeds table limit", "ip", ip) + return false } - // Add to end of bucket: - b.entries = append(b.entries, n) - b.replacements = deleteNode(b.replacements, n) - n.addedAt = time.Now() - if tab.nodeAddedHook != nil { - tab.nodeAddedHook(n) + if !b.ips.AddAddr(ip) { + tab.log.Debug("IP exceeds bucket limit", "ip", ip) + tab.ips.RemoveAddr(ip) + return false } + return true } -// addVerifiedNode adds a node whose existence has been verified recently to the front of a -// bucket. If the node is already in the bucket, it is moved to the front. If the bucket -// has no space, the node is added to the replacements list. -// -// There is an additional safety measure: if the table is still initializing the node -// is not added. This prevents an attack where the table could be filled by just sending -// ping repeatedly. -// -// The caller must not hold tab.mutex. -func (tab *Table) addVerifiedNode(n *node) { - if !tab.isInitDone() { - return - } - if n.ID() == tab.self().ID() { +func (tab *Table) removeIP(b *bucket, ip netip.Addr) { + if netutil.AddrIsLAN(ip) { return } + tab.ips.RemoveAddr(ip) + b.ips.RemoveAddr(ip) +} - tab.mutex.Lock() - defer tab.mutex.Unlock() +// handleAddNode adds the node in the request to the table, if there is space. +// The caller must hold tab.mutex. +func (tab *Table) handleAddNode(req addNodeOp) bool { + if req.node.ID() == tab.self().ID() { + return false + } + // For nodes from inbound contact, there is an additional safety measure: if the table + // is still initializing the node is not added. + if req.isInbound && !tab.isInitDone() { + return false + } - b := tab.bucket(n.ID()) - if tab.bumpInBucket(b, n) { - // Already in bucket, moved to front. - return + b := tab.bucket(req.node.ID()) + n, _ := tab.bumpInBucket(b, req.node, req.isInbound) + if n != nil { + // Already in bucket. + return false } if len(b.entries) >= bucketSize { // Bucket full, maybe add as replacement. - tab.addReplacement(b, n) - return + tab.addReplacement(b, req.node) + return false } - if !tab.addIP(b, n.IP()) { + if !tab.addIP(b, req.node.IPAddr()) { // Can't add: IP limit reached. - return - } - // Add to front of bucket. - b.entries, _ = pushNode(b.entries, n, bucketSize) - b.replacements = deleteNode(b.replacements, n) - n.addedAt = time.Now() - if tab.nodeAddedHook != nil { - tab.nodeAddedHook(n) + return false } -} - -// delete removes an entry from the node table. It is used to evacuate dead nodes. -func (tab *Table) delete(node *node) { - tab.mutex.Lock() - defer tab.mutex.Unlock() - tab.deleteInBucket(tab.bucket(node.ID()), node) + // Add to bucket. + wn := &tableNode{Node: req.node} + if req.forceSetLive { + wn.livenessChecks = 1 + wn.isValidatedLive = true + } + b.entries = append(b.entries, wn) + b.replacements = deleteNode(b.replacements, wn.ID()) + tab.nodeAdded(b, wn) + return true } -func (tab *Table) addIP(b *bucket, ip net.IP) bool { - if len(ip) == 0 { - return false // Nodes without IP cannot be added. - } - if netutil.IsLAN(ip) { - return true +// addReplacement adds n to the replacement cache of bucket b. +func (tab *Table) addReplacement(b *bucket, n *enode.Node) { + if containsID(b.replacements, n.ID()) { + // TODO: update ENR + return } - if !tab.ips.Add(ip) { - tab.log.Trace("IP exceeds table limit", "ip", ip) - return false + if !tab.addIP(b, n.IPAddr()) { + return } - if !b.ips.Add(ip) { - tab.log.Trace("IP exceeds bucket limit", "ip", ip) - tab.ips.Remove(ip) - return false + + wn := &tableNode{Node: n, addedToTable: time.Now()} + var removed *tableNode + b.replacements, removed = pushNode(b.replacements, wn, maxReplacements) + if removed != nil { + tab.removeIP(b, removed.IPAddr()) } - return true } -func (tab *Table) removeIP(b *bucket, ip net.IP) { - if netutil.IsLAN(ip) { - return +func (tab *Table) nodeAdded(b *bucket, n *tableNode) { + if n.addedToTable.IsZero() { + n.addedToTable = time.Now() } - tab.ips.Remove(ip) - b.ips.Remove(ip) -} + n.addedToBucket = time.Now() + tab.revalidation.nodeAdded(tab, n) -func (tab *Table) addReplacement(b *bucket, n *node) { - for _, e := range b.replacements { - if e.ID() == n.ID() { - return // already in list - } + tab.nodeFeed.Send(n.Node) + if tab.nodeAddedHook != nil { + tab.nodeAddedHook(b, n) } - if !tab.addIP(b, n.IP()) { - return + if metrics.Enabled() { + bucketsCounter[b.index].Inc(1) } - var removed *node - b.replacements, removed = pushNode(b.replacements, n, maxReplacements) - if removed != nil { - tab.removeIP(b, removed.IP()) +} + +func (tab *Table) nodeRemoved(b *bucket, n *tableNode) { + tab.revalidation.nodeRemoved(n) + if tab.nodeRemovedHook != nil { + tab.nodeRemovedHook(b, n) + } + if metrics.Enabled() { + bucketsCounter[b.index].Dec(1) } } -// replace removes n from the replacement list and replaces 'last' with it if it is the -// last entry in the bucket. If 'last' isn't the last entry, it has either been replaced -// with someone else or became active. -func (tab *Table) replace(b *bucket, last *node) *node { - if len(b.entries) == 0 || b.entries[len(b.entries)-1].ID() != last.ID() { - // Entry has moved, don't replace it. +// deleteInBucket removes node n from the table. +// If there are replacement nodes in the bucket, the node is replaced. +func (tab *Table) deleteInBucket(b *bucket, id enode.ID) *tableNode { + index := slices.IndexFunc(b.entries, func(e *tableNode) bool { return e.ID() == id }) + if index == -1 { + // Entry has been removed already. return nil } - // Still the last entry. + + // Remove the node. + n := b.entries[index] + b.entries = slices.Delete(b.entries, index, index+1) + tab.removeIP(b, n.IPAddr()) + tab.nodeRemoved(b, n) + + // Add replacement. if len(b.replacements) == 0 { - tab.deleteInBucket(b, last) + tab.log.Debug("Removed dead node", "b", b.index, "id", n.ID(), "ip", n.IPAddr()) return nil } - r := b.replacements[tab.rand.Intn(len(b.replacements))] - b.replacements = deleteNode(b.replacements, r) - b.entries[len(b.entries)-1] = r - tab.removeIP(b, last.IP()) - return r -} - -// bumpInBucket moves the given node to the front of the bucket entry list -// if it is contained in that list. -func (tab *Table) bumpInBucket(b *bucket, n *node) bool { - for i := range b.entries { - if b.entries[i].ID() == n.ID() { - if !n.IP().Equal(b.entries[i].IP()) { - // Endpoint has changed, ensure that the new IP fits into table limits. - tab.removeIP(b, b.entries[i].IP()) - if !tab.addIP(b, n.IP()) { - // It doesn't, put the previous one back. - tab.addIP(b, b.entries[i].IP()) - return false - } - } - // Move it to the front. - copy(b.entries[1:], b.entries[:i]) - b.entries[0] = n - return true + rindex := tab.rand.Intn(len(b.replacements)) + rep := b.replacements[rindex] + b.replacements = slices.Delete(b.replacements, rindex, rindex+1) + b.entries = append(b.entries, rep) + tab.nodeAdded(b, rep) + tab.log.Debug("Replaced dead node", "b", b.index, "id", n.ID(), "ip", n.IPAddr(), "r", rep.ID(), "rip", rep.IPAddr()) + return rep +} + +// bumpInBucket updates a node record if it exists in the bucket. +// The second return value reports whether the node's endpoint (IP/port) was updated. +func (tab *Table) bumpInBucket(b *bucket, newRecord *enode.Node, isInbound bool) (n *tableNode, endpointChanged bool) { + i := slices.IndexFunc(b.entries, func(elem *tableNode) bool { + return elem.ID() == newRecord.ID() + }) + if i == -1 { + return nil, false // not in bucket + } + n = b.entries[i] + + // For inbound updates (from the node itself) we accept any change, even if it sets + // back the sequence number. For found nodes (!isInbound), seq has to advance. Note + // this check also ensures found discv4 nodes (which always have seq=0) can't be + // updated. + if newRecord.Seq() <= n.Seq() && !isInbound { + return n, false + } + + // Check endpoint update against IP limits. + ipchanged := newRecord.IPAddr() != n.IPAddr() + portchanged := newRecord.UDP() != n.UDP() + if ipchanged { + tab.removeIP(b, n.IPAddr()) + if !tab.addIP(b, newRecord.IPAddr()) { + // It doesn't fit with the limit, put the previous record back. + tab.addIP(b, n.IPAddr()) + return n, false } } - return false -} -func (tab *Table) deleteInBucket(b *bucket, n *node) { - b.entries = deleteNode(b.entries, n) - tab.removeIP(b, n.IP()) + // Apply update. + n.Node = newRecord + if ipchanged || portchanged { + // Ensure node is revalidated quickly for endpoint changes. + tab.revalidation.nodeEndpointChanged(tab, n) + return n, true + } + return n, false } -func contains(ns []*node, id enode.ID) bool { - for _, n := range ns { - if n.ID() == id { - return true - } +func (tab *Table) handleTrackRequest(op trackRequestOp) { + var fails int + if op.success { + // Reset failure counter because it counts _consecutive_ failures. + tab.db.UpdateFindFails(op.node.ID(), op.node.IPAddr(), 0) + } else { + fails = tab.db.FindFails(op.node.ID(), op.node.IPAddr()) + fails++ + tab.db.UpdateFindFails(op.node.ID(), op.node.IPAddr(), fails) + } + + tab.mutex.Lock() + defer tab.mutex.Unlock() + + b := tab.bucket(op.node.ID()) + // Remove the node from the local table if it fails to return anything useful too + // many times, but only if there are enough other nodes in the bucket. This latter + // condition specifically exists to make bootstrapping in smaller test networks more + // reliable. + if fails >= maxFindnodeFailures && len(b.entries) >= bucketSize/4 { + tab.deleteInBucket(b, op.node.ID()) + } + + // Add found nodes. + for _, n := range op.foundNodes { + tab.handleAddNode(addNodeOp{n, false, false}) } - return false } // pushNode adds n to the front of list, keeping at most max items. -func pushNode(list []*node, n *node, _max int) ([]*node, *node) { - if len(list) < _max { +func pushNode(list []*tableNode, n *tableNode, max int) ([]*tableNode, *tableNode) { + if len(list) < max { list = append(list, nil) } removed := list[len(list)-1] @@ -760,37 +698,45 @@ func pushNode(list []*node, n *node, _max int) ([]*node, *node) { return list, removed } -// deleteNode removes n from list. -func deleteNode(list []*node, n *node) []*node { - for i := range list { - if list[i].ID() == n.ID() { - return append(list[:i], list[i+1:]...) +// deleteNode removes a node from the table. +func (tab *Table) deleteNode(n *enode.Node) { + tab.mutex.Lock() + defer tab.mutex.Unlock() + b := tab.bucket(n.ID()) + tab.deleteInBucket(b, n.ID()) +} + +// waitForNodes blocks until the table contains at least n nodes. +func (tab *Table) waitForNodes(ctx context.Context, n int) error { + getlength := func() (count int) { + for _, b := range &tab.buckets { + count += len(b.entries) } + return count } - return list -} -// nodesByDistance is a list of nodes, ordered by distance to target. -type nodesByDistance struct { - entries []*node - target enode.ID -} + var ch chan *enode.Node + for { + tab.mutex.Lock() + if getlength() >= n { + tab.mutex.Unlock() + return nil + } + if ch == nil { + // Init subscription. + ch = make(chan *enode.Node) + sub := tab.nodeFeed.Subscribe(ch) + defer sub.Unsubscribe() + } + tab.mutex.Unlock() -// push adds the given node to the list, keeping the total size below maxElems. -func (h *nodesByDistance) push(n *node, maxElems int) { - ix := sort.Search(len(h.entries), func(i int) bool { - return enode.DistCmp(h.target, h.entries[i].ID(), n.ID()) > 0 - }) - if len(h.entries) < maxElems { - h.entries = append(h.entries, n) - } - if ix == len(h.entries) { - // farther away than all nodes we already have. - // if there was room for it, the node is now the last element. - } else { - // slide existing entries down to make room - // this will overwrite the entry we just appended. - copy(h.entries[ix+1:], h.entries[ix:]) - h.entries[ix] = n + // Wait for a node add event. + select { + case <-ch: + case <-ctx.Done(): + return ctx.Err() + case <-tab.closeReq: + return errClosed + } } } diff --git a/p2p/discover/table_integration_test.go b/p2p/discover/table_integration_test.go deleted file mode 100644 index edc5c7a2629..00000000000 --- a/p2p/discover/table_integration_test.go +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright 2024 The Erigon Authors -// This file is part of Erigon. -// -// Erigon is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// Erigon is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with Erigon. If not, see . - -package discover - -import ( - "math/rand" - "testing" - "testing/quick" - "time" -) - -func TestTable_bumpNoDuplicates_quickCheck(t *testing.T) { - if testing.Short() { - t.Skip() - } - - t.Parallel() - - config := quick.Config{ - MaxCount: 200, - Rand: rand.New(rand.NewSource(time.Now().Unix())), - } - - test := func(bucketCountGen byte, bumpCountGen byte) bool { - return testTableBumpNoDuplicatesRun(t, bucketCountGen, bumpCountGen, config.Rand) - } - - if err := quick.Check(test, &config); err != nil { - t.Error(err) - } -} - -func TestTable_findNodeByID_quickCheck(t *testing.T) { - if testing.Short() { - t.Skip() - } - - t.Parallel() - - config := quick.Config{ - MaxCount: 1000, - Rand: rand.New(rand.NewSource(time.Now().Unix())), - } - - test := func(nodesCount uint16, resultsCount byte) bool { - return testTableFindNodeByIDRun(t, nodesCount, resultsCount, config.Rand) - } - - if err := quick.Check(test, &config); err != nil { - t.Error(err) - } -} - -func TestTable_ReadRandomNodesGetAll_quickCheck(t *testing.T) { - if testing.Short() { - t.Skip() - } - - t.Parallel() - - config := quick.Config{ - MaxCount: 200, - Rand: rand.New(rand.NewSource(time.Now().Unix())), - } - - test := func(nodesCount uint16) bool { - return testTableReadRandomNodesGetAllRun(t, nodesCount, config.Rand) - } - - if err := quick.Check(test, &config); err != nil { - t.Error(err) - } -} diff --git a/p2p/discover/table_reval.go b/p2p/discover/table_reval.go new file mode 100644 index 00000000000..9cb1254a04c --- /dev/null +++ b/p2p/discover/table_reval.go @@ -0,0 +1,248 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package discover + +import ( + "fmt" + "math" + "slices" + "time" + + "github.com/erigontech/erigon/common/mclock" + "github.com/erigontech/erigon/p2p/enode" +) + +const never = mclock.AbsTime(math.MaxInt64) + +const slowRevalidationFactor = 3 + +// tableRevalidation implements the node revalidation process. +// It tracks all nodes contained in Table, and schedules sending PING to them. +type tableRevalidation struct { + fast revalidationList + slow revalidationList + activeReq map[enode.ID]struct{} +} + +type revalidationResponse struct { + n *tableNode + newRecord *enode.Node + didRespond bool +} + +func (tr *tableRevalidation) init(cfg *Config) { + tr.activeReq = make(map[enode.ID]struct{}) + tr.fast.nextTime = never + tr.fast.interval = cfg.PingInterval + tr.fast.name = "fast" + tr.slow.nextTime = never + tr.slow.interval = cfg.PingInterval * slowRevalidationFactor + tr.slow.name = "slow" +} + +// nodeAdded is called when the table receives a new node. +func (tr *tableRevalidation) nodeAdded(tab *Table, n *tableNode) { + tr.fast.push(n, tab.cfg.Clock.Now(), &tab.rand) +} + +// nodeRemoved is called when a node was removed from the table. +func (tr *tableRevalidation) nodeRemoved(n *tableNode) { + if n.revalList == nil { + panic(fmt.Errorf("removed node %v has nil revalList", n.ID())) + } + n.revalList.remove(n) +} + +// nodeEndpointChanged is called when a change in IP or port is detected. +func (tr *tableRevalidation) nodeEndpointChanged(tab *Table, n *tableNode) { + n.isValidatedLive = false + tr.moveToList(&tr.fast, n, tab.cfg.Clock.Now(), &tab.rand) +} + +// run performs node revalidation. +// It returns the next time it should be invoked, which is used in the Table main loop +// to schedule a timer. However, run can be called at any time. +func (tr *tableRevalidation) run(tab *Table, now mclock.AbsTime) (nextTime mclock.AbsTime) { + reval := func(list *revalidationList) { + if list.nextTime <= now { + if n := list.get(&tab.rand, tr.activeReq); n != nil { + tr.startRequest(tab, n) + } + // Update nextTime regardless if any requests were started because + // current value has passed. + list.schedule(now, &tab.rand) + } + } + reval(&tr.fast) + reval(&tr.slow) + + return min(tr.fast.nextTime, tr.slow.nextTime) +} + +// startRequest spawns a revalidation request for node n. +func (tr *tableRevalidation) startRequest(tab *Table, n *tableNode) { + if _, ok := tr.activeReq[n.ID()]; ok { + panic(fmt.Errorf("duplicate startRequest (node %v)", n.ID())) + } + tr.activeReq[n.ID()] = struct{}{} + resp := revalidationResponse{n: n} + + // Fetch the node while holding lock. + tab.mutex.Lock() + node := n.Node + tab.mutex.Unlock() + + go tab.doRevalidate(resp, node) +} + +func (tab *Table) doRevalidate(resp revalidationResponse, node *enode.Node) { + // Ping the selected node and wait for a pong response. + remoteSeq, err := tab.net.ping(node) + resp.didRespond = err == nil + + // Also fetch record if the node replied and returned a higher sequence number. + if remoteSeq > node.Seq() { + newrec, err := tab.net.RequestENR(node) + if err != nil { + tab.log.Debug("ENR request failed", "id", node.ID(), "err", err) + } else { + resp.newRecord = newrec + } + } + + select { + case tab.revalResponseCh <- resp: + case <-tab.closed: + } +} + +// handleResponse processes the result of a revalidation request. +func (tr *tableRevalidation) handleResponse(tab *Table, resp revalidationResponse) { + var ( + now = tab.cfg.Clock.Now() + n = resp.n + b = tab.bucket(n.ID()) + ) + delete(tr.activeReq, n.ID()) + + // If the node was removed from the table while getting checked, we need to stop + // processing here to avoid re-adding it. + if n.revalList == nil { + return + } + + // Store potential seeds in database. + // This is done via defer to avoid holding Table lock while writing to DB. + defer func() { + if n.isValidatedLive && n.livenessChecks > 5 { + tab.db.UpdateNode(resp.n.Node) + } + }() + + // Remaining logic needs access to Table internals. + tab.mutex.Lock() + defer tab.mutex.Unlock() + + if !resp.didRespond { + n.livenessChecks /= 3 + if n.livenessChecks <= 0 { + tab.deleteInBucket(b, n.ID()) + } else { + tab.log.Debug("Node revalidation failed", "b", b.index, "id", n.ID(), "checks", n.livenessChecks, "q", n.revalList.name) + tr.moveToList(&tr.fast, n, now, &tab.rand) + } + return + } + + // The node responded. + n.livenessChecks++ + n.isValidatedLive = true + tab.log.Debug("Node revalidated", "b", b.index, "id", n.ID(), "checks", n.livenessChecks, "q", n.revalList.name) + var endpointChanged bool + if resp.newRecord != nil { + _, endpointChanged = tab.bumpInBucket(b, resp.newRecord, false) + } + + // Node moves to slow list if it passed and hasn't changed. + if !endpointChanged { + tr.moveToList(&tr.slow, n, now, &tab.rand) + } +} + +// moveToList ensures n is in the 'dest' list. +func (tr *tableRevalidation) moveToList(dest *revalidationList, n *tableNode, now mclock.AbsTime, rand randomSource) { + if n.revalList == dest { + return + } + if n.revalList != nil { + n.revalList.remove(n) + } + dest.push(n, now, rand) +} + +// revalidationList holds a list nodes and the next revalidation time. +type revalidationList struct { + nodes []*tableNode + nextTime mclock.AbsTime + interval time.Duration + name string +} + +// get returns a random node from the queue. Nodes in the 'exclude' map are not returned. +func (list *revalidationList) get(rand randomSource, exclude map[enode.ID]struct{}) *tableNode { + if len(list.nodes) == 0 { + return nil + } + for i := 0; i < len(list.nodes)*3; i++ { + n := list.nodes[rand.Intn(len(list.nodes))] + _, excluded := exclude[n.ID()] + if !excluded { + return n + } + } + return nil +} + +func (list *revalidationList) schedule(now mclock.AbsTime, rand randomSource) { + list.nextTime = now.Add(time.Duration(rand.Int63n(int64(list.interval)))) +} + +func (list *revalidationList) push(n *tableNode, now mclock.AbsTime, rand randomSource) { + list.nodes = append(list.nodes, n) + if list.nextTime == never { + list.schedule(now, rand) + } + n.revalList = list +} + +func (list *revalidationList) remove(n *tableNode) { + i := slices.Index(list.nodes, n) + if i == -1 { + panic(fmt.Errorf("node %v not found in list", n.ID())) + } + list.nodes = slices.Delete(list.nodes, i, i+1) + if len(list.nodes) == 0 { + list.nextTime = never + } + n.revalList = nil +} + +func (list *revalidationList) contains(id enode.ID) bool { + return slices.ContainsFunc(list.nodes, func(n *tableNode) bool { + return n.ID() == id + }) +} diff --git a/p2p/discover/table_reval_test.go b/p2p/discover/table_reval_test.go new file mode 100644 index 00000000000..fb18e19f499 --- /dev/null +++ b/p2p/discover/table_reval_test.go @@ -0,0 +1,119 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package discover + +import ( + "net" + "testing" + "time" + + "github.com/erigontech/erigon/common/mclock" + "github.com/erigontech/erigon/p2p/enode" + "github.com/erigontech/erigon/p2p/enr" +) + +// This test checks that revalidation can handle a node disappearing while +// a request is active. +func TestRevalidation_nodeRemoved(t *testing.T) { + var ( + clock mclock.Simulated + transport = newPingRecorder() + tab, db = newInactiveTestTable(transport, Config{Clock: &clock}) + tr = &tab.revalidation + ) + defer db.Close() + + // Add a node to the table. + node := nodeAtDistance(tab.self().ID(), 255, net.IP{77, 88, 99, 1}) + tab.handleAddNode(addNodeOp{node: node}) + + // Start a revalidation request. Schedule once to get the next start time, + // then advance the clock to that point and schedule again to start. + next := tr.run(tab, clock.Now()) + clock.Run(time.Duration(next + 1)) + tr.run(tab, clock.Now()) + if len(tr.activeReq) != 1 { + t.Fatal("revalidation request did not start:", tr.activeReq) + } + + // Delete the node. + tab.deleteInBucket(tab.bucket(node.ID()), node.ID()) + + // Now finish the revalidation request. + var resp revalidationResponse + select { + case resp = <-tab.revalResponseCh: + case <-time.After(1 * time.Second): + t.Fatal("timed out waiting for revalidation") + } + tr.handleResponse(tab, resp) + + // Ensure the node was not re-added to the table. + if tab.getNode(node.ID()) != nil { + t.Fatal("node was re-added to Table") + } + if tr.fast.contains(node.ID()) || tr.slow.contains(node.ID()) { + t.Fatal("removed node contained in revalidation list") + } +} + +// This test checks that nodes with an updated endpoint remain in the fast revalidation list. +func TestRevalidation_endpointUpdate(t *testing.T) { + var ( + clock mclock.Simulated + transport = newPingRecorder() + tab, db = newInactiveTestTable(transport, Config{Clock: &clock}) + tr = &tab.revalidation + ) + defer db.Close() + + // Add node to table. + node := nodeAtDistance(tab.self().ID(), 255, net.IP{77, 88, 99, 1}) + tab.handleAddNode(addNodeOp{node: node}) + + // Update the record in transport, including endpoint update. + record := node.Record() + record.Set(enr.IP{100, 100, 100, 100}) + record.Set(enr.UDP(9999)) + nodev2 := enode.SignNull(record, node.ID()) + transport.updateRecord(nodev2) + + // Start a revalidation request. Schedule once to get the next start time, + // then advance the clock to that point and schedule again to start. + next := tr.run(tab, clock.Now()) + clock.Run(time.Duration(next + 1)) + tr.run(tab, clock.Now()) + if len(tr.activeReq) != 1 { + t.Fatal("revalidation request did not start:", tr.activeReq) + } + + // Now finish the revalidation request. + var resp revalidationResponse + select { + case resp = <-tab.revalResponseCh: + case <-time.After(1 * time.Second): + t.Fatal("timed out waiting for revalidation") + } + tr.handleResponse(tab, resp) + + if tr.fast.nodes[0].ID() != node.ID() { + t.Fatal("node not contained in fast revalidation list") + } + if tr.fast.nodes[0].isValidatedLive { + t.Fatal("node is marked live after endpoint change") + } +} diff --git a/p2p/discover/table_test.go b/p2p/discover/table_test.go index 197a382dfea..2762a68784e 100644 --- a/p2p/discover/table_test.go +++ b/p2p/discover/table_test.go @@ -1,21 +1,18 @@ // Copyright 2015 The go-ethereum Authors -// (original work) -// Copyright 2024 The Erigon Authors -// (modifications) -// This file is part of Erigon. +// This file is part of the go-ethereum library. // -// Erigon is free software: you can redistribute it and/or modify +// The go-ethereum library is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // -// Erigon is distributed in the hope that it will be useful, +// The go-ethereum library is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Lesser General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License -// along with Erigon. If not, see . +// along with the go-ethereum library. If not, see . package discover @@ -25,11 +22,15 @@ import ( "math/rand" "net" "reflect" + "slices" "testing" + "testing/quick" "time" "github.com/erigontech/erigon/common/crypto" "github.com/erigontech/erigon/common/log/v3" + "github.com/erigontech/erigon/common/mclock" + "github.com/erigontech/erigon/common/testlog" "github.com/erigontech/erigon/p2p/enode" "github.com/erigontech/erigon/p2p/enr" "github.com/erigontech/erigon/p2p/netutil" @@ -51,136 +52,109 @@ func TestTable_pingReplace(t *testing.T) { } func testPingReplace(t *testing.T, newNodeIsResponding, lastInBucketIsResponding bool) { + simclock := new(mclock.Simulated) transport := newPingRecorder() - tmpDir := t.TempDir() - tab, db := newTestTable(transport, tmpDir, log.Root()) + tab, db := newTestTable(transport, Config{ + Clock: simclock, + Log: testlog.Logger(t, log.LvlTrace), + }) defer db.Close() defer tab.close() <-tab.initDone // Fill up the sender's bucket. - pingKey, _ := crypto.HexToECDSA("45a915e4d060149eb4365960e6a7a45f334393093061116b197e3240065ff2d8") - pingSender := wrapNode(enode.NewV4(&pingKey.PublicKey, net.IP{127, 0, 0, 1}, 99, 99)) - last := fillBucket(tab, pingSender) + replacementNodeKey, _ := crypto.HexToECDSA("45a915e4d060149eb4365960e6a7a45f334393093061116b197e3240065ff2d8") + replacementNode := enode.NewV4(&replacementNodeKey.PublicKey, net.IP{127, 0, 0, 1}, 99, 99) + last := fillBucket(tab, replacementNode.ID()) + tab.mutex.Lock() + nodeEvents := newNodeEventRecorder(128) + tab.nodeAddedHook = nodeEvents.nodeAdded + tab.nodeRemovedHook = nodeEvents.nodeRemoved + tab.mutex.Unlock() - // Add the sender as if it just pinged us. Revalidate should replace the last node in - // its bucket if it is unresponsive. Revalidate again to ensure that + // The revalidation process should replace + // this node in the bucket if it is unresponsive. transport.dead[last.ID()] = !lastInBucketIsResponding - transport.dead[pingSender.ID()] = !newNodeIsResponding - tab.addSeenNode(pingSender) - tab.doRevalidate(make(chan struct{}, 1)) - tab.doRevalidate(make(chan struct{}, 1)) - - if !transport.pinged[last.ID()] { - // Oldest node in bucket is pinged to see whether it is still alive. - t.Error("table did not ping last node in bucket") + transport.dead[replacementNode.ID()] = !newNodeIsResponding + + // Add replacement node to table. + tab.addFoundNode(replacementNode, false) + + t.Log("last:", last.ID()) + t.Log("replacement:", replacementNode.ID()) + + // Wait until the last node was pinged. + waitForRevalidationPing(t, transport, tab, last.ID()) + + if !lastInBucketIsResponding { + if !nodeEvents.waitNodeAbsent(last.ID(), 2*time.Second) { + t.Error("last node was not removed") + } + if !nodeEvents.waitNodePresent(replacementNode.ID(), 2*time.Second) { + t.Error("replacement node was not added") + } + + // If a replacement is expected, we also need to wait until the replacement node + // was pinged and added/removed. + waitForRevalidationPing(t, transport, tab, replacementNode.ID()) + if !newNodeIsResponding { + if !nodeEvents.waitNodeAbsent(replacementNode.ID(), 2*time.Second) { + t.Error("replacement node was not removed") + } + } } + // Check bucket content. tab.mutex.Lock() defer tab.mutex.Unlock() wantSize := bucketSize if !lastInBucketIsResponding && !newNodeIsResponding { wantSize-- } - if l := len(tab.bucket(pingSender.ID()).entries); l != wantSize { - t.Errorf("wrong bucket size after bond: got %d, want %d", l, wantSize) + bucket := tab.bucket(replacementNode.ID()) + if l := len(bucket.entries); l != wantSize { + t.Errorf("wrong bucket size after revalidation: got %d, want %d", l, wantSize) } - if found := contains(tab.bucket(pingSender.ID()).entries, last.ID()); found != lastInBucketIsResponding { - t.Errorf("last entry found: %t, want: %t", found, lastInBucketIsResponding) + if ok := containsID(bucket.entries, last.ID()); ok != lastInBucketIsResponding { + t.Errorf("revalidated node found: %t, want: %t", ok, lastInBucketIsResponding) } wantNewEntry := newNodeIsResponding && !lastInBucketIsResponding - if found := contains(tab.bucket(pingSender.ID()).entries, pingSender.ID()); found != wantNewEntry { - t.Errorf("new entry found: %t, want: %t", found, wantNewEntry) + if ok := containsID(bucket.entries, replacementNode.ID()); ok != wantNewEntry { + t.Errorf("replacement node found: %t, want: %t", ok, wantNewEntry) } } -func testTableBumpNoDuplicatesRun(t *testing.T, bucketCountGen byte, bumpCountGen byte, randGen *rand.Rand) bool { - generateBucketNodes := func(bucketCountGen byte) []*node { - bucketCount := bucketCountGen % (bucketSize + 1) // [0...bucketSize] - nodes := make([]*node, bucketCount) - for i := range nodes { - nodes[i] = nodeAtDistance(enode.ID{}, 200, intIP(200)) - } - return nodes - } +// waitForRevalidationPing waits until a PING message is sent to a node with the given id. +func waitForRevalidationPing(t *testing.T, transport *pingRecorder, tab *Table, id enode.ID) *enode.Node { + t.Helper() - generateRandomBumpPositions := func(bumpCountGen byte, bucketCount int, randGen *rand.Rand) []int { - bumpCount := bumpCountGen % 100 - bumps := make([]int, bumpCount) - for i := range bumps { - bumps[i] = randGen.Intn(bucketCount) + simclock := tab.cfg.Clock.(*mclock.Simulated) + maxAttempts := tab.len() * 8 + for i := 0; i < maxAttempts; i++ { + simclock.Run(tab.cfg.PingInterval * slowRevalidationFactor) + p := transport.waitPing(2 * time.Second) + if p == nil { + continue } - return bumps - } - - nodes := generateBucketNodes(bucketCountGen) - if len(nodes) == 0 { - return true - } - bumps := generateRandomBumpPositions(bumpCountGen, len(nodes), randGen) - - if len(bumps) > 0 { - tmpDir := t.TempDir() - tab, db := newTestTable(newPingRecorder(), tmpDir, log.Root()) - defer db.Close() - defer tab.close() - - b := &bucket{entries: make([]*node, len(nodes))} - copy(b.entries, nodes) - - for i, pos := range bumps { - tab.bumpInBucket(b, b.entries[pos]) - if hasDuplicates(b.entries) { - t.Logf("bucket has duplicates after %d/%d bumps:", i+1, len(bumps)) - for _, n := range b.entries { - t.Logf(" %p", n) - } - return false - } + if id == (enode.ID{}) || p.ID() == id { + return p } - checkIPLimitInvariant(t, tab) - return true } - return true -} - -func TestTable_bumpNoDuplicates_examples(t *testing.T) { - t.Parallel() - - randGen := rand.New(rand.NewSource(time.Now().Unix())) - - t.Run("n1b1", func(t *testing.T) { - testTableBumpNoDuplicatesRun(t, 1, 1, randGen) - }) - t.Run("n1b5", func(t *testing.T) { - testTableBumpNoDuplicatesRun(t, 1, 5, randGen) - }) - t.Run("n5b1", func(t *testing.T) { - testTableBumpNoDuplicatesRun(t, 5, 1, randGen) - }) - t.Run("n5b10", func(t *testing.T) { - testTableBumpNoDuplicatesRun(t, 5, 10, randGen) - }) - t.Run("n16b10", func(t *testing.T) { - testTableBumpNoDuplicatesRun(t, 16, 10, randGen) - }) - t.Run("n16b90", func(t *testing.T) { - testTableBumpNoDuplicatesRun(t, 16, 90, randGen) - }) + t.Fatalf("Table did not ping node %v (%d attempts)", id, maxAttempts) + return nil } // This checks that the table-wide IP limit is applied correctly. func TestTable_IPLimit(t *testing.T) { transport := newPingRecorder() - tmpDir := t.TempDir() - tab, db := newTestTable(transport, tmpDir, log.Root()) + tab, db := newTestTable(transport, Config{}) defer db.Close() defer tab.close() for i := 0; i < tableIPLimit+1; i++ { n := nodeAtDistance(tab.self().ID(), i, net.IP{172, 0, 1, byte(i)}) - tab.addSeenNode(n) + tab.addFoundNode(n, false) } if tab.len() > tableIPLimit { t.Errorf("too many nodes in table") @@ -191,15 +165,14 @@ func TestTable_IPLimit(t *testing.T) { // This checks that the per-bucket IP limit is applied correctly. func TestTable_BucketIPLimit(t *testing.T) { transport := newPingRecorder() - tmpDir := t.TempDir() - tab, db := newTestTable(transport, tmpDir, log.Root()) + tab, db := newTestTable(transport, Config{}) defer db.Close() defer tab.close() d := 3 for i := 0; i < bucketIPLimit+1; i++ { n := nodeAtDistance(tab.self().ID(), d, net.IP{172, 0, 1, byte(i)}) - tab.addSeenNode(n) + tab.addFoundNode(n, false) } if tab.len() > bucketIPLimit { t.Errorf("too many nodes in table") @@ -215,7 +188,7 @@ func checkIPLimitInvariant(t *testing.T, tab *Table) { tabset := netutil.DistinctNetSet{Subnet: tableSubnet, Limit: tableIPLimit} for _, b := range tab.buckets { for _, n := range b.entries { - tabset.Add(n.IP()) + tabset.AddAddr(n.IPAddr()) } } if tabset.String() != tab.ips.String() { @@ -223,36 +196,31 @@ func checkIPLimitInvariant(t *testing.T, tab *Table) { } } -func testTableFindNodeByIDRun(t *testing.T, nodesCountGen uint16, resultsCountGen byte, rand *rand.Rand) bool { - if !t.Skipped() { +func TestTable_findnodeByID(t *testing.T) { + t.Parallel() + + test := func(test *closeTest) bool { // for any node table, Target and N transport := newPingRecorder() - tmpDir := t.TempDir() - tab, db := newTestTable(transport, tmpDir, log.Root()) + tab, db := newTestTable(transport, Config{}) defer db.Close() defer tab.close() - - nodesCount := int(nodesCountGen) % (bucketSize*nBuckets + 1) - testNodes := generateNodes(rand, nodesCount) - fillTable(tab, testNodes) - - target := enode.ID{} - resultsCount := int(resultsCountGen) % (bucketSize + 1) + fillTable(tab, test.All, true) // check that closest(Target, N) returns nodes - result := tab.findnodeByID(target, resultsCount, false).entries + result := tab.findnodeByID(test.Target, test.N, false).entries if hasDuplicates(result) { t.Errorf("result contains duplicates") return false } - if !sortedByDistanceTo(target, result) { + if !sortedByDistanceTo(test.Target, result) { t.Errorf("result is not sorted by distance to target") return false } // check that the number of results is min(N, tablen) - wantN := resultsCount - if tlen := tab.len(); tlen < resultsCount { + wantN := test.N + if tlen := tab.len(); tlen < test.N { wantN = tlen } if len(result) != wantN { @@ -265,13 +233,13 @@ func testTableFindNodeByIDRun(t *testing.T, nodesCountGen uint16, resultsCountGe // check that the result nodes have minimum distance to target. for _, b := range tab.buckets { for _, n := range b.entries { - if contains(result, n.ID()) { + if containsID(result, n.ID()) { continue // don't run the check below for nodes in result } farthestResult := result[len(result)-1].ID() - if enode.DistCmp(target, n.ID(), farthestResult) < 0 { + if enode.DistCmp(test.Target, n.ID(), farthestResult) < 0 { t.Errorf("table contains node that is closer to target but it's not in result") - t.Logf(" Target: %v", target) + t.Logf(" Target: %v", test.Target) t.Logf(" Farthest Result: %v", farthestResult) t.Logf(" ID: %v", n.ID()) return false @@ -280,127 +248,35 @@ func testTableFindNodeByIDRun(t *testing.T, nodesCountGen uint16, resultsCountGe } return true } - return true -} - -func TestTable_findNodeByID_examples(t *testing.T) { - if testing.Short() { - t.Skip() - } - - t.Parallel() - - randGen := rand.New(rand.NewSource(time.Now().Unix())) - - t.Run("n0r1", func(t *testing.T) { - testTableFindNodeByIDRun(t, 0, 1, randGen) - }) - t.Run("n1r1", func(t *testing.T) { - testTableFindNodeByIDRun(t, 1, 1, randGen) - }) - t.Run("n16r1", func(t *testing.T) { - testTableFindNodeByIDRun(t, bucketSize, 1, randGen) - }) - t.Run("nMr1", func(t *testing.T) { - testTableFindNodeByIDRun(t, uint16(bucketSize*nBuckets), 1, randGen) - }) - t.Run("n0r2", func(t *testing.T) { - testTableFindNodeByIDRun(t, 0, 2, randGen) - }) - t.Run("n1r2", func(t *testing.T) { - testTableFindNodeByIDRun(t, 1, 2, randGen) - }) - t.Run("n16r2", func(t *testing.T) { - testTableFindNodeByIDRun(t, bucketSize, 2, randGen) - }) - t.Run("nMr2", func(t *testing.T) { - testTableFindNodeByIDRun(t, uint16(bucketSize*nBuckets), 2, randGen) - }) - t.Run("n0rM", func(t *testing.T) { - testTableFindNodeByIDRun(t, 0, bucketSize, randGen) - }) - t.Run("n1rM", func(t *testing.T) { - testTableFindNodeByIDRun(t, 1, bucketSize, randGen) - }) - t.Run("n16rM", func(t *testing.T) { - testTableFindNodeByIDRun(t, bucketSize, bucketSize, randGen) - }) - t.Run("nMrM", func(t *testing.T) { - testTableFindNodeByIDRun(t, uint16(bucketSize*nBuckets), bucketSize, randGen) - }) -} - -func testTableReadRandomNodesGetAllRun(t *testing.T, nodesCountGen uint16, rand *rand.Rand) bool { - nodesCount := nodesCountGen % 1000 - if nodesCount > 0 { - buf := make([]*enode.Node, nodesCount) - transport := newPingRecorder() - tmpDir := t.TempDir() - tab, db := newTestTable(transport, tmpDir, log.Root()) - defer db.Close() - defer tab.close() - <-tab.initDone - - for i := 0; i < len(buf); i++ { - ld := rand.Intn(len(tab.buckets)) - fillTable(tab, []*node{nodeAtDistance(tab.self().ID(), ld, intIP(ld))}) - } - gotN := tab.ReadRandomNodes(buf) - if gotN != tab.len() { - t.Errorf("wrong number of nodes, got %d, want %d", gotN, tab.len()) - return false - } - if hasDuplicates(wrapNodes(buf[:gotN])) { - t.Errorf("result contains duplicates") - return false - } - return true + if err := quick.Check(test, quickcfg()); err != nil { + t.Error(err) } - return true } -func TestTable_ReadRandomNodesGetAll_examples(t *testing.T) { - t.Parallel() - - randGen := rand.New(rand.NewSource(time.Now().Unix())) - - t.Run("n1", func(t *testing.T) { - testTableReadRandomNodesGetAllRun(t, 1, randGen) - }) - t.Run("n2", func(t *testing.T) { - testTableReadRandomNodesGetAllRun(t, 2, randGen) - }) - t.Run("n20", func(t *testing.T) { - testTableReadRandomNodesGetAllRun(t, 20, randGen) - }) - t.Run("n200", func(t *testing.T) { - testTableReadRandomNodesGetAllRun(t, 200, randGen) - }) +type closeTest struct { + Self enode.ID + Target enode.ID + All []*enode.Node + N int } -func generateNodes(rand *rand.Rand, count int) []*node { - nodes := make([]*node, 0, count) - for i := 0; i < count; i++ { - nodes = append(nodes, generateNode(rand)) +func (*closeTest) Generate(rand *rand.Rand, size int) reflect.Value { + t := &closeTest{ + Self: gen(enode.ID{}, rand).(enode.ID), + Target: gen(enode.ID{}, rand).(enode.ID), + N: rand.Intn(bucketSize), } - return nodes -} - -func generateNode(rand *rand.Rand) *node { - var id enode.ID - rand.Read(id[:]) - - r := new(enr.Record) - r.Set(enr.IP(genIP(rand))) - - n := wrapNode(enode.SignNull(r, id)) - n.livenessChecks = 1 - return n + for _, id := range gen([]enode.ID{}, rand).([]enode.ID) { + r := new(enr.Record) + r.Set(enr.IPv4Addr(netutil.RandomAddr(rand, true))) + n := enode.SignNull(r, id) + t.All = append(t.All, n) + } + return reflect.ValueOf(t) } -func TestTable_addVerifiedNode(t *testing.T) { - tmpDir := t.TempDir() - tab, db := newTestTable(newPingRecorder(), tmpDir, log.Root()) +func TestTable_addInboundNode(t *testing.T) { + tab, db := newTestTable(newPingRecorder(), Config{}) <-tab.initDone defer db.Close() defer tab.close() @@ -408,32 +284,29 @@ func TestTable_addVerifiedNode(t *testing.T) { // Insert two nodes. n1 := nodeAtDistance(tab.self().ID(), 256, net.IP{88, 77, 66, 1}) n2 := nodeAtDistance(tab.self().ID(), 256, net.IP{88, 77, 66, 2}) - tab.addSeenNode(n1) - tab.addSeenNode(n2) + tab.addFoundNode(n1, false) + tab.addFoundNode(n2, false) + checkBucketContent(t, tab, []*enode.Node{n1, n2}) - // Verify bucket content: - bcontent := []*node{n1, n2} - if !reflect.DeepEqual(tab.bucket(n1.ID()).entries, bcontent) { - t.Fatalf("wrong bucket content: %v", tab.bucket(n1.ID()).entries) - } - - // Add a changed version of n2. + // Add a changed version of n2. The bucket should be updated. newrec := n2.Record() newrec.Set(enr.IP{99, 99, 99, 99}) - newn2 := wrapNode(enode.SignNull(newrec, n2.ID())) - tab.addVerifiedNode(newn2) - - // Check that bucket is updated correctly. - newBcontent := []*node{newn2, n1} - if !reflect.DeepEqual(tab.bucket(n1.ID()).entries, newBcontent) { - t.Fatalf("wrong bucket content after update: %v", tab.bucket(n1.ID()).entries) - } - checkIPLimitInvariant(t, tab) + n2v2 := enode.SignNull(newrec, n2.ID()) + tab.addInboundNode(n2v2) + checkBucketContent(t, tab, []*enode.Node{n1, n2v2}) + + // Try updating n2 without sequence number change. The update is accepted + // because it's inbound. + newrec = n2.Record() + newrec.Set(enr.IP{100, 100, 100, 100}) + newrec.SetSeq(n2.Seq()) + n2v3 := enode.SignNull(newrec, n2.ID()) + tab.addInboundNode(n2v3) + checkBucketContent(t, tab, []*enode.Node{n1, n2v3}) } -func TestTable_addSeenNode(t *testing.T) { - tmpDir := t.TempDir() - tab, db := newTestTable(newPingRecorder(), tmpDir, log.Root()) +func TestTable_addFoundNode(t *testing.T) { + tab, db := newTestTable(newPingRecorder(), Config{}) <-tab.initDone defer db.Close() defer tab.close() @@ -441,25 +314,86 @@ func TestTable_addSeenNode(t *testing.T) { // Insert two nodes. n1 := nodeAtDistance(tab.self().ID(), 256, net.IP{88, 77, 66, 1}) n2 := nodeAtDistance(tab.self().ID(), 256, net.IP{88, 77, 66, 2}) - tab.addSeenNode(n1) - tab.addSeenNode(n2) - - // Verify bucket content: - bcontent := []*node{n1, n2} - if !reflect.DeepEqual(tab.bucket(n1.ID()).entries, bcontent) { - t.Fatalf("wrong bucket content: %v", tab.bucket(n1.ID()).entries) - } + tab.addFoundNode(n1, false) + tab.addFoundNode(n2, false) + checkBucketContent(t, tab, []*enode.Node{n1, n2}) - // Add a changed version of n2. + // Add a changed version of n2. The bucket should be updated. newrec := n2.Record() newrec.Set(enr.IP{99, 99, 99, 99}) - newn2 := wrapNode(enode.SignNull(newrec, n2.ID())) - tab.addSeenNode(newn2) + n2v2 := enode.SignNull(newrec, n2.ID()) + tab.addFoundNode(n2v2, false) + checkBucketContent(t, tab, []*enode.Node{n1, n2v2}) + + // Try updating n2 without a sequence number change. + // The update should not be accepted. + newrec = n2.Record() + newrec.Set(enr.IP{100, 100, 100, 100}) + newrec.SetSeq(n2.Seq()) + n2v3 := enode.SignNull(newrec, n2.ID()) + tab.addFoundNode(n2v3, false) + checkBucketContent(t, tab, []*enode.Node{n1, n2v2}) +} + +// This test checks that discv4 nodes can update their own endpoint via PING. +func TestTable_addInboundNodeUpdateV4Accept(t *testing.T) { + tab, db := newTestTable(newPingRecorder(), Config{}) + <-tab.initDone + defer db.Close() + defer tab.close() + + // Add a v4 node. + key, _ := crypto.HexToECDSA("dd3757a8075e88d0f2b1431e7d3c5b1562e1c0aab9643707e8cbfcc8dae5cfe3") + n1 := enode.NewV4(&key.PublicKey, net.IP{88, 77, 66, 1}, 9000, 9000) + tab.addInboundNode(n1) + checkBucketContent(t, tab, []*enode.Node{n1}) + + // Add an updated version with changed IP. + // The update will be accepted because it is inbound. + n1v2 := enode.NewV4(&key.PublicKey, net.IP{99, 99, 99, 99}, 9000, 9000) + tab.addInboundNode(n1v2) + checkBucketContent(t, tab, []*enode.Node{n1v2}) +} + +// This test checks that discv4 node entries will NOT be updated when a +// changed record is found. +func TestTable_addFoundNodeV4UpdateReject(t *testing.T) { + tab, db := newTestTable(newPingRecorder(), Config{}) + <-tab.initDone + defer db.Close() + defer tab.close() - // Check that bucket content is unchanged. - if !reflect.DeepEqual(tab.bucket(n1.ID()).entries, bcontent) { - t.Fatalf("wrong bucket content after update: %v", tab.bucket(n1.ID()).entries) + // Add a v4 node. + key, _ := crypto.HexToECDSA("dd3757a8075e88d0f2b1431e7d3c5b1562e1c0aab9643707e8cbfcc8dae5cfe3") + n1 := enode.NewV4(&key.PublicKey, net.IP{88, 77, 66, 1}, 9000, 9000) + tab.addFoundNode(n1, false) + checkBucketContent(t, tab, []*enode.Node{n1}) + + // Add an updated version with changed IP. + // The update won't be accepted because it isn't inbound. + n1v2 := enode.NewV4(&key.PublicKey, net.IP{99, 99, 99, 99}, 9000, 9000) + tab.addFoundNode(n1v2, false) + checkBucketContent(t, tab, []*enode.Node{n1}) +} + +func checkBucketContent(t *testing.T, tab *Table, nodes []*enode.Node) { + t.Helper() + + b := tab.bucket(nodes[0].ID()) + if reflect.DeepEqual(unwrapNodes(b.entries), nodes) { + return + } + t.Log("wrong bucket content. have nodes:") + for _, n := range b.entries { + t.Logf(" %v (seq=%v, ip=%v)", n.ID(), n.Seq(), n.IPAddr()) } + t.Log("want nodes:") + for _, n := range nodes { + t.Logf(" %v (seq=%v, ip=%v)", n.ID(), n.Seq(), n.IPAddr()) + } + t.FailNow() + + // Also check IP limits. checkIPLimitInvariant(t, tab) } @@ -467,8 +401,10 @@ func TestTable_addSeenNode(t *testing.T) { // announces a new sequence number, the new record should be pulled. func TestTable_revalidateSyncRecord(t *testing.T) { transport := newPingRecorder() - tmpDir := t.TempDir() - tab, db := newTestTable(transport, tmpDir, log.Root()) + tab, db := newTestTable(transport, Config{ + Clock: new(mclock.Simulated), + Log: testlog.Logger(t, log.LvlTrace), + }) <-tab.initDone defer db.Close() defer tab.close() @@ -477,127 +413,87 @@ func TestTable_revalidateSyncRecord(t *testing.T) { var r enr.Record r.Set(enr.IP(net.IP{127, 0, 0, 1})) id := enode.ID{1} - n1 := wrapNode(enode.SignNull(&r, id)) - tab.addSeenNode(n1) + n1 := enode.SignNull(&r, id) + tab.addFoundNode(n1, false) // Update the node record. r.Set(enr.WithEntry("foo", "bar")) n2 := enode.SignNull(&r, id) transport.updateRecord(n2) - tab.doRevalidate(make(chan struct{}, 1)) + // Wait for revalidation. We wait for the node to be revalidated two times + // in order to synchronize with the update in the table. + waitForRevalidationPing(t, transport, tab, n2.ID()) + waitForRevalidationPing(t, transport, tab, n2.ID()) + intable := tab.getNode(id) if !reflect.DeepEqual(intable, n2) { t.Fatalf("table contains old record with seq %d, want seq %d", intable.Seq(), n2.Seq()) } } -func genIP(rand *rand.Rand) net.IP { - ip := make(net.IP, 4) - rand.Read(ip) - return ip -} - -func newkey() *ecdsa.PrivateKey { - key, err := crypto.GenerateKey() - if err != nil { - panic("couldn't generate key: " + err.Error()) +func TestNodesPush(t *testing.T) { + var target enode.ID + n1 := nodeAtDistance(target, 255, intIP(1)) + n2 := nodeAtDistance(target, 254, intIP(2)) + n3 := nodeAtDistance(target, 253, intIP(3)) + perm := [][]*enode.Node{ + {n3, n2, n1}, + {n3, n1, n2}, + {n2, n3, n1}, + {n2, n1, n3}, + {n1, n3, n2}, + {n1, n2, n3}, } - return key -} -func Benchmark_findnodeByID(b *testing.B) { - benchmarks := []struct { - name string - tableSize int - nresults int - preferLive bool - liveRatio float64 - }{ - {"SmallTable_5Results_NoPreferLive", 50, 5, false, 0.0}, - {"SmallTable_16Results_NoPreferLive", 50, 16, false, 0.0}, - {"SmallTable_5Results_PreferLive_AllLive", 50, 5, true, 1.0}, - {"SmallTable_16Results_PreferLive_AllLive", 50, 16, true, 1.0}, - {"SmallTable_5Results_PreferLive_HalfLive", 50, 5, true, 0.5}, - {"SmallTable_16Results_PreferLive_HalfLive", 50, 16, true, 0.5}, - {"MediumTable_5Results_NoPreferLive", 200, 5, false, 0.0}, - {"MediumTable_16Results_NoPreferLive", 200, 16, false, 0.0}, - {"MediumTable_5Results_PreferLive_AllLive", 200, 5, true, 1.0}, - {"MediumTable_16Results_PreferLive_AllLive", 200, 16, true, 1.0}, - {"MediumTable_5Results_PreferLive_HalfLive", 200, 5, true, 0.5}, - {"MediumTable_16Results_PreferLive_HalfLive", 200, 16, true, 0.5}, - {"LargeTable_5Results_NoPreferLive", 500, 5, false, 0.0}, - {"LargeTable_16Results_NoPreferLive", 500, 16, false, 0.0}, - {"LargeTable_5Results_PreferLive_AllLive", 500, 5, true, 1.0}, - {"LargeTable_16Results_PreferLive_AllLive", 500, 16, true, 1.0}, - {"LargeTable_5Results_PreferLive_HalfLive", 500, 5, true, 0.5}, - {"LargeTable_16Results_PreferLive_HalfLive", 500, 16, true, 0.5}, - {"FullTable_5Results_NoPreferLive", nBuckets * bucketSize, 5, false, 0.0}, - {"FullTable_16Results_NoPreferLive", nBuckets * bucketSize, 16, false, 0.0}, - {"FullTable_5Results_PreferLive_AllLive", nBuckets * bucketSize, 5, true, 1.0}, - {"FullTable_16Results_PreferLive_AllLive", nBuckets * bucketSize, 16, true, 1.0}, + // Insert all permutations into lists with size limit 3. + for _, nodes := range perm { + list := nodesByDistance{target: target} + for _, n := range nodes { + list.push(n, 3) + } + if !slices.EqualFunc(list.entries, perm[0], nodeIDEqual) { + t.Fatal("not equal") + } } - for _, bm := range benchmarks { - b.Run(bm.name, func(b *testing.B) { - transport := newPingRecorder() - tmpDir := b.TempDir() - tab, db := newTestTable(transport, tmpDir, log.Root()) - defer db.Close() - defer tab.close() - - <-tab.initDone - - nodes := generateTestNodes(bm.tableSize, tab.self().ID()) - fillTable(tab, nodes) - - if bm.preferLive && bm.liveRatio > 0 { - setNodesLive(tab, nodes, bm.liveRatio) - } - - target := generateRandomID() - - b.ResetTimer() - b.ReportAllocs() - for b.Loop() { - _ = tab.findnodeByID(target, bm.nresults, bm.preferLive) - } - b.StopTimer() - }) + // Insert all permutations into lists with size limit 2. + for _, nodes := range perm { + list := nodesByDistance{target: target} + for _, n := range nodes { + list.push(n, 2) + } + if !slices.EqualFunc(list.entries, perm[0][:2], nodeIDEqual) { + t.Fatal("not equal") + } } } -func generateTestNodes(count int, baseID enode.ID) []*node { - nodes := make([]*node, count) - for i := 0; i < count; i++ { - var r enr.Record - r.Set(enr.IP(net.IP{10, 0, byte(i >> 8), byte(i & 0xFF)})) - // Generate nodes at various distances - distance := i % 256 - nodeID := idAtDistance(baseID, distance) - nodes[i] = wrapNode(enode.SignNull(&r, nodeID)) - } - return nodes +func nodeIDEqual[N nodeType](n1, n2 N) bool { + return n1.ID() == n2.ID() } -func setNodesLive(tab *Table, nodes []*node, liveRatio float64) { - tab.mutex.Lock() - defer tab.mutex.Unlock() +// gen wraps quick.Value so it's easier to use. +// it generates a random value of the given value's type. +func gen(typ interface{}, rand *rand.Rand) interface{} { + v, ok := quick.Value(reflect.TypeOf(typ), rand) + if !ok { + panic(fmt.Sprintf("couldn't generate random value of type %T", typ)) + } + return v.Interface() +} - liveCount := int(float64(len(nodes)) * liveRatio) - for i := 0; i < liveCount && i < len(nodes); i++ { - b := tab.bucket(nodes[i].ID()) - for _, n := range b.entries { - if n.ID() == nodes[i].ID() { - n.livenessChecks = 1 - break - } - } +func quickcfg() *quick.Config { + return &quick.Config{ + MaxCount: 5000, + Rand: rand.New(rand.NewSource(time.Now().Unix())), } } -func generateRandomID() enode.ID { - var id enode.ID - rand.Read(id[:]) - return id +func newkey() *ecdsa.PrivateKey { + key, err := crypto.GenerateKey() + if err != nil { + panic("couldn't generate key: " + err.Error()) + } + return key } diff --git a/p2p/discover/table_util_test.go b/p2p/discover/table_util_test.go index 63fa87e0fe3..b9d42303a59 100644 --- a/p2p/discover/table_util_test.go +++ b/p2p/discover/table_util_test.go @@ -1,39 +1,36 @@ // Copyright 2018 The go-ethereum Authors -// (original work) -// Copyright 2024 The Erigon Authors -// (modifications) -// This file is part of Erigon. +// This file is part of the go-ethereum library. // -// Erigon is free software: you can redistribute it and/or modify +// The go-ethereum library is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // -// Erigon is distributed in the hope that it will be useful, +// The go-ethereum library is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Lesser General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License -// along with Erigon. If not, see . +// along with the go-ethereum library. If not, see . package discover import ( "bytes" - "context" "crypto/ecdsa" "encoding/hex" "errors" "fmt" "math/rand" "net" - "sort" + "slices" "sync" + "sync/atomic" "time" "github.com/erigontech/erigon/common/crypto" - "github.com/erigontech/erigon/common/log/v3" + "github.com/erigontech/erigon/p2p/discover/v4wire" "github.com/erigontech/erigon/p2p/enode" "github.com/erigontech/erigon/p2p/enr" ) @@ -46,35 +43,32 @@ func init() { nullNode = enode.SignNull(&r, enode.ID{}) } -func newTestTable(t transport, tmpDir string, logger log.Logger) (*Table, *enode.DB) { - db, err := enode.OpenDB(context.Background(), "", tmpDir, logger) - if err != nil { - panic(err) - } - tab, _ := newTable(t, "test", db, nil, time.Hour, log.Root()) +func newTestTable(t transport, cfg Config) (*Table, *enode.DB) { + tab, db := newInactiveTestTable(t, cfg) go tab.loop() return tab, db } +// newInactiveTestTable creates a Table without running the main loop. +func newInactiveTestTable(t transport, cfg Config) (*Table, *enode.DB) { + db, _ := enode.OpenDB("") + tab, _ := newTable(t, db, cfg) + return tab, db +} + // nodeAtDistance creates a node for which enode.LogDist(base, n.id) == ld. -func nodeAtDistance(base enode.ID, ld int, ip net.IP) *node { +func nodeAtDistance(base enode.ID, ld int, ip net.IP) *enode.Node { var r enr.Record r.Set(enr.IP(ip)) - return wrapNode(enode.SignNull(&r, idAtDistance(base, ld))) + r.Set(enr.UDP(30303)) + return enode.SignNull(&r, idAtDistance(base, ld)) } // nodesAtDistance creates n nodes for which enode.LogDist(base, node.ID()) == ld. func nodesAtDistance(base enode.ID, ld int, n int) []*enode.Node { - results := make([]*enode.Node, 0, n) - nodeSet := make(map[enode.ID]bool, n) - for len(results) < n { - node := unwrapNode(nodeAtDistance(base, ld, intIP(len(results)+1))) - // idAtDistance might return an ID that's already generated - // make sure that the node has a unique ID, otherwise regenerate - if !nodeSet[node.ID()] { - nodeSet[node.ID()] = true - results = append(results, node) - } + results := make([]*enode.Node, n) + for i := range results { + results[i] = nodeAtDistance(base, ld, intIP(i)) } return results } @@ -107,33 +101,39 @@ func idAtDistance(a enode.ID, n int) (b enode.ID) { return b } +// intIP returns a LAN IP address based on i. func intIP(i int) net.IP { - return net.IP{byte(i), 0, 2, byte(i)} + return net.IP{10, 0, byte(i >> 8), byte(i & 0xFF)} } // fillBucket inserts nodes into the given bucket until it is full. -func fillBucket(tab *Table, n *node) (last *node) { - ld := enode.LogDist(tab.self().ID(), n.ID()) - b := tab.bucket(n.ID()) +func fillBucket(tab *Table, id enode.ID) (last *tableNode) { + ld := enode.LogDist(tab.self().ID(), id) + b := tab.bucket(id) for len(b.entries) < bucketSize { - b.entries = append(b.entries, nodeAtDistance(tab.self().ID(), ld, intIP(ld))) + node := nodeAtDistance(tab.self().ID(), ld, intIP(ld)) + if !tab.addFoundNode(node, false) { + panic("node not added") + } } return b.entries[bucketSize-1] } // fillTable adds nodes the table to the end of their corresponding bucket // if the bucket is not full. The caller must not hold tab.mutex. -func fillTable(tab *Table, nodes []*node) { +func fillTable(tab *Table, nodes []*enode.Node, setLive bool) { for _, n := range nodes { - tab.addSeenNode(n) + tab.addFoundNode(n, setLive) } } type pingRecorder struct { - mu sync.Mutex - dead, pinged map[enode.ID]bool - records map[enode.ID]*enode.Node - n *enode.Node + mu sync.Mutex + cond *sync.Cond + dead map[enode.ID]bool + records map[enode.ID]*enode.Node + pinged []*enode.Node + n *enode.Node } func newPingRecorder() *pingRecorder { @@ -141,16 +141,17 @@ func newPingRecorder() *pingRecorder { r.Set(enr.IP{0, 0, 0, 0}) n := enode.SignNull(&r, enode.ID{}) - return &pingRecorder{ + t := &pingRecorder{ dead: make(map[enode.ID]bool), - pinged: make(map[enode.ID]bool), records: make(map[enode.ID]*enode.Node), n: n, } + t.cond = sync.NewCond(&t.mu) + return t } -// setRecord updates a node record. Future calls to ping and -// requestENR will return this record. +// updateRecord updates a node record. Future calls to ping and +// RequestENR will return this record. func (t *pingRecorder) updateRecord(n *enode.Node) { t.mu.Lock() defer t.mu.Unlock() @@ -159,18 +160,43 @@ func (t *pingRecorder) updateRecord(n *enode.Node) { // Stubs to satisfy the transport interface. func (t *pingRecorder) Self() *enode.Node { return nullNode } -func (t *pingRecorder) Version() string { return "none" } -func (t *pingRecorder) Errors() map[string]uint { return nil } -func (t *pingRecorder) LenUnsolicited() int { return 0 } func (t *pingRecorder) lookupSelf() []*enode.Node { return nil } func (t *pingRecorder) lookupRandom() []*enode.Node { return nil } +func (t *pingRecorder) waitPing(timeout time.Duration) *enode.Node { + t.mu.Lock() + defer t.mu.Unlock() + + // Wake up the loop on timeout. + var timedout atomic.Bool + timer := time.AfterFunc(timeout, func() { + timedout.Store(true) + t.cond.Broadcast() + }) + defer timer.Stop() + + // Wait for a ping. + for { + if timedout.Load() { + return nil + } + if len(t.pinged) > 0 { + n := t.pinged[0] + t.pinged = append(t.pinged[:0], t.pinged[1:]...) + return n + } + t.cond.Wait() + } +} + // ping simulates a ping request. func (t *pingRecorder) ping(n *enode.Node) (seq uint64, err error) { t.mu.Lock() defer t.mu.Unlock() - t.pinged[n.ID()] = true + t.pinged = append(t.pinged, n) + t.cond.Broadcast() + if t.dead[n.ID()] { return 0, errTimeout } @@ -180,7 +206,7 @@ func (t *pingRecorder) ping(n *enode.Node) (seq uint64, err error) { return seq, nil } -// requestENR simulates an ENR request. +// RequestENR simulates an ENR request. func (t *pingRecorder) RequestENR(n *enode.Node) (*enode.Node, error) { t.mu.Lock() defer t.mu.Unlock() @@ -191,8 +217,8 @@ func (t *pingRecorder) RequestENR(n *enode.Node) (*enode.Node, error) { return t.records[n.ID()], nil } -func hasDuplicates(slice []*node) bool { - seen := make(map[enode.ID]bool) +func hasDuplicates(slice []*enode.Node) bool { + seen := make(map[enode.ID]bool, len(slice)) for i, e := range slice { if e == nil { panic(fmt.Sprintf("nil *Node at %d", i)) @@ -230,18 +256,18 @@ NotEqual: } func nodeEqual(n1 *enode.Node, n2 *enode.Node) bool { - return n1.ID() == n2.ID() && n1.IP().Equal(n2.IP()) + return n1.ID() == n2.ID() && n1.IPAddr() == n2.IPAddr() } -func sortByID(nodes []*enode.Node) { - sort.Slice(nodes, func(i, j int) bool { - return string(nodes[i].ID().Bytes()) < string(nodes[j].ID().Bytes()) +func sortByID[N nodeType](nodes []N) { + slices.SortFunc(nodes, func(a, b N) int { + return bytes.Compare(a.ID().Bytes(), b.ID().Bytes()) }) } -func sortedByDistanceTo(distbase enode.ID, slice []*node) bool { - return sort.SliceIsSorted(slice, func(i, j int) bool { - return enode.DistCmp(distbase, slice[i].ID(), slice[j].ID()) < 0 +func sortedByDistanceTo(distbase enode.ID, slice []*enode.Node) bool { + return slices.IsSortedFunc(slice, func(a, b *enode.Node) int { + return enode.DistCmp(distbase, a.ID(), b.ID()) }) } @@ -259,7 +285,7 @@ func hexEncPrivkey(h string) *ecdsa.PrivateKey { } // hexEncPubkey decodes h as a public key. -func hexEncPubkey(h string) (ret enode.PubkeyEncoded) { +func hexEncPubkey(h string) (ret v4wire.Pubkey) { b, err := hex.DecodeString(h) if err != nil { panic(err) @@ -270,3 +296,57 @@ func hexEncPubkey(h string) (ret enode.PubkeyEncoded) { copy(ret[:], b) return ret } + +type nodeEventRecorder struct { + evc chan recordedNodeEvent +} + +type recordedNodeEvent struct { + node *tableNode + added bool +} + +func newNodeEventRecorder(buffer int) *nodeEventRecorder { + return &nodeEventRecorder{ + evc: make(chan recordedNodeEvent, buffer), + } +} + +func (set *nodeEventRecorder) nodeAdded(b *bucket, n *tableNode) { + select { + case set.evc <- recordedNodeEvent{n, true}: + default: + panic("no space in event buffer") + } +} + +func (set *nodeEventRecorder) nodeRemoved(b *bucket, n *tableNode) { + select { + case set.evc <- recordedNodeEvent{n, false}: + default: + panic("no space in event buffer") + } +} + +func (set *nodeEventRecorder) waitNodePresent(id enode.ID, timeout time.Duration) bool { + return set.waitNodeEvent(id, timeout, true) +} + +func (set *nodeEventRecorder) waitNodeAbsent(id enode.ID, timeout time.Duration) bool { + return set.waitNodeEvent(id, timeout, false) +} + +func (set *nodeEventRecorder) waitNodeEvent(id enode.ID, timeout time.Duration, added bool) bool { + timer := time.NewTimer(timeout) + defer timer.Stop() + for { + select { + case ev := <-set.evc: + if ev.node.ID() == id && ev.added == added { + return true + } + case <-timer.C: + return false + } + } +} diff --git a/p2p/discover/v4_lookup_test.go b/p2p/discover/v4_lookup_test.go index c9ac0c7219e..49e0f07462d 100644 --- a/p2p/discover/v4_lookup_test.go +++ b/p2p/discover/v4_lookup_test.go @@ -1,64 +1,60 @@ // Copyright 2019 The go-ethereum Authors -// (original work) -// Copyright 2024 The Erigon Authors -// (modifications) -// This file is part of Erigon. +// This file is part of the go-ethereum library. // -// Erigon is free software: you can redistribute it and/or modify +// The go-ethereum library is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // -// Erigon is distributed in the hope that it will be useful, +// The go-ethereum library is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Lesser General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License -// along with Erigon. If not, see . +// along with the go-ethereum library. If not, see . package discover import ( - "context" "crypto/ecdsa" - "net" + "fmt" + "net/netip" + "slices" + "sync" "testing" - "time" "github.com/erigontech/erigon/common/crypto" - "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/p2p/discover/v4wire" "github.com/erigontech/erigon/p2p/enode" + "github.com/erigontech/erigon/p2p/enr" ) func TestUDPv4_Lookup(t *testing.T) { - t.Skip("issue #14714") t.Parallel() - logger := log.New() - - ctx := context.Background() - ctx = contextWithReplyTimeout(ctx, time.Second) - - test := newUDPTestContext(ctx, t, logger) - defer test.close() + test := newUDPTest(t) // Lookup on empty table returns no nodes. - targetKey, _ := v4wire.DecodePubkey(crypto.S256(), v4wire.Pubkey(lookupTestnet.target)) + targetKey, _ := v4wire.DecodePubkey(crypto.S256(), lookupTestnet.target) if results := test.udp.LookupPubkey(targetKey); len(results) > 0 { t.Fatalf("lookup on empty table returned %d results: %#v", len(results), results) } // Seed table with initial node. - fillTable(test.table, []*node{wrapNode(lookupTestnet.node(256, 0))}) - - // Answer lookup packets. - go serveTestnet(test, lookupTestnet) + fillTable(test.table, []*enode.Node{lookupTestnet.node(256, 0)}, true) // Start the lookup. - results := test.udp.LookupPubkey(targetKey) + resultC := make(chan []*enode.Node, 1) + go func() { + resultC <- test.udp.LookupPubkey(targetKey) + test.close() + }() + + // Answer lookup packets. + serveTestnet(test, lookupTestnet) // Verify result nodes. + results := <-resultC t.Logf("results:") for _, e := range results { t.Logf(" ld=%d, %x", enode.LogDist(lookupTestnet.target.ID(), e.ID()), e.ID().Bytes()) @@ -70,31 +66,25 @@ func TestUDPv4_Lookup(t *testing.T) { } func TestUDPv4_LookupIterator(t *testing.T) { - t.Skip("issue #14924") t.Parallel() - logger := log.New() - - // Set up RandomNodes() to use expected keys instead of generating random ones. - testNetPrivateKeys := lookupTestnet.privateKeys() - testNetPrivateKeyIndex := -1 - privateKeyGenerator := func() (*ecdsa.PrivateKey, error) { - testNetPrivateKeyIndex = (testNetPrivateKeyIndex + 1) % len(testNetPrivateKeys) - return testNetPrivateKeys[testNetPrivateKeyIndex], nil - } - ctx := context.Background() - ctx = contextWithReplyTimeout(ctx, time.Second) - ctx = contextWithPrivateKeyGenerator(ctx, privateKeyGenerator) - - test := newUDPTestContext(ctx, t, logger) - defer test.close() + test := newUDPTest(t) + var wg sync.WaitGroup + defer func() { + test.close() + wg.Wait() + }() // Seed table with initial nodes. - bootnodes := make([]*node, len(lookupTestnet.dists[256])) + bootnodes := make([]*enode.Node, len(lookupTestnet.dists[256])) for i := range lookupTestnet.dists[256] { - bootnodes[i] = wrapNode(lookupTestnet.node(256, i)) + bootnodes[i] = lookupTestnet.node(256, i) } - fillTable(test.table, bootnodes) - go serveTestnet(test, lookupTestnet) + fillTable(test.table, bootnodes, true) + wg.Add(1) + go func() { + serveTestnet(test, lookupTestnet) + wg.Done() + }() // Create the iterator and collect the nodes it yields. iter := test.udp.RandomNodes() @@ -120,17 +110,25 @@ func TestUDPv4_LookupIterator(t *testing.T) { // method is called. func TestUDPv4_LookupIteratorClose(t *testing.T) { t.Parallel() - logger := log.New() - test := newUDPTest(t, logger) - defer test.close() + test := newUDPTest(t) + var wg sync.WaitGroup + defer func() { + test.close() + wg.Wait() + }() // Seed table with initial nodes. - bootnodes := make([]*node, len(lookupTestnet.dists[256])) + bootnodes := make([]*enode.Node, len(lookupTestnet.dists[256])) for i := range lookupTestnet.dists[256] { - bootnodes[i] = wrapNode(lookupTestnet.node(256, i)) + bootnodes[i] = lookupTestnet.node(256, i) } - fillTable(test.table, bootnodes) - go serveTestnet(test, lookupTestnet) + fillTable(test.table, bootnodes, true) + + wg.Add(1) + go func() { + serveTestnet(test, lookupTestnet) + wg.Done() + }() it := test.udp.RandomNodes() if ok := it.Next(); !ok || it.Node() == nil { @@ -156,7 +154,7 @@ func TestUDPv4_LookupIteratorClose(t *testing.T) { func serveTestnet(test *udpTest, testnet *preminedTestnet) { for done := false; !done; { - done = test.waitPacketOut(func(p v4wire.Packet, to *net.UDPAddr, hash []byte) { + done = test.waitPacketOut(func(p v4wire.Packet, to netip.AddrPort, hash []byte) { n, key := testnet.nodeByAddr(to) switch p.(type) { case *v4wire.Ping: @@ -169,3 +167,200 @@ func serveTestnet(test *udpTest, testnet *preminedTestnet) { }) } } + +// checkLookupResults verifies that the results of a lookup are the closest nodes to +// the testnet's target. +func checkLookupResults(t *testing.T, tn *preminedTestnet, results []*enode.Node) { + t.Helper() + t.Logf("results:") + for _, e := range results { + t.Logf(" ld=%d, %x", enode.LogDist(tn.target.ID(), e.ID()), e.ID().Bytes()) + } + if hasDuplicates(results) { + t.Errorf("result set contains duplicate entries") + } + if !sortedByDistanceTo(tn.target.ID(), results) { + t.Errorf("result set not sorted by distance to target") + } + wantNodes := tn.closest(len(results)) + if err := checkNodesEqual(results, wantNodes); err != nil { + t.Error(err) + } +} + +// This is the test network for the Lookup test. +// The nodes were obtained by running lookupTestnet.mine with a random NodeID as target. +var lookupTestnet = &preminedTestnet{ + target: hexEncPubkey("5d485bdcbe9bc89314a10ae9231e429d33853e3a8fa2af39f5f827370a2e4185e344ace5d16237491dad41f278f1d3785210d29ace76cd627b9147ee340b1125"), + dists: [257][]*ecdsa.PrivateKey{ + 251: { + hexEncPrivkey("29738ba0c1a4397d6a65f292eee07f02df8e58d41594ba2be3cf84ce0fc58169"), + hexEncPrivkey("511b1686e4e58a917f7f848e9bf5539d206a68f5ad6b54b552c2399fe7d174ae"), + hexEncPrivkey("d09e5eaeec0fd596236faed210e55ef45112409a5aa7f3276d26646080dcfaeb"), + hexEncPrivkey("c1e20dbbf0d530e50573bd0a260b32ec15eb9190032b4633d44834afc8afe578"), + hexEncPrivkey("ed5f38f5702d92d306143e5d9154fb21819777da39af325ea359f453d179e80b"), + }, + 252: { + hexEncPrivkey("1c9b1cafbec00848d2c174b858219914b42a7d5c9359b1ca03fd650e8239ae94"), + hexEncPrivkey("e0e1e8db4a6f13c1ffdd3e96b72fa7012293ced187c9dcdcb9ba2af37a46fa10"), + hexEncPrivkey("3d53823e0a0295cb09f3e11d16c1b44d07dd37cec6f739b8df3a590189fe9fb9"), + }, + 253: { + hexEncPrivkey("2d0511ae9bf590166597eeab86b6f27b1ab761761eaea8965487b162f8703847"), + hexEncPrivkey("6cfbd7b8503073fc3dbdb746a7c672571648d3bd15197ccf7f7fef3d904f53a2"), + hexEncPrivkey("a30599b12827b69120633f15b98a7f6bc9fc2e9a0fd6ae2ebb767c0e64d743ab"), + hexEncPrivkey("14a98db9b46a831d67eff29f3b85b1b485bb12ae9796aea98d91be3dc78d8a91"), + hexEncPrivkey("2369ff1fc1ff8ca7d20b17e2673adc3365c3674377f21c5d9dafaff21fe12e24"), + hexEncPrivkey("9ae91101d6b5048607f41ec0f690ef5d09507928aded2410aabd9237aa2727d7"), + hexEncPrivkey("05e3c59090a3fd1ae697c09c574a36fcf9bedd0afa8fe3946f21117319ca4973"), + hexEncPrivkey("06f31c5ea632658f718a91a1b1b9ae4b7549d7b3bc61cbc2be5f4a439039f3ad"), + }, + 254: { + hexEncPrivkey("dec742079ec00ff4ec1284d7905bc3de2366f67a0769431fd16f80fd68c58a7c"), + hexEncPrivkey("ff02c8861fa12fbd129d2a95ea663492ef9c1e51de19dcfbbfe1c59894a28d2b"), + hexEncPrivkey("4dded9e4eefcbce4262be4fd9e8a773670ab0b5f448f286ec97dfc8cf681444a"), + hexEncPrivkey("750d931e2a8baa2c9268cb46b7cd851f4198018bed22f4dceb09dd334a2395f6"), + hexEncPrivkey("ce1435a956a98ffec484cd11489c4f165cf1606819ab6b521cee440f0c677e9e"), + hexEncPrivkey("996e7f8d1638be92d7328b4770f47e5420fc4bafecb4324fd33b1f5d9f403a75"), + hexEncPrivkey("ebdc44e77a6cc0eb622e58cf3bb903c3da4c91ca75b447b0168505d8fc308b9c"), + hexEncPrivkey("46bd1eddcf6431bea66fc19ebc45df191c1c7d6ed552dcdc7392885009c322f0"), + }, + 255: { + hexEncPrivkey("da8645f90826e57228d9ea72aff84500060ad111a5d62e4af831ed8e4b5acfb8"), + hexEncPrivkey("3c944c5d9af51d4c1d43f5d0f3a1a7ef65d5e82744d669b58b5fed242941a566"), + hexEncPrivkey("5ebcde76f1d579eebf6e43b0ffe9157e65ffaa391175d5b9aa988f47df3e33da"), + hexEncPrivkey("97f78253a7d1d796e4eaabce721febcc4550dd68fb11cc818378ba807a2cb7de"), + hexEncPrivkey("a38cd7dc9b4079d1c0406afd0fdb1165c285f2c44f946eca96fc67772c988c7d"), + hexEncPrivkey("d64cbb3ffdf712c372b7a22a176308ef8f91861398d5dbaf326fd89c6eaeef1c"), + hexEncPrivkey("d269609743ef29d6446e3355ec647e38d919c82a4eb5837e442efd7f4218944f"), + hexEncPrivkey("d8f7bcc4a530efde1d143717007179e0d9ace405ddaaf151c4d863753b7fd64c"), + }, + 256: { + hexEncPrivkey("8c5b422155d33ea8e9d46f71d1ad3e7b24cb40051413ffa1a81cff613d243ba9"), + hexEncPrivkey("937b1af801def4e8f5a3a8bd225a8bcff1db764e41d3e177f2e9376e8dd87233"), + hexEncPrivkey("120260dce739b6f71f171da6f65bc361b5fad51db74cf02d3e973347819a6518"), + hexEncPrivkey("1fa56cf25d4b46c2bf94e82355aa631717b63190785ac6bae545a88aadc304a9"), + hexEncPrivkey("3c38c503c0376f9b4adcbe935d5f4b890391741c764f61b03cd4d0d42deae002"), + hexEncPrivkey("3a54af3e9fa162bc8623cdf3e5d9b70bf30ade1d54cc3abea8659aba6cff471f"), + hexEncPrivkey("6799a02ea1999aefdcbcc4d3ff9544478be7365a328d0d0f37c26bd95ade0cda"), + hexEncPrivkey("e24a7bc9051058f918646b0f6e3d16884b2a55a15553b89bab910d55ebc36116"), + }, + }, +} + +type preminedTestnet struct { + target v4wire.Pubkey + dists [hashBits + 1][]*ecdsa.PrivateKey +} + +func (tn *preminedTestnet) len() int { + n := 0 + for _, keys := range tn.dists { + n += len(keys) + } + return n +} + +func (tn *preminedTestnet) nodes() []*enode.Node { + result := make([]*enode.Node, 0, tn.len()) + for dist, keys := range tn.dists { + for index := range keys { + result = append(result, tn.node(dist, index)) + } + } + sortByID(result) + return result +} + +func (tn *preminedTestnet) node(dist, index int) *enode.Node { + key := tn.dists[dist][index] + rec := new(enr.Record) + rec.Set(enr.IP{127, byte(dist >> 8), byte(dist), byte(index)}) + rec.Set(enr.UDP(5000)) + enode.SignV4(rec, key) + n, _ := enode.New(enode.ValidSchemes, rec) + return n +} + +func (tn *preminedTestnet) nodeByAddr(addr netip.AddrPort) (*enode.Node, *ecdsa.PrivateKey) { + ip := addr.Addr().As4() + dist := int(ip[1])<<8 + int(ip[2]) + index := int(ip[3]) + key := tn.dists[dist][index] + return tn.node(dist, index), key +} + +func (tn *preminedTestnet) nodesAtDistance(dist int) []v4wire.Node { + result := make([]v4wire.Node, len(tn.dists[dist])) + for i := range result { + result[i] = nodeToRPC(tn.node(dist, i)) + } + return result +} + +func (tn *preminedTestnet) neighborsAtDistances(base *enode.Node, distances []uint, elems int) []*enode.Node { + var result []*enode.Node + for d := range lookupTestnet.dists { + for i := range lookupTestnet.dists[d] { + n := lookupTestnet.node(d, i) + d := enode.LogDist(base.ID(), n.ID()) + if slices.Contains(distances, uint(d)) { + result = append(result, n) + if len(result) >= elems { + return result + } + } + } + } + return result +} + +func (tn *preminedTestnet) closest(n int) (nodes []*enode.Node) { + for d := range tn.dists { + for i := range tn.dists[d] { + nodes = append(nodes, tn.node(d, i)) + } + } + slices.SortFunc(nodes, func(a, b *enode.Node) int { + return enode.DistCmp(tn.target.ID(), a.ID(), b.ID()) + }) + return nodes[:n] +} + +var _ = (*preminedTestnet).mine // avoid linter warning about mine being dead code. + +// mine generates a testnet struct literal with nodes at +// various distances to the network's target. +func (tn *preminedTestnet) mine() { + // Clear existing slices first (useful when re-mining). + for i := range tn.dists { + tn.dists[i] = nil + } + + targetSha := tn.target.ID() + found, need := 0, 40 + for found < need { + k := newkey() + ld := enode.LogDist(targetSha, v4wire.EncodePubkey(&k.PublicKey).ID()) + if len(tn.dists[ld]) < 8 { + tn.dists[ld] = append(tn.dists[ld], k) + found++ + fmt.Printf("found ID with ld %d (%d/%d)\n", ld, found, need) + } + } + fmt.Printf("&preminedTestnet{\n") + fmt.Printf(" target: hexEncPubkey(\"%x\"),\n", tn.target[:]) + fmt.Printf(" dists: [%d][]*ecdsa.PrivateKey{\n", len(tn.dists)) + for ld, ns := range tn.dists { + if len(ns) == 0 { + continue + } + fmt.Printf(" %d: {\n", ld) + for _, key := range ns { + fmt.Printf(" hexEncPrivkey(\"%x\"),\n", crypto.FromECDSA(key)) + } + fmt.Printf(" },\n") + } + fmt.Printf(" },\n") + fmt.Printf("}\n") +} diff --git a/p2p/discover/v4_udp.go b/p2p/discover/v4_udp.go index ab717eae427..3332739dc2d 100644 --- a/p2p/discover/v4_udp.go +++ b/p2p/discover/v4_udp.go @@ -1,21 +1,18 @@ // Copyright 2019 The go-ethereum Authors -// (original work) -// Copyright 2024 The Erigon Authors -// (modifications) -// This file is part of Erigon. +// This file is part of the go-ethereum library. // -// Erigon is free software: you can redistribute it and/or modify +// The go-ethereum library is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // -// Erigon is distributed in the hope that it will be useful, +// The go-ethereum library is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Lesser General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License -// along with Erigon. If not, see . +// along with the go-ethereum library. If not, see . package discover @@ -24,18 +21,15 @@ import ( "container/list" "context" "crypto/ecdsa" + crand "crypto/rand" "errors" "fmt" "io" - "maps" - "net" + "net/netip" "sync" "time" - lru "github.com/hashicorp/golang-lru/v2" - "github.com/erigontech/erigon/common/crypto" - "github.com/erigontech/erigon/common/dbg" "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/p2p/discover/v4wire" "github.com/erigontech/erigon/p2p/enode" @@ -51,16 +45,11 @@ var ( errClockWarp = errors.New("reply deadline too far in the future") errClosed = errors.New("socket closed") errLowPort = errors.New("low port") -) - -var ( - errExpiredStr = errExpired.Error() - errUnsolicitedReplyStr = errUnsolicitedReply.Error() - errUnknownNodeStr = errUnknownNode.Error() + errNoUDPEndpoint = errors.New("node has no UDP endpoint") ) const ( - respTimeout = 750 * time.Millisecond + respTimeout = 500 * time.Millisecond expiration = 20 * time.Second bondExpiration = 24 * time.Hour @@ -77,7 +66,6 @@ const ( // UDPv4 implements the v4 wire protocol. type UDPv4 struct { - mutex sync.Mutex conn UDPConn log log.Logger netrestrict *netutil.Netlist @@ -88,21 +76,10 @@ type UDPv4 struct { closeOnce sync.Once wg sync.WaitGroup - addReplyMatcher chan *replyMatcher - addReplyMatcherMutex sync.Mutex - - gotreply chan reply - gotkey chan v4wire.Pubkey - gotnodes chan nodes - replyTimeout time.Duration - pingBackDelay time.Duration - closeCtx context.Context - cancelCloseCtx context.CancelFunc - errors map[string]uint - unsolicitedNodes *lru.Cache[enode.ID, *enode.Node] - privateKeyGenerator func() (*ecdsa.PrivateKey, error) - - trace bool + addReplyMatcher chan *replyMatcher + gotreply chan reply + closeCtx context.Context + cancelCloseCtx context.CancelFunc } // replyMatcher represents a pending reply. @@ -117,8 +94,7 @@ type UDPv4 struct { type replyMatcher struct { // these fields must match in the reply. from enode.ID - ip net.IP - port int + ip netip.Addr ptype byte // time when the request must complete @@ -144,45 +120,30 @@ type replyMatchFunc func(v4wire.Packet) (matched bool, requestDone bool) // reply is a reply packet from a certain node. type reply struct { from enode.ID - ip net.IP - port int + ip netip.Addr data v4wire.Packet // loop indicates whether there was // a matching request by sending on this channel. matched chan<- bool } -type nodes struct { - addr *net.UDPAddr - nodes []v4wire.Node -} - -func ListenV4(ctx context.Context, protocol string, c UDPConn, ln *enode.LocalNode, cfg Config) (*UDPv4, error) { - cfg = cfg.withDefaults(respTimeout) - closeCtx, cancel := context.WithCancel(ctx) - unsolicitedNodes, _ := lru.New[enode.ID, *enode.Node](500) - +func ListenV4(c UDPConn, ln *enode.LocalNode, cfg Config) (*UDPv4, error) { + cfg = cfg.withDefaults() + closeCtx, cancel := context.WithCancel(context.Background()) t := &UDPv4{ - conn: c, - priv: cfg.PrivateKey, - netrestrict: cfg.NetRestrict, - localNode: ln, - db: ln.Database(), - gotreply: make(chan reply, 10), - addReplyMatcher: make(chan *replyMatcher, 10), - gotkey: make(chan v4wire.Pubkey, 10), - gotnodes: make(chan nodes, 10), - replyTimeout: cfg.ReplyTimeout, - pingBackDelay: cfg.PingBackDelay, - closeCtx: closeCtx, - cancelCloseCtx: cancel, - log: cfg.Log, - errors: map[string]uint{}, - unsolicitedNodes: unsolicitedNodes, - privateKeyGenerator: cfg.PrivateKeyGenerator, - } - - tab, err := newTable(t, protocol, ln.Database(), cfg.Bootnodes, cfg.TableRevalidateInterval, cfg.Log) + conn: newMeteredConn(c), + priv: cfg.PrivateKey, + netrestrict: cfg.NetRestrict, + localNode: ln, + db: ln.Database(), + gotreply: make(chan reply), + addReplyMatcher: make(chan *replyMatcher), + closeCtx: closeCtx, + cancelCloseCtx: cancel, + log: cfg.Log, + } + + tab, err := newTable(t, ln.Database(), cfg) if err != nil { return nil, err } @@ -200,26 +161,6 @@ func (t *UDPv4) Self() *enode.Node { return t.localNode.Node() } -func (t *UDPv4) Version() string { - return "v4" -} - -func (t *UDPv4) Errors() map[string]uint { - errors := map[string]uint{} - - t.mutex.Lock() - maps.Copy(errors, t.errors) - t.mutex.Unlock() - - return errors -} - -func (t *UDPv4) LenUnsolicited() int { - t.mutex.Lock() - defer t.mutex.Unlock() - return t.unsolicitedNodes.Len() -} - // Close shuts down the socket and aborts any running queries. func (t *UDPv4) Close() { t.closeOnce.Do(func() { @@ -252,8 +193,8 @@ func (t *UDPv4) Resolve(n *enode.Node) *enode.Node { result := t.LookupPubkey((*ecdsa.PublicKey)(&key)) for _, rn := range result { if rn.ID() == n.ID() { - if rn1, err := t.RequestENR(rn); err == nil { - return rn1 + if rn, err := t.RequestENR(rn); err == nil { + return rn } } } @@ -261,29 +202,43 @@ func (t *UDPv4) Resolve(n *enode.Node) *enode.Node { } func (t *UDPv4) ourEndpoint() v4wire.Endpoint { - n := t.Self() - a := &net.UDPAddr{IP: n.IP(), Port: n.UDP()} - return v4wire.NewEndpoint(a, uint16(n.TCP())) -} - -// Ping sends a ping message to the given node. -func (t *UDPv4) Ping(n *enode.Node) error { - _, err := t.ping(n) - return err + node := t.Self() + addr, ok := node.UDPEndpoint() + if !ok { + return v4wire.Endpoint{} + } + return v4wire.NewEndpoint(addr, uint16(node.TCP())) } // ping sends a ping message to the given node and waits for a reply. func (t *UDPv4) ping(n *enode.Node) (seq uint64, err error) { - rm := t.sendPing(n.ID(), &net.UDPAddr{IP: n.IP(), Port: n.UDP()}, nil) + addr, ok := n.UDPEndpoint() + if !ok { + return 0, errNoUDPEndpoint + } + rm := t.sendPing(n.ID(), addr, nil) if err = <-rm.errc; err == nil { seq = rm.reply.(*v4wire.Pong).ENRSeq } return seq, err } +// Ping calls PING on a node and waits for a PONG response. +func (t *UDPv4) Ping(n *enode.Node) (pong *v4wire.Pong, err error) { + addr, ok := n.UDPEndpoint() + if !ok { + return nil, errNoUDPEndpoint + } + rm := t.sendPing(n.ID(), addr, nil) + if err = <-rm.errc; err == nil { + pong = rm.reply.(*v4wire.Pong) + } + return pong, err +} + // sendPing sends a ping message to the given node and invokes the callback // when the reply arrives. -func (t *UDPv4) sendPing(toid enode.ID, toaddr *net.UDPAddr, callback func()) *replyMatcher { +func (t *UDPv4) sendPing(toid enode.ID, toaddr netip.AddrPort, callback func()) *replyMatcher { req := t.makePing(toaddr) packet, hash, err := v4wire.Encode(t.priv, req) if err != nil { @@ -293,7 +248,7 @@ func (t *UDPv4) sendPing(toid enode.ID, toaddr *net.UDPAddr, callback func()) *r } // Add a matcher for the reply to the pending reply queue. Pongs are matched if they // reference the ping we're about to send. - rm := t.pending(toid, toaddr.IP, toaddr.Port, v4wire.PongPacket, func(p v4wire.Packet) (matched bool, requestDone bool) { + rm := t.pending(toid, toaddr.Addr(), v4wire.PongPacket, func(p v4wire.Packet) (matched bool, requestDone bool) { matched = bytes.Equal(p.(*v4wire.Pong).ReplyTok, hash) if matched && callback != nil { callback() @@ -302,11 +257,11 @@ func (t *UDPv4) sendPing(toid enode.ID, toaddr *net.UDPAddr, callback func()) *r }) // Send the packet. t.localNode.UDPContact(toaddr) - t.write(toaddr, toid, req.Name(), packet) //nolint:errcheck + t.write(toaddr, toid, req.Name(), packet) return rm } -func (t *UDPv4) makePing(toaddr *net.UDPAddr) *v4wire.Ping { +func (t *UDPv4) makePing(toaddr netip.AddrPort) *v4wire.Ping { return &v4wire.Ping{ Version: 4, From: t.ourEndpoint(), @@ -323,7 +278,7 @@ func (t *UDPv4) LookupPubkey(key *ecdsa.PublicKey) []*enode.Node { // case and run the bootstrapping logic. <-t.tab.refresh() } - return t.newLookup(t.closeCtx, key).run() + return t.newLookup(t.closeCtx, v4wire.EncodePubkey(key)).run() } // RandomNodes is an iterator yielding nodes from a random walk of the DHT. @@ -338,83 +293,69 @@ func (t *UDPv4) lookupRandom() []*enode.Node { // lookupSelf implements transport. func (t *UDPv4) lookupSelf() []*enode.Node { - return t.newLookup(t.closeCtx, &t.priv.PublicKey).run() + pubkey := v4wire.EncodePubkey(&t.priv.PublicKey) + return t.newLookup(t.closeCtx, pubkey).run() } func (t *UDPv4) newRandomLookup(ctx context.Context) *lookup { - key, err := t.privateKeyGenerator() - if err != nil { - t.log.Warn("Failed to generate a random node key for newRandomLookup", "err", err) - key = t.priv - } - return t.newLookup(ctx, &key.PublicKey) + var target v4wire.Pubkey + crand.Read(target[:]) + return t.newLookup(ctx, target) } -func (t *UDPv4) newLookup(ctx context.Context, targetKey *ecdsa.PublicKey) *lookup { - targetKeyEnc := v4wire.EncodePubkey(targetKey) - target := enode.PubkeyEncoded(targetKeyEnc).ID() - - it := newLookup(ctx, t.tab, target, func(n *node) ([]*node, error) { - return t.findnode(n.ID(), n.addr(), targetKeyEnc) +func (t *UDPv4) newLookup(ctx context.Context, targetKey v4wire.Pubkey) *lookup { + target := enode.ID(crypto.Keccak256Hash(targetKey[:])) + it := newLookup(ctx, t.tab, target, func(n *enode.Node) ([]*enode.Node, error) { + addr, ok := n.UDPEndpoint() + if !ok { + return nil, errNoUDPEndpoint + } + return t.findnode(n.ID(), addr, targetKey) }) return it } -// FindNode sends a "FindNode" request to the given node and waits until -// the node has sent up to bucketSize neighbors or a respTimeout has passed. -func (t *UDPv4) FindNode(toNode *enode.Node, targetKey *ecdsa.PublicKey) ([]*enode.Node, error) { - targetKeyEnc := v4wire.EncodePubkey(targetKey) - nodes, err := t.findnode(toNode.ID(), wrapNode(toNode).addr(), targetKeyEnc) - return unwrapNodes(nodes), err -} - -func (t *UDPv4) findnode(toid enode.ID, toaddr *net.UDPAddr, target v4wire.Pubkey) ([]*node, error) { - t.ensureBond(toid, toaddr) +// findnode sends a findnode request to the given node and waits until +// the node has sent up to k neighbors. +func (t *UDPv4) findnode(toid enode.ID, toAddrPort netip.AddrPort, target v4wire.Pubkey) ([]*enode.Node, error) { + t.ensureBond(toid, toAddrPort) // Add a matcher for 'neighbours' replies to the pending reply queue. The matcher is // active until enough nodes have been received. - nodes := make([]*node, 0, bucketSize) + nodes := make([]*enode.Node, 0, bucketSize) nreceived := 0 - rm := t.pending(toid, toaddr.IP, toaddr.Port, v4wire.NeighborsPacket, func(r v4wire.Packet) (matched bool, requestDone bool) { + rm := t.pending(toid, toAddrPort.Addr(), v4wire.NeighborsPacket, func(r v4wire.Packet) (matched bool, requestDone bool) { reply := r.(*v4wire.Neighbors) for _, rn := range reply.Nodes { nreceived++ - n, err := t.nodeFromRPC(toaddr, rn) + n, err := t.nodeFromRPC(toAddrPort, rn) if err != nil { - t.log.Trace("Invalid neighbor node received", "ip", rn.IP, "addr", toaddr, "err", err) + t.log.Trace("Invalid neighbor node received", "ip", rn.IP, "addr", toAddrPort, "err", err) continue } nodes = append(nodes, n) } return true, nreceived >= bucketSize }) - _, err := t.send(toaddr, toid, &v4wire.Findnode{ + t.send(toAddrPort, toid, &v4wire.Findnode{ Target: target, Expiration: uint64(time.Now().Add(expiration).Unix()), }) - // Ensure that callers don't see a timeout if the node actually responded. Since // findnode can receive more than one neighbors response, the reply matcher will be // active until the remote node sends enough nodes. If the remote end doesn't have // enough nodes the reply matcher will time out waiting for the second reply, but // there's no need for an error in that case. - if errors.Is(err, errTimeout) && rm.reply != nil { - err = nil - } - if err != nil { - return nodes, err - } - - err = <-rm.errc + err := <-rm.errc if errors.Is(err, errTimeout) && rm.reply != nil { err = nil } return nodes, err } -// RequestENR sends enrRequest to the given node and waits for a response. +// RequestENR sends ENRRequest to the given node and waits for a response. func (t *UDPv4) RequestENR(n *enode.Node) (*enode.Node, error) { - addr := &net.UDPAddr{IP: n.IP(), Port: n.UDP()} + addr, _ := n.UDPEndpoint() t.ensureBond(n.ID(), addr) req := &v4wire.ENRRequest{ @@ -427,17 +368,13 @@ func (t *UDPv4) RequestENR(n *enode.Node) (*enode.Node, error) { // Add a matcher for the reply to the pending reply queue. Responses are matched if // they reference the request we're about to send. - rm := t.pending(n.ID(), addr.IP, addr.Port, v4wire.ENRResponsePacket, func(r v4wire.Packet) (matched bool, requestDone bool) { + rm := t.pending(n.ID(), addr.Addr(), v4wire.ENRResponsePacket, func(r v4wire.Packet) (matched bool, requestDone bool) { matched = bytes.Equal(r.(*v4wire.ENRResponse).ReplyTok, hash) return matched, matched }) // Send the packet and wait for the reply. - - err = t.write(addr, n.ID(), req.Name(), packet) - if err != nil { - return nil, err - } - if err = <-rm.errc; err != nil { + t.write(addr, n.ID(), req.Name(), packet) + if err := <-rm.errc; err != nil { return nil, err } // Verify the response record. @@ -451,25 +388,21 @@ func (t *UDPv4) RequestENR(n *enode.Node) (*enode.Node, error) { if respN.Seq() < n.Seq() { return n, nil // response record is older } - if err := netutil.CheckRelayIP(addr.IP, respN.IP()); err != nil { - return nil, fmt.Errorf("invalid IP in response record: %w", err) + if err := netutil.CheckRelayAddr(addr.Addr(), respN.IPAddr()); err != nil { + return nil, fmt.Errorf("invalid IP in response record: %v", err) } return respN, nil } +func (t *UDPv4) TableBuckets() [][]BucketNode { + return t.tab.Nodes() +} + // pending adds a reply matcher to the pending reply queue. // see the documentation of type replyMatcher for a detailed explanation. -func (t *UDPv4) pending(id enode.ID, ip net.IP, port int, ptype byte, callback replyMatchFunc) *replyMatcher { +func (t *UDPv4) pending(id enode.ID, ip netip.Addr, ptype byte, callback replyMatchFunc) *replyMatcher { ch := make(chan error, 1) - p := &replyMatcher{from: id, ip: ip, port: port, ptype: ptype, callback: callback, errc: ch} - - t.addReplyMatcherMutex.Lock() - defer t.addReplyMatcherMutex.Unlock() - if t.addReplyMatcher == nil { - ch <- errClosed - return p - } - + p := &replyMatcher{from: id, ip: ip, ptype: ptype, callback: callback, errc: ch} select { case t.addReplyMatcher <- p: // loop will handle it @@ -481,10 +414,10 @@ func (t *UDPv4) pending(id enode.ID, ip net.IP, port int, ptype byte, callback r // handleReply dispatches a reply packet, invoking reply matchers. It returns // whether any matcher considered the packet acceptable. -func (t *UDPv4) handleReply(from enode.ID, fromIP net.IP, port int, req v4wire.Packet) bool { +func (t *UDPv4) handleReply(from enode.ID, fromIP netip.Addr, req v4wire.Packet) bool { matched := make(chan bool, 1) select { - case t.gotreply <- reply{from, fromIP, port, req, matched}: + case t.gotreply <- reply{from, fromIP, req, matched}: // loop will handle it return <-matched case <-t.closeCtx.Done(): @@ -495,188 +428,98 @@ func (t *UDPv4) handleReply(from enode.ID, fromIP net.IP, port int, req v4wire.P // loop runs in its own goroutine. it keeps track of // the refresh timer and the pending reply queue. func (t *UDPv4) loop() { - defer dbg.LogPanic() defer t.wg.Done() var ( plist = list.New() - mutex = sync.Mutex{} - contTimeouts = 0 // number of continuous timeouts to do NTP checks + timeout = time.NewTimer(0) + nextTimeout *replyMatcher // head of plist when timeout was last reset + contTimeouts = 0 // number of continuous timeouts to do NTP checks ntpWarnTime = time.Unix(0, 0) ) + <-timeout.C // ignore first timeout + defer timeout.Stop() - listUpdate := make(chan *list.Element, 10) - - go func() { - var ( - timeout = time.NewTimer(0) - nextTimeout *replyMatcher // head of plist when timeout was last reset - ) - - <-timeout.C // ignore first timeout - defer timeout.Stop() - - resetTimeout := func() { - mutex.Lock() - defer mutex.Unlock() - - if plist.Front() == nil || nextTimeout == plist.Front().Value { - return - } - - // Start the timer so it fires when the next pending reply has expired. - now := time.Now() - for el := plist.Front(); el != nil; el = el.Next() { - nextTimeout = el.Value.(*replyMatcher) - if dist := nextTimeout.deadline.Sub(now); dist < 2*t.replyTimeout { - timeout.Reset(dist) - return - } - // Remove pending replies whose deadline is too far in the - // future. These can occur if the system clock jumped - // backwards after the deadline was assigned. - nextTimeout.errc <- errClockWarp - plist.Remove(el) - } - - nextTimeout = nil - timeout.Stop() + resetTimeout := func() { + if plist.Front() == nil || nextTimeout == plist.Front().Value { + return } - - for { - select { - case <-t.closeCtx.Done(): + // Start the timer so it fires when the next pending reply has expired. + now := time.Now() + for el := plist.Front(); el != nil; el = el.Next() { + nextTimeout = el.Value.(*replyMatcher) + if dist := nextTimeout.deadline.Sub(now); dist < 2*respTimeout { + timeout.Reset(dist) return - - case now := <-timeout.C: - func() { - mutex.Lock() - defer mutex.Unlock() - - nextTimeout = nil - // Notify and remove callbacks whose deadline is in the past. - for el := plist.Front(); el != nil; el = el.Next() { - p := el.Value.(*replyMatcher) - if !now.Before(p.deadline) { - p.errc <- errTimeout - plist.Remove(el) - contTimeouts++ - } - } - // If we've accumulated too many timeouts, do an NTP time sync check - if contTimeouts > ntpFailureThreshold { - if time.Since(ntpWarnTime) >= ntpWarningCooldown { - ntpWarnTime = time.Now() - go checkClockDrift() - } - contTimeouts = 0 - } - }() - - resetTimeout() - - case el := <-listUpdate: - if el == nil { - return - } - - resetTimeout() } + // Remove pending replies whose deadline is too far in the + // future. These can occur if the system clock jumped + // backwards after the deadline was assigned. + nextTimeout.errc <- errClockWarp + plist.Remove(el) } - }() + nextTimeout = nil + timeout.Stop() + } for { + resetTimeout() + select { case <-t.closeCtx.Done(): - listUpdate <- nil - func() { - mutex.Lock() - defer mutex.Unlock() - for el := plist.Front(); el != nil; el = el.Next() { - el.Value.(*replyMatcher).errc <- errClosed - } - }() - - t.addReplyMatcherMutex.Lock() - defer t.addReplyMatcherMutex.Unlock() - close(t.addReplyMatcher) - for matcher := range t.addReplyMatcher { - matcher.errc <- errClosed + for el := plist.Front(); el != nil; el = el.Next() { + el.Value.(*replyMatcher).errc <- errClosed } - t.addReplyMatcher = nil return case p := <-t.addReplyMatcher: - mutex.Lock() - p.deadline = time.Now().Add(t.replyTimeout) - back := plist.PushBack(p) - mutex.Unlock() - listUpdate <- back + p.deadline = time.Now().Add(respTimeout) + plist.PushBack(p) case r := <-t.gotreply: - var removals []*list.Element - - func() { - mutex.Lock() - defer mutex.Unlock() - - var matched bool // whether any replyMatcher considered the reply acceptable. - for el := plist.Front(); el != nil; el = el.Next() { - p := el.Value.(*replyMatcher) - if (p.ptype == r.data.Kind()) && p.ip.Equal(r.ip) && (p.port == r.port) { - ok, requestDone := p.callback(r.data) - matched = matched || ok - p.reply = r.data - // Remove the matcher if callback indicates that all replies have been received. - if requestDone { - p.errc <- nil - plist.Remove(el) - removals = append(removals, el) - } - // Reset the continuous timeout counter (time drift detection) - contTimeouts = 0 + var matched bool // whether any replyMatcher considered the reply acceptable. + for el := plist.Front(); el != nil; el = el.Next() { + p := el.Value.(*replyMatcher) + if p.from == r.from && p.ptype == r.data.Kind() && p.ip == r.ip { + ok, requestDone := p.callback(r.data) + matched = matched || ok + p.reply = r.data + // Remove the matcher if callback indicates that all replies have been received. + if requestDone { + p.errc <- nil + plist.Remove(el) } + // Reset the continuous timeout counter (time drift detection) + contTimeouts = 0 } - r.matched <- matched - }() - - for _, el := range removals { - listUpdate <- el } + r.matched <- matched - case key := <-t.gotkey: - go func() { - if key, err := v4wire.DecodePubkey(crypto.S256(), key); err == nil { - nodes := t.LookupPubkey(key) - mutex.Lock() - defer mutex.Unlock() + case now := <-timeout.C: + nextTimeout = nil - for _, n := range nodes { - t.unsolicitedNodes.Add(n.ID(), n) - } + // Notify and remove callbacks whose deadline is in the past. + for el := plist.Front(); el != nil; el = el.Next() { + p := el.Value.(*replyMatcher) + if now.After(p.deadline) || now.Equal(p.deadline) { + p.errc <- errTimeout + plist.Remove(el) + contTimeouts++ } - }() - - case nodes := <-t.gotnodes: - - func() { - mutex.Lock() - defer mutex.Unlock() - for _, rn := range nodes.nodes { - n, err := t.nodeFromRPC(nodes.addr, rn) - if err != nil { - t.log.Trace("Invalid neighbor node received", "ip", rn.IP, "addr", nodes.addr, "err", err) - continue - } - t.unsolicitedNodes.Add(n.ID(), &n.Node) + } + // If we've accumulated too many timeouts, do an NTP time sync check + if contTimeouts > ntpFailureThreshold { + if time.Since(ntpWarnTime) >= ntpWarningCooldown { + ntpWarnTime = time.Now() + go checkClockDrift() } - }() + contTimeouts = 0 + } } } } -//nolint:unparam -func (t *UDPv4) send(toaddr *net.UDPAddr, toid enode.ID, req v4wire.Packet) ([]byte, error) { +func (t *UDPv4) send(toaddr netip.AddrPort, toid enode.ID, req v4wire.Packet) ([]byte, error) { packet, hash, err := v4wire.Encode(t.priv, req) if err != nil { return hash, err @@ -684,88 +527,60 @@ func (t *UDPv4) send(toaddr *net.UDPAddr, toid enode.ID, req v4wire.Packet) ([]b return hash, t.write(toaddr, toid, req.Name(), packet) } -func (t *UDPv4) write(toaddr *net.UDPAddr, toid enode.ID, what string, packet []byte) error { - _, err := t.conn.WriteToUDP(packet, toaddr) - if t.trace { - t.log.Trace(">> "+what, "id", toid, "addr", toaddr, "err", err) - } +func (t *UDPv4) write(toaddr netip.AddrPort, toid enode.ID, what string, packet []byte) error { + _, err := t.conn.WriteToUDPAddrPort(packet, toaddr) + t.log.Trace(">> "+what, "id", toid, "addr", toaddr, "err", err) return err } // readLoop runs in its own goroutine. it handles incoming UDP packets. func (t *UDPv4) readLoop(unhandled chan<- ReadPacket) { defer t.wg.Done() - defer dbg.LogPanic() - if unhandled != nil { defer close(unhandled) } - unknownKeys, _ := lru.New[v4wire.Pubkey, any](100) - buf := make([]byte, maxPacketSize) for { - nbytes, from, err := t.conn.ReadFromUDP(buf) + nbytes, from, err := t.conn.ReadFromUDPAddrPort(buf) if netutil.IsTemporaryError(err) { // Ignore temporary read errors. - t.log.Trace("Temporary UDP read error", "err", err) + t.log.Debug("Temporary UDP read error", "err", err) continue - } - if err != nil { - // Shut down the loop for permament errors. - if err != io.EOF { - t.log.Trace("UDP read error", "err", err) + } else if err != nil { + // Shut down the loop for permanent errors. + if !errors.Is(err, io.EOF) { + t.log.Debug("UDP read error", "err", err) } return } - if err := t.handlePacket(from, buf[:nbytes]); err != nil { - func() { - switch { - case errors.Is(err, errUnsolicitedReply): - if packet, fromKey, _, err := v4wire.Decode(buf[:nbytes]); err == nil { - switch packet.Kind() { - case v4wire.PongPacket: - if _, ok := unknownKeys.Get(fromKey); !ok { - fromId := enode.PubkeyEncoded(fromKey).ID() - t.log.Trace("Unsolicited packet", "type", packet.Name(), "from", fromId, "addr", from) - unknownKeys.Add(fromKey, nil) - t.gotkey <- fromKey - } - case v4wire.NeighborsPacket: - neighbors := packet.(*v4wire.Neighbors) - t.gotnodes <- nodes{from, neighbors.Nodes} - default: - fromId := enode.PubkeyEncoded(fromKey).ID() - t.log.Trace("Unsolicited packet", "type", packet.Name(), "from", fromId, "addr", from) - } - } else { - t.log.Trace("Unsolicited packet handling failed", "addr", from, "err", err) - } - default: - if unhandled != nil { - unhandled <- ReadPacket{buf[:nbytes], from} - } - } - }() + if err := t.handlePacket(from, buf[:nbytes]); err != nil && unhandled == nil { + t.log.Debug("Bad discv4 packet", "addr", from, "err", err) + } else if err != nil && unhandled != nil { + select { + case unhandled <- ReadPacket{buf[:nbytes], from}: + default: + } } } } -func (t *UDPv4) handlePacket(from *net.UDPAddr, buf []byte) error { +func (t *UDPv4) handlePacket(from netip.AddrPort, buf []byte) error { + // Unwrap IPv4-in-6 source address. + if from.Addr().Is4In6() { + from = netip.AddrPortFrom(netip.AddrFrom4(from.Addr().As4()), from.Port()) + } + rawpacket, fromKey, hash, err := v4wire.Decode(buf) if err != nil { - t.log.Trace("Bad discv4 packet", "addr", from, "err", err) return err } packet := t.wrapPacket(rawpacket) - fromID := enode.PubkeyEncoded(fromKey).ID() - + fromID := fromKey.ID() if packet.preverify != nil { err = packet.preverify(packet, from, fromID, fromKey) } - if t.trace { - t.log.Trace("<< "+packet.Name(), "id", fromID, "addr", from, "err", err) - } + t.log.Trace("<< "+packet.Name(), "id", fromID, "addr", from, "err", err) if err == nil && packet.handle != nil { packet.handle(packet, from, fromID, hash) } @@ -773,42 +588,42 @@ func (t *UDPv4) handlePacket(from *net.UDPAddr, buf []byte) error { } // checkBond checks if the given node has a recent enough endpoint proof. -func (t *UDPv4) checkBond(id enode.ID, ip net.IP) bool { - return time.Since(t.db.LastPongReceived(id, ip)) < bondExpiration +func (t *UDPv4) checkBond(id enode.ID, ip netip.AddrPort) bool { + return time.Since(t.db.LastPongReceived(id, ip.Addr())) < bondExpiration } // ensureBond solicits a ping from a node if we haven't seen a ping from it for a while. // This ensures there is a valid endpoint proof on the remote end. -func (t *UDPv4) ensureBond(toid enode.ID, toaddr *net.UDPAddr) { - tooOld := time.Since(t.db.LastPingReceived(toid, toaddr.IP)) > bondExpiration - if tooOld || t.db.FindFails(toid, toaddr.IP) > maxFindnodeFailures { +func (t *UDPv4) ensureBond(toid enode.ID, toaddr netip.AddrPort) { + tooOld := time.Since(t.db.LastPingReceived(toid, toaddr.Addr())) > bondExpiration + if tooOld || t.db.FindFails(toid, toaddr.Addr()) > maxFindnodeFailures { rm := t.sendPing(toid, toaddr, nil) <-rm.errc // Wait for them to ping back and process our pong. - time.Sleep(t.pingBackDelay) + time.Sleep(respTimeout) } } -func (t *UDPv4) nodeFromRPC(sender *net.UDPAddr, rn v4wire.Node) (*node, error) { +func (t *UDPv4) nodeFromRPC(sender netip.AddrPort, rn v4wire.Node) (*enode.Node, error) { if rn.UDP <= 1024 { return nil, errLowPort } - if err := netutil.CheckRelayIP(sender.IP, rn.IP); err != nil { + if err := netutil.CheckRelayIP(sender.Addr().AsSlice(), rn.IP); err != nil { return nil, err } if t.netrestrict != nil && !t.netrestrict.Contains(rn.IP) { - return nil, errors.New("not contained in netrestrict whitelist") + return nil, errors.New("not contained in netrestrict list") } key, err := v4wire.DecodePubkey(crypto.S256(), rn.ID) if err != nil { return nil, err } - n := wrapNode(enode.NewV4(key, rn.IP, int(rn.TCP), int(rn.UDP))) + n := enode.NewV4(key, rn.IP, int(rn.TCP), int(rn.UDP)) err = n.ValidateComplete() return n, err } -func nodeToRPC(n *node) v4wire.Node { +func nodeToRPC(n *enode.Node) v4wire.Node { var key ecdsa.PublicKey var ekey v4wire.Pubkey if err := n.Load((*enode.Secp256k1)(&key)); err == nil { @@ -847,39 +662,31 @@ type packetHandlerV4 struct { senderKey *ecdsa.PublicKey // used for ping // preverify checks whether the packet is valid and should be handled at all. - preverify func(p *packetHandlerV4, from *net.UDPAddr, fromID enode.ID, fromKey v4wire.Pubkey) error + preverify func(p *packetHandlerV4, from netip.AddrPort, fromID enode.ID, fromKey v4wire.Pubkey) error // handle handles the packet. - handle func(req *packetHandlerV4, from *net.UDPAddr, fromID enode.ID, mac []byte) + handle func(req *packetHandlerV4, from netip.AddrPort, fromID enode.ID, mac []byte) } // PING/v4 -func (t *UDPv4) verifyPing(h *packetHandlerV4, from *net.UDPAddr, fromID enode.ID, fromKey v4wire.Pubkey) error { +func (t *UDPv4) verifyPing(h *packetHandlerV4, from netip.AddrPort, fromID enode.ID, fromKey v4wire.Pubkey) error { req := h.Packet.(*v4wire.Ping) + if v4wire.Expired(req.Expiration) { + return errExpired + } senderKey, err := v4wire.DecodePubkey(crypto.S256(), fromKey) if err != nil { - errStr := err.Error() - t.mutex.Lock() - t.errors[errStr] = t.errors[errStr] + 1 - t.mutex.Unlock() return err } - if v4wire.Expired(req.Expiration) { - t.mutex.Lock() - t.errors[errExpiredStr] = t.errors[errExpiredStr] + 1 - t.mutex.Unlock() - return errExpired - } h.senderKey = senderKey return nil } -func (t *UDPv4) handlePing(h *packetHandlerV4, from *net.UDPAddr, fromID enode.ID, mac []byte) { +func (t *UDPv4) handlePing(h *packetHandlerV4, from netip.AddrPort, fromID enode.ID, mac []byte) { req := h.Packet.(*v4wire.Ping) // Reply. - //nolint:errcheck t.send(from, fromID, &v4wire.Pong{ To: v4wire.NewEndpoint(from, req.From.TCP), ReplyTok: mac, @@ -888,81 +695,73 @@ func (t *UDPv4) handlePing(h *packetHandlerV4, from *net.UDPAddr, fromID enode.I }) // Ping back if our last pong on file is too far in the past. - n := wrapNode(enode.NewV4(h.senderKey, from.IP, int(req.From.TCP), from.Port)) - if time.Since(t.db.LastPongReceived(n.ID(), from.IP)) > bondExpiration { + fromIP := from.Addr().AsSlice() + n := enode.NewV4(h.senderKey, fromIP, int(req.From.TCP), int(from.Port())) + if time.Since(t.db.LastPongReceived(n.ID(), from.Addr())) > bondExpiration { t.sendPing(fromID, from, func() { - t.tab.addVerifiedNode(n) + t.tab.addInboundNode(n) }) } else { - t.tab.addVerifiedNode(n) + t.tab.addInboundNode(n) } // Update node database and endpoint predictor. - t.db.UpdateLastPingReceived(n.ID(), from.IP, time.Now()) - t.localNode.UDPEndpointStatement(from, &net.UDPAddr{IP: req.To.IP, Port: int(req.To.UDP)}) + t.db.UpdateLastPingReceived(n.ID(), from.Addr(), time.Now()) + toaddr := netip.AddrPortFrom(netutil.IPToAddr(req.To.IP), req.To.UDP) + t.localNode.UDPEndpointStatement(from, toaddr) } // PONG/v4 -func (t *UDPv4) verifyPong(h *packetHandlerV4, from *net.UDPAddr, fromID enode.ID, fromKey v4wire.Pubkey) error { +func (t *UDPv4) verifyPong(h *packetHandlerV4, from netip.AddrPort, fromID enode.ID, fromKey v4wire.Pubkey) error { req := h.Packet.(*v4wire.Pong) if v4wire.Expired(req.Expiration) { - t.mutex.Lock() - t.errors[errExpiredStr] = t.errors[errExpiredStr] + 1 - t.mutex.Unlock() return errExpired } - if !t.handleReply(fromID, from.IP, from.Port, req) { - t.mutex.Lock() - t.errors[errUnsolicitedReplyStr] = t.errors[errUnsolicitedReplyStr] + 1 - t.mutex.Unlock() + if !t.handleReply(fromID, from.Addr(), req) { return errUnsolicitedReply } - t.localNode.UDPEndpointStatement(from, &net.UDPAddr{IP: req.To.IP, Port: int(req.To.UDP)}) - t.db.UpdateLastPongReceived(fromID, from.IP, time.Now()) + toaddr := netip.AddrPortFrom(netutil.IPToAddr(req.To.IP), req.To.UDP) + t.localNode.UDPEndpointStatement(from, toaddr) + t.db.UpdateLastPongReceived(fromID, from.Addr(), time.Now()) return nil } // FINDNODE/v4 -func (t *UDPv4) verifyFindnode(h *packetHandlerV4, from *net.UDPAddr, fromID enode.ID, fromKey v4wire.Pubkey) error { +func (t *UDPv4) verifyFindnode(h *packetHandlerV4, from netip.AddrPort, fromID enode.ID, fromKey v4wire.Pubkey) error { req := h.Packet.(*v4wire.Findnode) if v4wire.Expired(req.Expiration) { - t.mutex.Lock() - t.errors[errExpiredStr] = t.errors[errExpiredStr] + 1 - t.mutex.Unlock() return errExpired } - if !t.checkBond(fromID, from.IP) { + if !t.checkBond(fromID, from) { // No endpoint proof pong exists, we don't process the packet. This prevents an // attack vector where the discovery protocol could be used to amplify traffic in a // DDOS attack. A malicious actor would send a findnode request with the IP address // and UDP port of the target as the source address. The recipient of the findnode // packet would then send a neighbors packet (which is a much bigger packet than // findnode) to the victim. - t.mutex.Lock() - t.errors[errUnknownNodeStr] = t.errors[errUnknownNodeStr] + 1 - t.mutex.Unlock() return errUnknownNode } return nil } -func (t *UDPv4) handleFindnode(h *packetHandlerV4, from *net.UDPAddr, fromID enode.ID, mac []byte) { +func (t *UDPv4) handleFindnode(h *packetHandlerV4, from netip.AddrPort, fromID enode.ID, mac []byte) { req := h.Packet.(*v4wire.Findnode) // Determine closest nodes. - target := enode.PubkeyEncoded(req.Target).ID() - closest := t.tab.findnodeByID(target, bucketSize, true).entries + target := enode.ID(crypto.Keccak256Hash(req.Target[:])) + preferLive := !t.tab.cfg.NoFindnodeLivenessCheck + closest := t.tab.findnodeByID(target, bucketSize, preferLive).entries // Send neighbors in chunks with at most maxNeighbors per packet // to stay below the packet size limit. p := v4wire.Neighbors{Expiration: uint64(time.Now().Add(expiration).Unix())} var sent bool for _, n := range closest { - if netutil.CheckRelayIP(from.IP, n.IP()) == nil { + if netutil.CheckRelayAddr(from.Addr(), n.IPAddr()) == nil { p.Nodes = append(p.Nodes, nodeToRPC(n)) } if len(p.Nodes) == v4wire.MaxNeighbors { @@ -978,19 +777,13 @@ func (t *UDPv4) handleFindnode(h *packetHandlerV4, from *net.UDPAddr, fromID eno // NEIGHBORS/v4 -func (t *UDPv4) verifyNeighbors(h *packetHandlerV4, from *net.UDPAddr, fromID enode.ID, fromKey v4wire.Pubkey) error { +func (t *UDPv4) verifyNeighbors(h *packetHandlerV4, from netip.AddrPort, fromID enode.ID, fromKey v4wire.Pubkey) error { req := h.Packet.(*v4wire.Neighbors) if v4wire.Expired(req.Expiration) { - t.mutex.Lock() - t.errors[errExpiredStr] = t.errors[errExpiredStr] + 1 - t.mutex.Unlock() return errExpired } - if !t.handleReply(fromID, from.IP, from.Port, h.Packet) { - t.mutex.Lock() - t.errors[errUnsolicitedReplyStr] = t.errors[errUnsolicitedReplyStr] + 1 - t.mutex.Unlock() + if !t.handleReply(fromID, from.Addr(), h.Packet) { return errUnsolicitedReply } return nil @@ -998,45 +791,29 @@ func (t *UDPv4) verifyNeighbors(h *packetHandlerV4, from *net.UDPAddr, fromID en // ENRREQUEST/v4 -func (t *UDPv4) verifyENRRequest(h *packetHandlerV4, from *net.UDPAddr, fromID enode.ID, fromKey v4wire.Pubkey) error { +func (t *UDPv4) verifyENRRequest(h *packetHandlerV4, from netip.AddrPort, fromID enode.ID, fromKey v4wire.Pubkey) error { req := h.Packet.(*v4wire.ENRRequest) if v4wire.Expired(req.Expiration) { - t.mutex.Lock() - t.errors[errExpiredStr] = t.errors[errExpiredStr] + 1 - t.mutex.Unlock() return errExpired } - if !t.checkBond(fromID, from.IP) { - t.mutex.Lock() - t.errors[errUnknownNodeStr] = t.errors[errUnknownNodeStr] + 1 - t.mutex.Unlock() + if !t.checkBond(fromID, from) { return errUnknownNode } return nil } -func (t *UDPv4) handleENRRequest(h *packetHandlerV4, from *net.UDPAddr, fromID enode.ID, mac []byte) { - _, err := t.send(from, fromID, &v4wire.ENRResponse{ +func (t *UDPv4) handleENRRequest(h *packetHandlerV4, from netip.AddrPort, fromID enode.ID, mac []byte) { + t.send(from, fromID, &v4wire.ENRResponse{ ReplyTok: mac, Record: *t.localNode.Node().Record(), }) - - if err != nil { - errStr := err.Error() - t.mutex.Lock() - t.errors[errStr] = t.errors[errStr] + 1 - t.mutex.Unlock() - } } // ENRRESPONSE/v4 -func (t *UDPv4) verifyENRResponse(h *packetHandlerV4, from *net.UDPAddr, fromID enode.ID, fromKey v4wire.Pubkey) error { - if !t.handleReply(fromID, from.IP, from.Port, h.Packet) { - t.mutex.Lock() - t.errors[errUnsolicitedReplyStr] = t.errors[errUnsolicitedReplyStr] + 1 - t.mutex.Unlock() +func (t *UDPv4) verifyENRResponse(h *packetHandlerV4, from netip.AddrPort, fromID enode.ID, fromKey v4wire.Pubkey) error { + if !t.handleReply(fromID, from.Addr(), h.Packet) { return errUnsolicitedReply } return nil diff --git a/p2p/discover/v4_udp_test.go b/p2p/discover/v4_udp_test.go index f4e5a6a90c0..57143defc17 100644 --- a/p2p/discover/v4_udp_test.go +++ b/p2p/discover/v4_udp_test.go @@ -1,36 +1,36 @@ // Copyright 2015 The go-ethereum Authors -// (original work) -// Copyright 2024 The Erigon Authors -// (modifications) -// This file is part of Erigon. +// This file is part of the go-ethereum library. // -// Erigon is free software: you can redistribute it and/or modify +// The go-ethereum library is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // -// Erigon is distributed in the hope that it will be useful, +// The go-ethereum library is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Lesser General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License -// along with Erigon. If not, see . +// along with the go-ethereum library. If not, see . package discover import ( "bytes" - "context" "crypto/ecdsa" crand "crypto/rand" "encoding/binary" + "errors" "fmt" "io" + "maps" "math/rand" "net" + "net/netip" "reflect" - "runtime" + "slices" + "sync" "testing" "time" @@ -58,51 +58,24 @@ type udpTest struct { udp *UDPv4 sent [][]byte localkey, remotekey *ecdsa.PrivateKey - remoteaddr *net.UDPAddr + remoteaddr netip.AddrPort } -func newUDPTest(t *testing.T, logger log.Logger) *udpTest { - return newUDPTestContext(context.Background(), t, logger) -} - -func newUDPTestContext(ctx context.Context, t *testing.T, logger log.Logger) *udpTest { - ctx = disableLookupSlowdown(ctx) - - replyTimeout := contextGetReplyTimeout(ctx) - if replyTimeout == 0 { - replyTimeout = 50 * time.Millisecond - } - +func newUDPTest(t *testing.T) *udpTest { test := &udpTest{ t: t, pipe: newpipe(), localkey: newkey(), remotekey: newkey(), - remoteaddr: &net.UDPAddr{IP: net.IP{10, 0, 1, 99}, Port: 30303}, + remoteaddr: netip.MustParseAddrPort("10.0.1.99:30303"), } - tmpDir := t.TempDir() - var err error - test.db, err = enode.OpenDB(ctx, "", tmpDir, logger) - if err != nil { - panic(err) - } - ln := enode.NewLocalNode(test.db, test.localkey, logger) - test.udp, err = ListenV4(ctx, "test", test.pipe, ln, Config{ + test.db, _ = enode.OpenDB("") + ln := enode.NewLocalNode(test.db, test.localkey) + test.udp, _ = ListenV4(test.pipe, ln, Config{ PrivateKey: test.localkey, - Log: testlog.Logger(t, log.LvlError), - - ReplyTimeout: replyTimeout, - - PingBackDelay: time.Nanosecond, - - PrivateKeyGenerator: contextGetPrivateKeyGenerator(ctx), - - TableRevalidateInterval: time.Hour, + Log: testlog.Logger(t, log.LvlTrace), }) - if err != nil { - panic(err) - } test.table = test.udp.tab // Wait for initial refresh so the table doesn't send unexpected findnode. <-test.table.initDone @@ -122,7 +95,7 @@ func (test *udpTest) packetIn(wantError error, data v4wire.Packet) { } // handles a packet as if it had been sent to the transport by the key/endpoint. -func (test *udpTest) packetInFrom(wantError error, key *ecdsa.PrivateKey, addr *net.UDPAddr, data v4wire.Packet) { +func (test *udpTest) packetInFrom(wantError error, key *ecdsa.PrivateKey, addr netip.AddrPort, data v4wire.Packet) { test.t.Helper() enc, _, err := v4wire.Encode(key, data) @@ -130,17 +103,13 @@ func (test *udpTest) packetInFrom(wantError error, key *ecdsa.PrivateKey, addr * test.t.Errorf("%s encode error: %v", data.Name(), err) } test.sent = append(test.sent, enc) - - err = test.udp.handlePacket(addr, enc) - if (wantError == nil) && (err != nil) { - test.t.Errorf("handlePacket error: %q", err) - } else if (wantError != nil) && (err != wantError) { + if err = test.udp.handlePacket(addr, enc); err != wantError { test.t.Errorf("error mismatch: got %q, want %q", err, wantError) } } // waits for a packet to be sent by the transport. -// validate should have type func(X, *net.UDPAddr, []byte), where X is a packet type. +// validate should have type func(X, netip.AddrPort, []byte), where X is a packet type. func (test *udpTest) waitPacketOut(validate interface{}) (closed bool) { test.t.Helper() @@ -162,13 +131,12 @@ func (test *udpTest) waitPacketOut(validate interface{}) (closed bool) { test.t.Errorf("sent packet type mismatch, got: %v, want: %v", reflect.TypeOf(p), exptype) return false } - fn.Call([]reflect.Value{reflect.ValueOf(p), reflect.ValueOf(&dgram.to), reflect.ValueOf(hash)}) + fn.Call([]reflect.Value{reflect.ValueOf(p), reflect.ValueOf(dgram.to), reflect.ValueOf(hash)}) return false } func TestUDPv4_packetErrors(t *testing.T) { - logger := log.New() - test := newUDPTest(t, logger) + test := newUDPTest(t) defer test.close() test.packetIn(errExpired, &v4wire.Ping{From: testRemote, To: testLocalAnnounced, Version: 4}) @@ -179,8 +147,7 @@ func TestUDPv4_packetErrors(t *testing.T) { func TestUDPv4_pingTimeout(t *testing.T) { t.Parallel() - logger := log.New() - test := newUDPTest(t, logger) + test := newUDPTest(t) defer test.close() key := newkey() @@ -197,19 +164,10 @@ func (req testPacket) Kind() byte { return byte(req) } func (req testPacket) Name() string { return "" } func TestUDPv4_responseTimeouts(t *testing.T) { - if runtime.GOOS == `darwin` { - t.Skip("unstable test on darwin") - } t.Parallel() - logger := log.New() - - ctx := context.Background() - ctx = contextWithReplyTimeout(ctx, respTimeout) - - test := newUDPTestContext(ctx, t, logger) + test := newUDPTest(t) defer test.close() - rand.Seed(time.Now().UnixNano()) randomDuration := func(max time.Duration) time.Duration { return time.Duration(rand.Int63n(int64(max))) } @@ -238,7 +196,7 @@ func TestUDPv4_responseTimeouts(t *testing.T) { p.errc = nilErr test.udp.addReplyMatcher <- p time.AfterFunc(randomDuration(60*time.Millisecond), func() { - if !test.udp.handleReply(p.from, p.ip, p.port, testPacket(p.ptype)) { + if !test.udp.handleReply(p.from, p.ip, testPacket(p.ptype)) { t.Logf("not matched: %v", p) } }) @@ -278,11 +236,10 @@ func TestUDPv4_responseTimeouts(t *testing.T) { func TestUDPv4_findnodeTimeout(t *testing.T) { t.Parallel() - logger := log.New() - test := newUDPTest(t, logger) + test := newUDPTest(t) defer test.close() - toaddr := &net.UDPAddr{IP: net.ParseIP("1.2.3.4"), Port: 2222} + toaddr := netip.AddrPortFrom(netip.MustParseAddr("1.2.3.4"), 2222) toid := enode.ID{1, 2, 3, 4} target := v4wire.Pubkey{4, 5, 6, 7} result, err := test.udp.findnode(toid, toaddr, target) @@ -295,50 +252,47 @@ func TestUDPv4_findnodeTimeout(t *testing.T) { } func TestUDPv4_findnode(t *testing.T) { - logger := log.New() - test := newUDPTest(t, logger) + test := newUDPTest(t) defer test.close() // put a few nodes into the table. their exact // distribution shouldn't matter much, although we need to // take care not to overflow any bucket. - testTargetID := enode.PubkeyEncoded(testTarget).ID() - nodes := &nodesByDistance{target: testTargetID} + nodes := &nodesByDistance{target: testTarget.ID()} live := make(map[enode.ID]bool) numCandidates := 2 * bucketSize for i := 0; i < numCandidates; i++ { key := newkey() ip := net.IP{10, 13, 0, byte(i)} - n := wrapNode(enode.NewV4(&key.PublicKey, ip, 0, 2000)) + n := enode.NewV4(&key.PublicKey, ip, 0, 2000) // Ensure half of table content isn't verified live yet. if i > numCandidates/2 { - n.livenessChecks = 1 live[n.ID()] = true } + test.table.addFoundNode(n, live[n.ID()]) nodes.push(n, numCandidates) } - fillTable(test.table, nodes.entries) // ensure there's a bond with the test node, // findnode won't be accepted otherwise. - remoteID := enode.PubkeyToIDV4(&test.remotekey.PublicKey) - test.table.db.UpdateLastPongReceived(remoteID, test.remoteaddr.IP, time.Now()) + remoteID := v4wire.EncodePubkey(&test.remotekey.PublicKey).ID() + test.table.db.UpdateLastPongReceived(remoteID, test.remoteaddr.Addr(), time.Now()) // check that closest neighbors are returned. - expected := test.table.findnodeByID(testTargetID, bucketSize, true) + expected := test.table.findnodeByID(testTarget.ID(), bucketSize, true) test.packetIn(nil, &v4wire.Findnode{Target: testTarget, Expiration: futureExp}) - waitNeighbors := func(want []*node) { - test.waitPacketOut(func(p *v4wire.Neighbors, to *net.UDPAddr, hash []byte) { + waitNeighbors := func(want []*enode.Node) { + test.waitPacketOut(func(p *v4wire.Neighbors, to netip.AddrPort, hash []byte) { if len(p.Nodes) != len(want) { - t.Errorf("wrong number of results: got %d, want %d", len(p.Nodes), bucketSize) + t.Errorf("wrong number of results: got %d, want %d", len(p.Nodes), len(want)) + return } for i, n := range p.Nodes { - nodeID := enode.PubkeyEncoded(n.ID).ID() - if nodeID != want[i].ID() { - t.Errorf("result mismatch at %d:\n got: %v\n want: %v", i, n, expected.entries[i]) + if n.ID.ID() != want[i].ID() { + t.Errorf("result mismatch at %d:\n got: %v\n want: %v", i, n, expected.entries[i]) } - if !live[nodeID] { - t.Errorf("result includes dead node %v", nodeID) + if !live[n.ID.ID()] { + t.Errorf("result includes dead node %v", n.ID.ID()) } } }) @@ -353,17 +307,16 @@ func TestUDPv4_findnode(t *testing.T) { } func TestUDPv4_findnodeMultiReply(t *testing.T) { - logger := log.New() - test := newUDPTest(t, logger) + test := newUDPTest(t) defer test.close() rid := enode.PubkeyToIDV4(&test.remotekey.PublicKey) - test.table.db.UpdateLastPingReceived(rid, test.remoteaddr.IP, time.Now()) + test.table.db.UpdateLastPingReceived(rid, test.remoteaddr.Addr(), time.Now()) // queue a pending findnode request - resultc, errc := make(chan []*node), make(chan error) + resultc, errc := make(chan []*enode.Node, 1), make(chan error, 1) go func() { - rid := enode.PubkeyToIDV4(&test.remotekey.PublicKey) + rid := v4wire.EncodePubkey(&test.remotekey.PublicKey).ID() ns, err := test.udp.findnode(rid, test.remoteaddr, testTarget) if err != nil && len(ns) == 0 { errc <- err @@ -374,18 +327,18 @@ func TestUDPv4_findnodeMultiReply(t *testing.T) { // wait for the findnode to be sent. // after it is sent, the transport is waiting for a reply - test.waitPacketOut(func(p *v4wire.Findnode, to *net.UDPAddr, hash []byte) { + test.waitPacketOut(func(p *v4wire.Findnode, to netip.AddrPort, hash []byte) { if p.Target != testTarget { t.Errorf("wrong target: got %v, want %v", p.Target, testTarget) } }) // send the reply as two packets. - list := []*node{ - wrapNode(enode.MustParse("enode://ba85011c70bcc5c04d8607d3a0ed29aa6179c092cbdda10d5d32684fb33ed01bd94f588ca8f91ac48318087dcb02eaf36773a7a453f0eedd6742af668097b29c@10.0.1.16:30303?discport=30304")), - wrapNode(enode.MustParse("enode://81fa361d25f157cd421c60dcc28d8dac5ef6a89476633339c5df30287474520caca09627da18543d9079b5b288698b542d56167aa5c09111e55acdbbdf2ef799@10.0.1.16:30303")), - wrapNode(enode.MustParse("enode://9bffefd833d53fac8e652415f4973bee289e8b1a5c6c4cbe70abf817ce8a64cee11b823b66a987f51aaa9fba0d6a91b3e6bf0d5a5d1042de8e9eeea057b217f8@10.0.1.36:30301?discport=17")), - wrapNode(enode.MustParse("enode://1b5b4aa662d7cb44a7221bfba67302590b643028197a7d5214790f3bac7aaa4a3241be9e83c09cf1f6c69d007c634faae3dc1b1221793e8446c0b3a09de65960@10.0.1.16:30303")), + list := []*enode.Node{ + enode.MustParse("enode://ba85011c70bcc5c04d8607d3a0ed29aa6179c092cbdda10d5d32684fb33ed01bd94f588ca8f91ac48318087dcb02eaf36773a7a453f0eedd6742af668097b29c@10.0.1.16:30303?discport=30304"), + enode.MustParse("enode://81fa361d25f157cd421c60dcc28d8dac5ef6a89476633339c5df30287474520caca09627da18543d9079b5b288698b542d56167aa5c09111e55acdbbdf2ef799@10.0.1.16:30303"), + enode.MustParse("enode://9bffefd833d53fac8e652415f4973bee289e8b1a5c6c4cbe70abf817ce8a64cee11b823b66a987f51aaa9fba0d6a91b3e6bf0d5a5d1042de8e9eeea057b217f8@10.0.1.36:30301?discport=17"), + enode.MustParse("enode://1b5b4aa662d7cb44a7221bfba67302590b643028197a7d5214790f3bac7aaa4a3241be9e83c09cf1f6c69d007c634faae3dc1b1221793e8446c0b3a09de65960@10.0.1.16:30303"), } rpclist := make([]v4wire.Node, len(list)) for i := range list { @@ -410,30 +363,28 @@ func TestUDPv4_findnodeMultiReply(t *testing.T) { // This test checks that reply matching of pong verifies the ping hash. func TestUDPv4_pingMatch(t *testing.T) { - logger := log.New() - test := newUDPTest(t, logger) + test := newUDPTest(t) defer test.close() randToken := make([]byte, 32) crand.Read(randToken) test.packetIn(nil, &v4wire.Ping{From: testRemote, To: testLocalAnnounced, Version: 4, Expiration: futureExp}) - test.waitPacketOut(func(*v4wire.Pong, *net.UDPAddr, []byte) {}) - test.waitPacketOut(func(*v4wire.Ping, *net.UDPAddr, []byte) {}) + test.waitPacketOut(func(*v4wire.Pong, netip.AddrPort, []byte) {}) + test.waitPacketOut(func(*v4wire.Ping, netip.AddrPort, []byte) {}) test.packetIn(errUnsolicitedReply, &v4wire.Pong{ReplyTok: randToken, To: testLocalAnnounced, Expiration: futureExp}) } // This test checks that reply matching of pong verifies the sender IP address. func TestUDPv4_pingMatchIP(t *testing.T) { - logger := log.New() - test := newUDPTest(t, logger) + test := newUDPTest(t) defer test.close() test.packetIn(nil, &v4wire.Ping{From: testRemote, To: testLocalAnnounced, Version: 4, Expiration: futureExp}) - test.waitPacketOut(func(*v4wire.Pong, *net.UDPAddr, []byte) {}) + test.waitPacketOut(func(*v4wire.Pong, netip.AddrPort, []byte) {}) - test.waitPacketOut(func(p *v4wire.Ping, to *net.UDPAddr, hash []byte) { - wrongAddr := &net.UDPAddr{IP: net.IP{33, 44, 1, 2}, Port: 30000} + test.waitPacketOut(func(p *v4wire.Ping, to netip.AddrPort, hash []byte) { + wrongAddr := netip.MustParseAddrPort("33.44.1.2:30000") test.packetInFrom(errUnsolicitedReply, test.remotekey, wrongAddr, &v4wire.Pong{ ReplyTok: hash, To: testLocalAnnounced, @@ -443,44 +394,37 @@ func TestUDPv4_pingMatchIP(t *testing.T) { } func TestUDPv4_successfulPing(t *testing.T) { - t.Skip("issue #15000") - logger := log.New() - test := newUDPTest(t, logger) - added := make(chan *node, 1) - test.table.nodeAddedHook = func(n *node) { added <- n } + test := newUDPTest(t) + added := make(chan *tableNode, 1) + test.table.nodeAddedHook = func(b *bucket, n *tableNode) { added <- n } defer test.close() // The remote side sends a ping packet to initiate the exchange. go test.packetIn(nil, &v4wire.Ping{From: testRemote, To: testLocalAnnounced, Version: 4, Expiration: futureExp}) // The ping is replied to. - test.waitPacketOut(func(p *v4wire.Pong, to *net.UDPAddr, hash []byte) { + test.waitPacketOut(func(p *v4wire.Pong, to netip.AddrPort, hash []byte) { pinghash := test.sent[0][:32] if !bytes.Equal(p.ReplyTok, pinghash) { t.Errorf("got pong.ReplyTok %x, want %x", p.ReplyTok, pinghash) } - wantTo := v4wire.Endpoint{ - // The mirrored UDP address is the UDP packet sender - IP: test.remoteaddr.IP, UDP: uint16(test.remoteaddr.Port), - // The mirrored TCP port is the one from the ping packet - TCP: testRemote.TCP, - } + // The mirrored UDP address is the UDP packet sender. + // The mirrored TCP port is the one from the ping packet. + wantTo := v4wire.NewEndpoint(test.remoteaddr, testRemote.TCP) if !reflect.DeepEqual(p.To, wantTo) { t.Errorf("got pong.To %v, want %v", p.To, wantTo) } }) // Remote is unknown, the table pings back. - test.waitPacketOut(func(p *v4wire.Ping, to *net.UDPAddr, hash []byte) { - if !reflect.DeepEqual(p.From, test.udp.ourEndpoint()) { + test.waitPacketOut(func(p *v4wire.Ping, to netip.AddrPort, hash []byte) { + wantFrom := test.udp.ourEndpoint() + wantFrom.IP = net.IP{} + if !reflect.DeepEqual(p.From, wantFrom) { t.Errorf("got ping.From %#v, want %#v", p.From, test.udp.ourEndpoint()) } - wantTo := v4wire.Endpoint{ - // The mirrored UDP address is the UDP packet sender. - IP: test.remoteaddr.IP, - UDP: uint16(test.remoteaddr.Port), - TCP: 0, - } + // The mirrored UDP address is the UDP packet sender. + wantTo := v4wire.NewEndpoint(test.remoteaddr, 0) if !reflect.DeepEqual(p.To, wantTo) { t.Errorf("got ping.To %v, want %v", p.To, wantTo) } @@ -491,15 +435,15 @@ func TestUDPv4_successfulPing(t *testing.T) { // pong packet. select { case n := <-added: - rid := enode.PubkeyToIDV4(&test.remotekey.PublicKey) + rid := v4wire.EncodePubkey(&test.remotekey.PublicKey).ID() if n.ID() != rid { t.Errorf("node has wrong ID: got %v, want %v", n.ID(), rid) } - if !n.IP().Equal(test.remoteaddr.IP) { - t.Errorf("node has wrong IP: got %v, want: %v", n.IP(), test.remoteaddr.IP) + if n.IPAddr() != test.remoteaddr.Addr() { + t.Errorf("node has wrong IP: got %v, want: %v", n.IPAddr(), test.remoteaddr.Addr()) } - if n.UDP() != test.remoteaddr.Port { - t.Errorf("node has wrong UDP port: got %v, want: %v", n.UDP(), test.remoteaddr.Port) + if n.UDP() != int(test.remoteaddr.Port()) { + t.Errorf("node has wrong UDP port: got %v, want: %v", n.UDP(), test.remoteaddr.Port()) } if n.TCP() != int(testRemote.TCP) { t.Errorf("node has wrong TCP port: got %v, want: %v", n.TCP(), testRemote.TCP) @@ -511,8 +455,7 @@ func TestUDPv4_successfulPing(t *testing.T) { // This test checks that EIP-868 requests work. func TestUDPv4_EIP868(t *testing.T) { - logger := log.New() - test := newUDPTest(t, logger) + test := newUDPTest(t) defer test.close() test.udp.localNode.Set(enr.WithEntry("foo", "bar")) @@ -523,12 +466,12 @@ func TestUDPv4_EIP868(t *testing.T) { // Perform endpoint proof and check for sequence number in packet tail. test.packetIn(nil, &v4wire.Ping{Expiration: futureExp}) - test.waitPacketOut(func(p *v4wire.Pong, addr *net.UDPAddr, hash []byte) { + test.waitPacketOut(func(p *v4wire.Pong, addr netip.AddrPort, hash []byte) { if p.ENRSeq != wantNode.Seq() { t.Errorf("wrong sequence number in pong: %d, want %d", p.ENRSeq, wantNode.Seq()) } }) - test.waitPacketOut(func(p *v4wire.Ping, addr *net.UDPAddr, hash []byte) { + test.waitPacketOut(func(p *v4wire.Ping, addr netip.AddrPort, hash []byte) { if p.ENRSeq != wantNode.Seq() { t.Errorf("wrong sequence number in ping: %d, want %d", p.ENRSeq, wantNode.Seq()) } @@ -537,26 +480,20 @@ func TestUDPv4_EIP868(t *testing.T) { // Request should work now. test.packetIn(nil, &v4wire.ENRRequest{Expiration: futureExp}) - test.waitPacketOut(func(p *v4wire.ENRResponse, addr *net.UDPAddr, hash []byte) { + test.waitPacketOut(func(p *v4wire.ENRResponse, addr netip.AddrPort, hash []byte) { n, err := enode.New(enode.ValidSchemes, &p.Record) if err != nil { t.Fatalf("invalid record: %v", err) } if !reflect.DeepEqual(n, wantNode) { - t.Fatalf("wrong node in enrResponse: %v", n) + t.Fatalf("wrong node in ENRResponse: %v", n) } }) } // This test verifies that a small network of nodes can boot up into a healthy state. func TestUDPv4_smallNetConvergence(t *testing.T) { - t.Skip("FIXME: https://github.com/erigontech/erigon/issues/8731") - t.Parallel() - logger := log.New() - - ctx := context.Background() - ctx = disableLookupSlowdown(ctx) // Start the network. nodes := make([]*UDPv4, 4) @@ -566,87 +503,65 @@ func TestUDPv4_smallNetConvergence(t *testing.T) { bn := nodes[0].Self() cfg.Bootnodes = []*enode.Node{bn} } - cfg.ReplyTimeout = 50 * time.Millisecond - cfg.PingBackDelay = time.Nanosecond - cfg.TableRevalidateInterval = time.Hour - - nodes[i] = startLocalhostV4(ctx, t, cfg, logger) + nodes[i] = startLocalhostV4(t, cfg) + defer nodes[i].Close() } - defer func() { - for _, node := range nodes { - node.Close() - } - }() - // Run through the iterator on all nodes until // they have all found each other. status := make(chan error, len(nodes)) for i := range nodes { - node := nodes[i] + self := nodes[i] go func() { - found := make(map[enode.ID]bool, len(nodes)) - it := node.RandomNodes() + missing := make(map[enode.ID]bool, len(nodes)) + for _, n := range nodes { + if n.Self().ID() == self.Self().ID() { + continue // skip self + } + missing[n.Self().ID()] = true + } + + it := self.RandomNodes() for it.Next() { - found[it.Node().ID()] = true - if len(found) == len(nodes) { + delete(missing, it.Node().ID()) + if len(missing) == 0 { status <- nil return } } - status <- fmt.Errorf("node %s didn't find all nodes", node.Self().ID().TerminalString()) + missingIDs := slices.Collect(maps.Keys(missing)) + status <- fmt.Errorf("node %s didn't find all nodes, missing %v", self.Self().ID().TerminalString(), missingIDs) }() } // Wait for all status reports. - timeout := time.NewTimer(5 * time.Second) + timeout := time.NewTimer(30 * time.Second) defer timeout.Stop() for received := 0; received < len(nodes); { select { case <-timeout.C: - t.Fatalf("Failed to converge within timeout") - return + for _, node := range nodes { + node.Close() + } case err := <-status: received++ if err != nil { t.Error("ERROR:", err) - return } } } } -type testLogHandler struct { - lprefix string - lfmt log.Format - t *testing.T -} - -func (h testLogHandler) Log(r *log.Record) error { - h.t.Logf("%s %s", h.lprefix, h.lfmt.Format(r)) - return nil -} - -func (h testLogHandler) Enabled(ctx context.Context, lvl log.Lvl) bool { - return true -} - -func startLocalhostV4(ctx context.Context, t *testing.T, cfg Config, logger log.Logger) *UDPv4 { +func startLocalhostV4(t *testing.T, cfg Config) *UDPv4 { t.Helper() cfg.PrivateKey = newkey() - tmpDir := t.TempDir() - db, err := enode.OpenDB(context.Background(), "", tmpDir, logger) - if err != nil { - panic(err) - } - ln := enode.NewLocalNode(db, cfg.PrivateKey, logger) + db, _ := enode.OpenDB("") + ln := enode.NewLocalNode(db, cfg.PrivateKey) // Prefix logs with node ID. lprefix := fmt.Sprintf("(%s)", ln.ID().TerminalString()) - lfmt := log.TerminalFormat() - cfg.Log = testlog.Logger(t, log.LvlError) - cfg.Log.SetHandler(testLogHandler{lprefix: lprefix, lfmt: lfmt, t: t}) + cfg.Log = testlog.Logger(t, log.LvlTrace).New("node-id", lprefix) // Listen. socket, err := net.ListenUDP("udp4", &net.UDPAddr{IP: net.IP{127, 0, 0, 1}}) @@ -656,69 +571,71 @@ func startLocalhostV4(ctx context.Context, t *testing.T, cfg Config, logger log. realaddr := socket.LocalAddr().(*net.UDPAddr) ln.SetStaticIP(realaddr.IP) ln.SetFallbackUDP(realaddr.Port) - udp, err := ListenV4(ctx, "test", socket, ln, cfg) + udp, err := ListenV4(socket, ln, cfg) if err != nil { t.Fatal(err) } - return udp -} - -func contextWithReplyTimeout(ctx context.Context, value time.Duration) context.Context { - return context.WithValue(ctx, "p2p.discover.Config.ReplyTimeout", value) //nolint:staticcheck -} - -func contextGetReplyTimeout(ctx context.Context) time.Duration { - value, _ := ctx.Value("p2p.discover.Config.ReplyTimeout").(time.Duration) //nolint:staticcheck - return value -} -func contextWithPrivateKeyGenerator(ctx context.Context, value func() (*ecdsa.PrivateKey, error)) context.Context { - return context.WithValue(ctx, "p2p.discover.Config.PrivateKeyGenerator", value) //nolint:staticcheck -} - -func contextGetPrivateKeyGenerator(ctx context.Context) func() (*ecdsa.PrivateKey, error) { - value, _ := ctx.Value("p2p.discover.Config.PrivateKeyGenerator").(func() (*ecdsa.PrivateKey, error)) //nolint:staticcheck - return value + // Wait for bootstrap to complete. + select { + case <-udp.tab.initDone: + case <-time.After(5 * time.Second): + t.Fatalf("timed out waiting for table initialization") + } + return udp } // dgramPipe is a fake UDP socket. It queues all sent datagrams. type dgramPipe struct { - queue chan dgram - closed chan struct{} + mu *sync.Mutex + cond *sync.Cond + closing chan struct{} + closed bool + queue []dgram } type dgram struct { - to net.UDPAddr + to netip.AddrPort data []byte } func newpipe() *dgramPipe { + mu := new(sync.Mutex) return &dgramPipe{ - make(chan dgram, 1000), - make(chan struct{}), + closing: make(chan struct{}), + cond: &sync.Cond{L: mu}, + mu: mu, } } -// WriteToUDP queues a datagram. -func (c *dgramPipe) WriteToUDP(b []byte, to *net.UDPAddr) (n int, err error) { +// WriteToUDPAddrPort queues a datagram. +func (c *dgramPipe) WriteToUDPAddrPort(b []byte, to netip.AddrPort) (n int, err error) { msg := make([]byte, len(b)) copy(msg, b) - - defer recover() - - c.queue <- dgram{*to, b} + c.mu.Lock() + defer c.mu.Unlock() + if c.closed { + return 0, errors.New("closed") + } + c.queue = append(c.queue, dgram{to, b}) + c.cond.Signal() return len(b), nil } -// ReadFromUDP just hangs until the pipe is closed. -func (c *dgramPipe) ReadFromUDP(b []byte) (n int, addr *net.UDPAddr, err error) { - <-c.closed - return 0, nil, io.EOF +// ReadFromUDPAddrPort just hangs until the pipe is closed. +func (c *dgramPipe) ReadFromUDPAddrPort(b []byte) (n int, addr netip.AddrPort, err error) { + <-c.closing + return 0, netip.AddrPort{}, io.EOF } func (c *dgramPipe) Close() error { - close(c.queue) - close(c.closed) + c.mu.Lock() + defer c.mu.Unlock() + if !c.closed { + close(c.closing) + c.closed = true + } + c.cond.Broadcast() return nil } @@ -727,16 +644,29 @@ func (c *dgramPipe) LocalAddr() net.Addr { } func (c *dgramPipe) receive() (dgram, error) { - ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) - defer cancel() + c.mu.Lock() + defer c.mu.Unlock() + + var timedOut bool + timer := time.AfterFunc(3*time.Second, func() { + c.mu.Lock() + timedOut = true + c.mu.Unlock() + c.cond.Broadcast() + }) + defer timer.Stop() - select { - case p, isOpen := <-c.queue: - if isOpen { - return p, nil - } + for len(c.queue) == 0 && !c.closed && !timedOut { + c.cond.Wait() + } + if c.closed { return dgram{}, errClosed - case <-ctx.Done(): + } + if timedOut { return dgram{}, errTimeout } + p := c.queue[0] + copy(c.queue, c.queue[1:]) + c.queue = c.queue[:len(c.queue)-1] + return p, nil } diff --git a/p2p/discover/v4wire/v4wire.go b/p2p/discover/v4wire/v4wire.go index e2b687d5a3e..ecff4d1909d 100644 --- a/p2p/discover/v4wire/v4wire.go +++ b/p2p/discover/v4wire/v4wire.go @@ -1,21 +1,18 @@ -// Copyright 2019 The go-ethereum Authors -// (original work) -// Copyright 2024 The Erigon Authors -// (modifications) -// This file is part of Erigon. +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. // -// Erigon is free software: you can redistribute it and/or modify +// The go-ethereum library is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // -// Erigon is distributed in the hope that it will be useful, +// The go-ethereum library is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Lesser General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License -// along with Erigon. If not, see . +// along with the go-ethereum library. If not, see . // Package v4wire implements the Discovery v4 Wire Protocol. package v4wire @@ -28,12 +25,14 @@ import ( "fmt" "math/big" "net" + "net/netip" "time" - "github.com/erigontech/erigon/common/crypto" "github.com/erigontech/erigon/common/math" - "github.com/erigontech/erigon/execution/rlp" + "github.com/erigontech/erigon/common/crypto" + "github.com/erigontech/erigon/p2p/enode" "github.com/erigontech/erigon/p2p/enr" + "github.com/erigontech/erigon/execution/rlp" ) // RPC packet types @@ -62,7 +61,7 @@ type ( Pong struct { // This field should mirror the UDP envelope address // of the ping packet, which provides a way to discover the - // the external address (after NAT). + // external address (after NAT). To Endpoint ReplyTok []byte // This contains the hash of the ping packet. Expiration uint64 // Absolute timestamp at which the packet becomes invalid. @@ -88,23 +87,23 @@ type ( Rest []rlp.RawValue `rlp:"tail"` } - // enrRequest queries for the remote node's record. + // ENRRequest queries for the remote node's record. ENRRequest struct { Expiration uint64 // Ignore additional fields (for forward compatibility). Rest []rlp.RawValue `rlp:"tail"` } - // enrResponse is the reply to enrRequest. + // ENRResponse is the reply to ENRRequest. ENRResponse struct { - ReplyTok []byte // Hash of the enrRequest packet. + ReplyTok []byte // Hash of the ENRRequest packet. Record enr.Record // Ignore additional fields (for forward compatibility). Rest []rlp.RawValue `rlp:"tail"` } ) -// This number is the maximum number of neighbor nodes in a Neigbors packet. +// MaxNeighbors is the maximum number of neighbor nodes in a Neighbors packet. const MaxNeighbors = 12 // This code computes the MaxNeighbors constant value. @@ -131,6 +130,11 @@ const MaxNeighbors = 12 // Pubkey represents an encoded 64-byte secp256k1 public key. type Pubkey [64]byte +// ID returns the node ID corresponding to the public key. +func (e Pubkey) ID() enode.ID { + return enode.ID(crypto.Keccak256Hash(e[:])) +} + // Node represents information about a node. type Node struct { IP net.IP // len 4 for IPv4 or 16 for IPv6 @@ -147,19 +151,21 @@ type Endpoint struct { } // NewEndpoint creates an endpoint. -func NewEndpoint(addr *net.UDPAddr, tcpPort uint16) Endpoint { - ip := net.IP{} - if ip4 := addr.IP.To4(); ip4 != nil { - ip = ip4 - } else if ip6 := addr.IP.To16(); ip6 != nil { - ip = ip6 +func NewEndpoint(addr netip.AddrPort, tcpPort uint16) Endpoint { + var ip net.IP + if addr.Addr().Is4() || addr.Addr().Is4In6() { + ip4 := addr.Addr().As4() + ip = ip4[:] + } else { + ip = addr.Addr().AsSlice() } - return Endpoint{IP: ip, UDP: uint16(addr.Port), TCP: tcpPort} + return Endpoint{IP: ip, UDP: addr.Port(), TCP: tcpPort} } type Packet interface { - // packet name and type for logging purposes. + // Name is the name of the package, for logging purposes. Name() string + // Kind is the packet type, for logging purposes. Kind() byte } @@ -234,6 +240,8 @@ func Decode(input []byte) (Packet, Pubkey, []byte, error) { default: return nil, fromKey, hash, fmt.Errorf("unknown type: %d", ptype) } + // Here we use NewStream to allow for additional data after the first + // RLP object (forward-compatibility). s := rlp.NewStream(bytes.NewReader(sigdata[1:]), 0) err = s.Decode(req) return req, fromKey, hash, err @@ -244,7 +252,7 @@ func Encode(priv *ecdsa.PrivateKey, req Packet) (packet, hash []byte, err error) b := new(bytes.Buffer) b.Write(headSpace) b.WriteByte(req.Kind()) - if err = rlp.Encode(b, req); err != nil { + if err := rlp.Encode(b, req); err != nil { return nil, nil, err } packet = b.Bytes() @@ -269,8 +277,7 @@ func recoverNodeKey(hash, sig []byte) (key Pubkey, err error) { return key, nil } -// EncodePubkey converts a public key into a binary format. -// The logic matches DecodePubkey. +// EncodePubkey encodes a secp256k1 public key. func EncodePubkey(key *ecdsa.PublicKey) Pubkey { var e Pubkey math.ReadBits(key.X, e[:len(e)/2]) diff --git a/p2p/discover/v4wire/v4wire_test.go b/p2p/discover/v4wire/v4wire_test.go index e580ba94776..5d1119b8419 100644 --- a/p2p/discover/v4wire/v4wire_test.go +++ b/p2p/discover/v4wire/v4wire_test.go @@ -1,21 +1,18 @@ -// Copyright 2019 The go-ethereum Authors -// (original work) -// Copyright 2024 The Erigon Authors -// (modifications) -// This file is part of Erigon. +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. // -// Erigon is free software: you can redistribute it and/or modify +// The go-ethereum library is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // -// Erigon is distributed in the hope that it will be useful, +// The go-ethereum library is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Lesser General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License -// along with Erigon. If not, see . +// along with the go-ethereum library. If not, see . package v4wire @@ -26,7 +23,6 @@ import ( "testing" "github.com/davecgh/go-spew/spew" - "github.com/erigontech/erigon/common/crypto" "github.com/erigontech/erigon/execution/rlp" ) diff --git a/p2p/discover/v5_lookup_test.go b/p2p/discover/v5_lookup_test.go deleted file mode 100644 index 4d5af2755e6..00000000000 --- a/p2p/discover/v5_lookup_test.go +++ /dev/null @@ -1,134 +0,0 @@ -// Copyright 2024 The Erigon Authors -// This file is part of Erigon. -// -// Erigon is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// Erigon is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with Erigon. If not, see . - -package discover - -import ( - "math/rand" - "net" - "runtime" - "sort" - "testing" - - "github.com/erigontech/erigon/common/log/v3" - "github.com/erigontech/erigon/p2p/discover/v5wire" - "github.com/erigontech/erigon/p2p/enode" -) - -// This test checks that lookup works. -func TestUDPv5_lookup(t *testing.T) { - t.Skip("issue #6223") - t.Parallel() - logger := log.New() - test := newUDPV5Test(t, logger) - t.Cleanup(test.close) - - // Lookup on empty table returns no nodes. - if results := test.udp.Lookup(lookupTestnet.target.ID()); len(results) > 0 { - t.Fatalf("lookup on empty table returned %d results: %#v", len(results), results) - } - - // Ensure the tester knows all nodes in lookupTestnet by IP. - for d, nn := range lookupTestnet.dists { - for i, key := range nn { - n := lookupTestnet.node(d, i) - test.getNode(key, &net.UDPAddr{IP: n.IP(), Port: n.UDP()}, logger) - } - } - - // Seed table with initial node. - initialNode := lookupTestnet.node(256, 0) - fillTable(test.table, []*node{wrapNode(initialNode)}) - - // Start the lookup. - resultC := make(chan []*enode.Node, 1) - go func() { - resultC <- test.udp.Lookup(lookupTestnet.target.ID()) - test.close() - }() - - // Answer lookup packets. - asked := make(map[enode.ID]bool) - for done := false; !done; { - done = test.waitPacketOut(func(p v5wire.Packet, to *net.UDPAddr, _ v5wire.Nonce) { - recipient, key := lookupTestnet.nodeByAddr(to) - switch p := p.(type) { - case *v5wire.Ping: - test.packetInFrom(key, to, &v5wire.Pong{ReqID: p.ReqID}, logger) - case *v5wire.Findnode: - if asked[recipient.ID()] { - t.Error("Asked node", recipient.ID(), "twice") - } - asked[recipient.ID()] = true - nodes := lookupTestnet.neighborsAtDistances(recipient, p.Distances, 16) - t.Logf("Got FINDNODE for %v, returning %d nodes", p.Distances, len(nodes)) - for _, resp := range packNodes(p.ReqID, nodes) { - test.packetInFrom(key, to, resp, logger) - } - } - }) - } - - // Verify result nodes. - results := <-resultC - checkLookupResults(t, lookupTestnet, results) -} - -// Real sockets, real crypto: this test checks end-to-end connectivity for UDPv5. -func TestUDPv5_lookupE2E(t *testing.T) { - if runtime.GOOS != "linux" { - t.Skip("fix me on win please") - } - t.Parallel() - logger := log.New() - - bootNode := startLocalhostV5(t, Config{}, logger) - bootNodeRec := bootNode.Self() - - const N = 5 - nodes := []*UDPv5{bootNode} - for len(nodes) < N { - cfg := Config{ - Bootnodes: []*enode.Node{bootNodeRec}, - } - node := startLocalhostV5(t, cfg, logger) - nodes = append(nodes, node) - } - - defer func() { - for _, node := range nodes { - node.Close() - } - }() - - last := nodes[N-1] - target := nodes[rand.Intn(N-2)].Self() - - // It is expected that all nodes can be found. - expectedResult := make([]*enode.Node, len(nodes)) - for i := range nodes { - expectedResult[i] = nodes[i].Self() - } - sort.Slice(expectedResult, func(i, j int) bool { - return enode.DistCmp(target.ID(), expectedResult[i].ID(), expectedResult[j].ID()) < 0 - }) - - // Do the lookup. - results := last.Lookup(target.ID()) - if err := checkNodesEqual(results, expectedResult); err != nil { - t.Fatalf("lookup returned wrong results: %v", err) - } -} diff --git a/p2p/discover/v5_talk.go b/p2p/discover/v5_talk.go new file mode 100644 index 00000000000..b1c9b8508cb --- /dev/null +++ b/p2p/discover/v5_talk.go @@ -0,0 +1,121 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package discover + +import ( + "net" + "net/netip" + "sync" + "time" + + "github.com/erigontech/erigon/common/log/v3" + "github.com/erigontech/erigon/p2p/discover/v5wire" + "github.com/erigontech/erigon/p2p/enode" +) + +// This is a limit for the number of concurrent talk requests. +const maxActiveTalkRequests = 1024 + +// This is the timeout for acquiring a handler execution slot for a talk request. +// The timeout should be short enough to fit within the request timeout. +const talkHandlerLaunchTimeout = 400 * time.Millisecond + +// TalkRequestHandler callback processes a talk request and returns a response. +// +// Note that talk handlers are expected to come up with a response very quickly, within at +// most 200ms or so. If the handler takes longer than that, the remote end may time out +// and wont receive the response. +type TalkRequestHandler func(*enode.Node, *net.UDPAddr, []byte) []byte + +type talkSystem struct { + transport *UDPv5 + + mutex sync.Mutex + handlers map[string]TalkRequestHandler + slots chan struct{} + lastLog time.Time + dropCount int +} + +func newTalkSystem(transport *UDPv5) *talkSystem { + t := &talkSystem{ + transport: transport, + handlers: make(map[string]TalkRequestHandler), + slots: make(chan struct{}, maxActiveTalkRequests), + } + for i := 0; i < cap(t.slots); i++ { + t.slots <- struct{}{} + } + return t +} + +// register adds a protocol handler. +func (t *talkSystem) register(protocol string, handler TalkRequestHandler) { + t.mutex.Lock() + t.handlers[protocol] = handler + t.mutex.Unlock() +} + +// handleRequest handles a talk request. +func (t *talkSystem) handleRequest(id enode.ID, addr netip.AddrPort, req *v5wire.TalkRequest) { + n := t.transport.codec.SessionNode(id, addr.String()) + if n == nil { + // The node must be contained in the session here, since we wouldn't have + // received the request otherwise. + panic("missing node in session") + } + t.mutex.Lock() + handler, ok := t.handlers[req.Protocol] + t.mutex.Unlock() + + if !ok { + resp := &v5wire.TalkResponse{ReqID: req.ReqID} + t.transport.sendResponse(n.ID(), addr, resp) + return + } + + // Wait for a slot to become available, then run the handler. + timeout := time.NewTimer(talkHandlerLaunchTimeout) + defer timeout.Stop() + select { + case <-t.slots: + go func() { + defer func() { t.slots <- struct{}{} }() + udpAddr := &net.UDPAddr{IP: addr.Addr().AsSlice(), Port: int(addr.Port())} + respMessage := handler(n, udpAddr, req.Message) + resp := &v5wire.TalkResponse{ReqID: req.ReqID, Message: respMessage} + t.transport.sendFromAnotherThread(n.ID(), addr, resp) + }() + case <-timeout.C: + // Couldn't get it in time, drop the request. + if time.Since(t.lastLog) > 5*time.Second { + log.Warn("Dropping TALKREQ due to overload", "ndrop", t.dropCount) + t.lastLog = time.Now() + t.dropCount++ + } + case <-t.transport.closeCtx.Done(): + // Transport closed, drop the request. + } +} + +// wait blocks until all active requests have finished, and prevents new request +// handlers from being launched. +func (t *talkSystem) wait() { + for i := 0; i < cap(t.slots); i++ { + <-t.slots + } +} diff --git a/p2p/discover/v5_udp.go b/p2p/discover/v5_udp.go index 11603f459f5..5943ca0d3d6 100644 --- a/p2p/discover/v5_udp.go +++ b/p2p/discover/v5_udp.go @@ -1,21 +1,18 @@ -// Copyright 2019 The go-ethereum Authors -// (original work) -// Copyright 2024 The Erigon Authors -// (modifications) -// This file is part of Erigon. +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. // -// Erigon is free software: you can redistribute it and/or modify +// The go-ethereum library is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // -// Erigon is distributed in the hope that it will be useful, +// The go-ethereum library is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Lesser General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License -// along with Erigon. If not, see . +// along with the go-ethereum library. If not, see . package discover @@ -27,15 +24,14 @@ import ( "errors" "fmt" "io" - "math" "net" + "net/netip" "slices" "sync" "time" - "github.com/erigontech/erigon/common/dbg" - "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/common/mclock" + "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/p2p/discover/v5wire" "github.com/erigontech/erigon/p2p/enode" "github.com/erigontech/erigon/p2p/enr" @@ -46,9 +42,6 @@ const ( lookupRequestLimit = 3 // max requests against a single node during lookup findnodeResultLimit = 16 // applies in FINDNODE handler totalNodesResponseLimit = 5 // applies in waitForNodes - nodesResponseItemLimit = 3 // applies in sendNodes - - respTimeoutV5 = 700 * time.Millisecond ) // codecV5 is implemented by v5wire.Codec (and testCodec). @@ -57,11 +50,23 @@ const ( // encoding/decoding and with the handshake; the UDPv5 object handles higher-level concerns. type codecV5 interface { // Encode encodes a packet. - Encode(enode.ID, string, v5wire.Packet, *v5wire.Whoareyou) ([]byte, v5wire.Nonce, error) - - // decode decodes a packet. It returns a *v5wire.Unknown packet if decryption fails. + // + // If the underlying type of 'p' is *v5wire.Whoareyou, a Whoareyou challenge packet is + // encoded. If the 'challenge' parameter is non-nil, the packet is encoded as a + // handshake message packet. Otherwise, the packet will be encoded as an ordinary + // message packet. + Encode(id enode.ID, addr string, p v5wire.Packet, challenge *v5wire.Whoareyou) ([]byte, v5wire.Nonce, error) + + // Decode decodes a packet. It returns a *v5wire.Unknown packet if decryption fails. // The *enode.Node return value is non-nil when the input contains a handshake response. - Decode([]byte, string) (enode.ID, *enode.Node, v5wire.Packet, error) + Decode(b []byte, addr string) (enode.ID, *enode.Node, v5wire.Packet, error) + + // CurrentChallenge returns the most recent WHOAREYOU challenge that was encoded to given node. + // This will return a non-nil value if there is an active handshake attempt with the node, and nil otherwise. + CurrentChallenge(id enode.ID, addr string) *v5wire.Whoareyou + + // SessionNode returns a node that has completed the handshake. + SessionNode(id enode.ID, addr string) *enode.Node } // UDPv5 is the implementation of protocol version 5. @@ -76,10 +81,13 @@ type UDPv5 struct { log log.Logger clock mclock.Clock validSchemes enr.IdentityScheme + respTimeout time.Duration + + // misc buffers used during message handling + logcontext []interface{} // talkreq handler registry - trlock sync.Mutex - trhandlers map[string]TalkRequestHandler + talk *talkSystem // channels into dispatch packetInCh chan ReadPacket @@ -87,7 +95,8 @@ type UDPv5 struct { callCh chan *callV5 callDoneCh chan *callV5 respTimeoutCh chan *callTimeout - replyTimeout time.Duration + sendCh chan sendRequest + unhandled chan<- ReadPacket // state of dispatch codec codecV5 @@ -100,17 +109,20 @@ type UDPv5 struct { closeCtx context.Context cancelCloseCtx context.CancelFunc wg sync.WaitGroup - errors map[string]uint - - trace bool } -// TalkRequestHandler callback processes a talk request and optionally returns a reply -type TalkRequestHandler func(enode.ID, *net.UDPAddr, []byte) []byte +type sendRequest struct { + destID enode.ID + destAddr netip.AddrPort + msg v5wire.Packet +} // callV5 represents a remote procedure call against another node. type callV5 struct { - node *enode.Node + id enode.ID + addr netip.AddrPort + node *enode.Node // This is required to perform handshakes. + packet v5wire.Packet responseType byte // expected packet type of response reqid []byte @@ -131,8 +143,8 @@ type callTimeout struct { } // ListenV5 listens on the given connection. -func ListenV5(ctx context.Context, protocol string, conn UDPConn, ln *enode.LocalNode, cfg Config) (*UDPv5, error) { - t, err := newUDPv5(ctx, protocol, conn, ln, cfg) +func ListenV5(conn UDPConn, ln *enode.LocalNode, cfg Config) (*UDPv5, error) { + t, err := newUDPv5(conn, ln, cfg) if err != nil { return nil, err } @@ -144,12 +156,12 @@ func ListenV5(ctx context.Context, protocol string, conn UDPConn, ln *enode.Loca } // newUDPv5 creates a UDPv5 transport, but doesn't start any goroutines. -func newUDPv5(ctx context.Context, protocol string, conn UDPConn, ln *enode.LocalNode, cfg Config) (*UDPv5, error) { - closeCtx, cancelCloseCtx := context.WithCancel(ctx) - cfg = cfg.withDefaults(respTimeoutV5) +func newUDPv5(conn UDPConn, ln *enode.LocalNode, cfg Config) (*UDPv5, error) { + closeCtx, cancelCloseCtx := context.WithCancel(context.Background()) + cfg = cfg.withDefaults() t := &UDPv5{ // static fields - conn: conn, + conn: newMeteredConn(conn), localNode: ln, db: ln.Database(), netrestrict: cfg.NetRestrict, @@ -157,25 +169,26 @@ func newUDPv5(ctx context.Context, protocol string, conn UDPConn, ln *enode.Loca log: cfg.Log, validSchemes: cfg.ValidSchemes, clock: cfg.Clock, - trhandlers: make(map[string]TalkRequestHandler), + respTimeout: cfg.V5RespTimeout, // channels into dispatch packetInCh: make(chan ReadPacket, 1), readNextCh: make(chan struct{}, 1), callCh: make(chan *callV5), callDoneCh: make(chan *callV5), + sendCh: make(chan sendRequest), respTimeoutCh: make(chan *callTimeout), - replyTimeout: cfg.ReplyTimeout, + unhandled: cfg.Unhandled, // state of dispatch - codec: v5wire.NewCodec(ln, cfg.PrivateKey, cfg.Clock), + codec: v5wire.NewCodec(ln, cfg.PrivateKey, cfg.Clock, cfg.V5ProtocolID), activeCallByNode: make(map[enode.ID]*callV5), activeCallByAuth: make(map[v5wire.Nonce]*callV5), callQueue: make(map[enode.ID][]*callV5), // shutdown closeCtx: closeCtx, cancelCloseCtx: cancelCloseCtx, - errors: map[string]uint{}, } - tab, err := newTable(t, protocol, t.db, cfg.Bootnodes, cfg.TableRevalidateInterval, cfg.Log) + t.talk = newTalkSystem(t) + tab, err := newTable(t, t.db, cfg) if err != nil { return nil, err } @@ -188,34 +201,17 @@ func (t *UDPv5) Self() *enode.Node { return t.localNode.Node() } -func (t *UDPv5) Version() string { - return "v5" -} - -func (t *UDPv5) Errors() map[string]uint { - return t.errors -} - -func (t *UDPv5) LenUnsolicited() int { - return 0 -} - // Close shuts down packet processing. func (t *UDPv5) Close() { t.closeOnce.Do(func() { t.cancelCloseCtx() t.conn.Close() + t.talk.wait() t.wg.Wait() t.tab.close() }) } -// Ping sends a ping message to the given node. -func (t *UDPv5) Ping(n *enode.Node) error { - _, err := t.ping(n) - return err -} - // Resolve searches for a specific node with the given ID and tries to get the most recent // version of the node record for it. It returns n if the node could not be resolved. func (t *UDPv5) Resolve(n *enode.Node) *enode.Node { @@ -236,25 +232,62 @@ func (t *UDPv5) Resolve(n *enode.Node) *enode.Node { return n } +// ResolveNodeId searches for a specific Node with the given ID. +// It returns nil if the nodeId could not be resolved. +func (t *UDPv5) ResolveNodeId(id enode.ID) *enode.Node { + if id == t.Self().ID() { + return t.Self() + } + + n := t.tab.getNode(id) + if n != nil { + // Try asking directly. This works if the Node is still responding on the endpoint we have. + if resp, err := t.RequestENR(n); err == nil { + return resp + } + } + + // Otherwise do a network lookup. + result := t.Lookup(id) + for _, rn := range result { + if rn.ID() == id { + if n != nil && rn.Seq() <= n.Seq() { + return n + } else { + return rn + } + } + } + + return n +} + // AllNodes returns all the nodes stored in the local table. func (t *UDPv5) AllNodes() []*enode.Node { t.tab.mutex.Lock() defer t.tab.mutex.Unlock() - capacity := 0 - for _, b := range &t.tab.buckets { - capacity += len(b.entries) - } - nodes := make([]*enode.Node, 0, capacity) + nodes := make([]*enode.Node, 0) for _, b := range &t.tab.buckets { for _, n := range b.entries { - nodes = append(nodes, unwrapNode(n)) + nodes = append(nodes, n.Node) } } return nodes } -// LocalNode returns the current local node running the +// AddKnownNode adds a node to the routing table. +// The function should be used for testing only. +func (t *UDPv5) AddKnownNode(n *enode.Node) bool { + return t.tab.addFoundNode(n, true) +} + +// DeleteNode removes a node from the routing table. Used for Portal discv5 DeleteEnr API. +func (t *UDPv5) DeleteNode(n *enode.Node) { + t.tab.deleteNode(n) +} + +// LocalNode returns the current local Node running the // protocol. func (t *UDPv5) LocalNode() *enode.LocalNode { return t.localNode @@ -264,15 +297,13 @@ func (t *UDPv5) LocalNode() *enode.LocalNode { // whenever a request for the given protocol is received and should return the response // data or nil. func (t *UDPv5) RegisterTalkHandler(protocol string, handler TalkRequestHandler) { - t.trlock.Lock() - defer t.trlock.Unlock() - t.trhandlers[protocol] = handler + t.talk.register(protocol, handler) } -// TalkRequest sends a talk request to n and waits for a response. +// TalkRequest sends a talk request to a node and waits for a response. func (t *UDPv5) TalkRequest(n *enode.Node, protocol string, request []byte) ([]byte, error) { req := &v5wire.TalkRequest{Protocol: protocol, Message: request} - resp := t.call(n, v5wire.TalkResponseMsg, req) + resp := t.callToNode(n, v5wire.TalkResponseMsg, req) defer t.callDone(resp) select { case respMsg := <-resp.ch: @@ -282,14 +313,21 @@ func (t *UDPv5) TalkRequest(n *enode.Node, protocol string, request []byte) ([]b } } -// RandomNodes returns an iterator that finds random nodes in the DHT. -func (t *UDPv5) RandomNodes() enode.Iterator { - if t.tab.len() == 0 { - // All nodes were dropped, refresh. The very first query will hit this - // case and run the bootstrapping logic. - <-t.tab.refresh() +// TalkRequestToID sends a talk request to a node and waits for a response. +func (t *UDPv5) TalkRequestToID(id enode.ID, addr netip.AddrPort, protocol string, request []byte) ([]byte, error) { + req := &v5wire.TalkRequest{Protocol: protocol, Message: request} + resp := t.callToID(id, addr, v5wire.TalkResponseMsg, req) + defer t.callDone(resp) + select { + case respMsg := <-resp.ch: + return respMsg.(*v5wire.TalkResponse).Message, nil + case err := <-resp.err: + return nil, err } +} +// RandomNodes returns an iterator that finds random nodes in the DHT. +func (t *UDPv5) RandomNodes() enode.Iterator { return newLookupIterator(t.closeCtx, t.newRandomLookup) } @@ -318,26 +356,26 @@ func (t *UDPv5) newRandomLookup(ctx context.Context) *lookup { } func (t *UDPv5) newLookup(ctx context.Context, target enode.ID) *lookup { - return newLookup(ctx, t.tab, target, func(n *node) ([]*node, error) { + return newLookup(ctx, t.tab, target, func(n *enode.Node) ([]*enode.Node, error) { return t.lookupWorker(n, target) }) } // lookupWorker performs FINDNODE calls against a single node during lookup. -func (t *UDPv5) lookupWorker(destNode *node, target enode.ID) ([]*node, error) { +func (t *UDPv5) lookupWorker(destNode *enode.Node, target enode.ID) ([]*enode.Node, error) { var ( dists = lookupDistances(target, destNode.ID()) nodes = nodesByDistance{target: target} err error ) var r []*enode.Node - r, err = t.findnode(unwrapNode(destNode), dists) + r, err = t.Findnode(destNode, dists) if errors.Is(err, errClosed) { return nil, err } for _, n := range r { if n.ID() != t.Self().ID() { - nodes.push(wrapNode(n), findnodeResultLimit) + nodes.push(n, findnodeResultLimit) } } return nodes.entries, err @@ -350,7 +388,7 @@ func lookupDistances(target, dest enode.ID) (dists []uint) { td := enode.LogDist(target, dest) dists = append(dists, uint(td)) for i := 1; len(dists) < lookupRequestLimit; i++ { - if td+i < 256 { + if td+i <= 256 { dists = append(dists, uint(td+i)) } if td-i > 0 { @@ -362,21 +400,31 @@ func lookupDistances(target, dest enode.ID) (dists []uint) { // ping calls PING on a node and waits for a PONG response. func (t *UDPv5) ping(n *enode.Node) (uint64, error) { + pong, err := t.Ping(n) + if err != nil { + return 0, err + } + + return pong.ENRSeq, nil +} + +// Ping calls PING on a node and waits for a PONG response. +func (t *UDPv5) Ping(n *enode.Node) (*v5wire.Pong, error) { req := &v5wire.Ping{ENRSeq: t.localNode.Node().Seq()} - resp := t.call(n, v5wire.PongMsg, req) + resp := t.callToNode(n, v5wire.PongMsg, req) defer t.callDone(resp) select { case pong := <-resp.ch: - return pong.(*v5wire.Pong).ENRSeq, nil + return pong.(*v5wire.Pong), nil case err := <-resp.err: - return 0, err + return nil, err } } -// requestENR requests n's record. +// RequestENR requests n's record. func (t *UDPv5) RequestENR(n *enode.Node) (*enode.Node, error) { - nodes, err := t.findnode(n, []uint{0}) + nodes, err := t.Findnode(n, []uint{0}) if err != nil { return nil, err } @@ -386,9 +434,9 @@ func (t *UDPv5) RequestENR(n *enode.Node) (*enode.Node, error) { return nodes[0], nil } -// findnode calls FINDNODE on a node and waits for responses. -func (t *UDPv5) findnode(n *enode.Node, distances []uint) ([]*enode.Node, error) { - resp := t.call(n, v5wire.NodesMsg, &v5wire.Findnode{Distances: distances}) +// Findnode calls FINDNODE on a node and waits for responses. +func (t *UDPv5) Findnode(n *enode.Node, distances []uint) ([]*enode.Node, error) { + resp := t.callToNode(n, v5wire.NodesMsg, &v5wire.Findnode{Distances: distances}) return t.waitForNodes(resp, distances) } @@ -408,13 +456,13 @@ func (t *UDPv5) waitForNodes(c *callV5, distances []uint) ([]*enode.Node, error) for _, record := range response.Nodes { node, err := t.verifyResponseNode(c, record, distances, seen) if err != nil { - t.log.Trace("Invalid record in "+response.Name(), "id", c.node.ID(), "err", err) + t.log.Debug("Invalid record in "+response.Name(), "id", c.node.ID(), "err", err) continue } nodes = append(nodes, node) } if total == -1 { - total = min(int(response.Total), totalNodesResponseLimit) + total = min(int(response.RespCount), totalNodesResponseLimit) } if received++; received == total { return nodes, nil @@ -431,14 +479,17 @@ func (t *UDPv5) verifyResponseNode(c *callV5, r *enr.Record, distances []uint, s if err != nil { return nil, err } - if err := netutil.CheckRelayIP(c.node.IP(), node.IP()); err != nil { + if err := netutil.CheckRelayAddr(c.addr.Addr(), node.IPAddr()); err != nil { return nil, err } - if c.node.UDP() <= 1024 { + if t.netrestrict != nil && !t.netrestrict.ContainsAddr(node.IPAddr()) { + return nil, errors.New("not contained in netrestrict list") + } + if node.UDP() <= 1024 { return nil, errLowPort } if distances != nil { - nd := enode.LogDist(c.node.ID(), node.ID()) + nd := enode.LogDist(c.id, node.ID()) if !slices.Contains(distances, uint(nd)) { return nil, errors.New("does not match any requested distance") } @@ -450,17 +501,28 @@ func (t *UDPv5) verifyResponseNode(c *callV5, r *enr.Record, distances []uint, s return node, nil } -// call sends the given call and sets up a handler for response packets (of message type -// responseType). Responses are dispatched to the call's response channel. -func (t *UDPv5) call(node *enode.Node, responseType byte, packet v5wire.Packet) *callV5 { - c := &callV5{ - node: node, - packet: packet, - responseType: responseType, - reqid: make([]byte, 8), - ch: make(chan v5wire.Packet, 1), - err: make(chan error, 1), - } +// callToNode sends the given call and sets up a handler for response packets (of message +// type responseType). Responses are dispatched to the call's response channel. +func (t *UDPv5) callToNode(n *enode.Node, responseType byte, req v5wire.Packet) *callV5 { + addr, _ := n.UDPEndpoint() + c := &callV5{id: n.ID(), addr: addr, node: n} + t.initCall(c, responseType, req) + return c +} + +// callToID is like callToNode, but for cases where the node record is not available. +func (t *UDPv5) callToID(id enode.ID, addr netip.AddrPort, responseType byte, req v5wire.Packet) *callV5 { + c := &callV5{id: id, addr: addr} + t.initCall(c, responseType, req) + return c +} + +func (t *UDPv5) initCall(c *callV5, responseType byte, packet v5wire.Packet) { + c.packet = packet + c.responseType = responseType + c.reqid = make([]byte, 8) + c.ch = make(chan v5wire.Packet, 1) + c.err = make(chan error, 1) // Assign request ID. crand.Read(c.reqid) packet.SetRequestID(c.reqid) @@ -470,7 +532,6 @@ func (t *UDPv5) call(node *enode.Node, responseType byte, packet v5wire.Packet) case <-t.closeCtx.Done(): c.err <- errClosed } - return c } // callDone tells dispatch that the active call is done. @@ -504,7 +565,6 @@ func (t *UDPv5) callDone(c *callV5) { // When that happens the call is simply re-sent to complete the handshake. We allow one // handshake attempt per call. func (t *UDPv5) dispatch() { - defer dbg.LogPanic() defer t.wg.Done() // Arm first read. @@ -513,26 +573,27 @@ func (t *UDPv5) dispatch() { for { select { case c := <-t.callCh: - id := c.node.ID() - t.callQueue[id] = append(t.callQueue[id], c) - t.sendNextCall(id) + t.callQueue[c.id] = append(t.callQueue[c.id], c) + t.sendNextCall(c.id) case ct := <-t.respTimeoutCh: - active := t.activeCallByNode[ct.c.node.ID()] + active := t.activeCallByNode[ct.c.id] if ct.c == active && ct.timer == active.timeout { ct.c.err <- errTimeout } case c := <-t.callDoneCh: - id := c.node.ID() - active := t.activeCallByNode[id] + active := t.activeCallByNode[c.id] if active != c { panic("BUG: callDone for inactive call") } c.timeout.Stop() delete(t.activeCallByAuth, c.nonce) - delete(t.activeCallByNode, id) - t.sendNextCall(id) + delete(t.activeCallByNode, c.id) + t.sendNextCall(c.id) + + case r := <-t.sendCh: + t.send(r.destID, r.destAddr, r.msg, nil) case p := <-t.packetInCh: t.handlePacket(p.Data, p.Addr) @@ -566,7 +627,7 @@ func (t *UDPv5) startResponseTimeout(c *callV5) { timer mclock.Timer done = make(chan struct{}) ) - timer = t.clock.AfterFunc(t.replyTimeout, func() { + timer = t.clock.AfterFunc(t.respTimeout, func() { <-done select { case t.respTimeoutCh <- &callTimeout{c, timer}: @@ -602,8 +663,7 @@ func (t *UDPv5) sendCall(c *callV5) { delete(t.activeCallByAuth, c.nonce) } - addr := &net.UDPAddr{IP: c.node.IP(), Port: c.node.UDP()} - newNonce, _ := t.send(c.node.ID(), addr, c.packet, c.challenge) + newNonce, _ := t.send(c.id, c.addr, c.packet, c.challenge) c.nonce = newNonce t.activeCallByAuth[newNonce] = c t.startResponseTimeout(c) @@ -611,44 +671,51 @@ func (t *UDPv5) sendCall(c *callV5) { // sendResponse sends a response packet to the given node. // This doesn't trigger a handshake even if no keys are available. -func (t *UDPv5) sendResponse(toID enode.ID, toAddr *net.UDPAddr, packet v5wire.Packet) error { +func (t *UDPv5) sendResponse(toID enode.ID, toAddr netip.AddrPort, packet v5wire.Packet) error { _, err := t.send(toID, toAddr, packet, nil) return err } +func (t *UDPv5) sendFromAnotherThread(toID enode.ID, toAddr netip.AddrPort, packet v5wire.Packet) { + select { + case t.sendCh <- sendRequest{toID, toAddr, packet}: + case <-t.closeCtx.Done(): + } +} + // send sends a packet to the given node. -func (t *UDPv5) send(toID enode.ID, toAddr *net.UDPAddr, packet v5wire.Packet, c *v5wire.Whoareyou) (v5wire.Nonce, error) { +func (t *UDPv5) send(toID enode.ID, toAddr netip.AddrPort, packet v5wire.Packet, c *v5wire.Whoareyou) (v5wire.Nonce, error) { addr := toAddr.String() + t.logcontext = append(t.logcontext[:0], "id", toID, "addr", addr) + t.logcontext = packet.AppendLogInfo(t.logcontext) + enc, nonce, err := t.codec.Encode(toID, addr, packet, c) if err != nil { - if t.trace { - t.log.Warn(">> "+packet.Name(), "id", toID, "addr", addr, "err", err) - } + t.logcontext = append(t.logcontext, "err", err) + t.log.Warn(">> "+packet.Name(), t.logcontext...) return nonce, err } - _, err = t.conn.WriteToUDP(enc, toAddr) - if t.trace { - t.log.Trace(">> "+packet.Name(), "id", toID, "addr", addr) - } + + _, err = t.conn.WriteToUDPAddrPort(enc, toAddr) + t.log.Trace(">> "+packet.Name(), t.logcontext...) return nonce, err } // readLoop runs in its own goroutine and reads packets from the network. func (t *UDPv5) readLoop() { - defer dbg.LogPanic() defer t.wg.Done() buf := make([]byte, maxPacketSize) for range t.readNextCh { - nbytes, from, err := t.conn.ReadFromUDP(buf) + nbytes, from, err := t.conn.ReadFromUDPAddrPort(buf) if netutil.IsTemporaryError(err) { // Ignore temporary read errors. - t.log.Trace("Temporary UDP read error", "err", err) + t.log.Debug("Temporary UDP read error", "err", err) continue } else if err != nil { - // Shut down the loop for permament errors. - if err != io.EOF { - t.log.Trace("UDP read error", "err", err) + // Shut down the loop for permanent errors. + if !errors.Is(err, io.EOF) { + t.log.Debug("UDP read error", "err", err) } return } @@ -657,7 +724,11 @@ func (t *UDPv5) readLoop() { } // dispatchReadPacket sends a packet into the dispatch loop. -func (t *UDPv5) dispatchReadPacket(from *net.UDPAddr, content []byte) bool { +func (t *UDPv5) dispatchReadPacket(from netip.AddrPort, content []byte) bool { + // Unwrap IPv4-in-6 source address. + if from.Addr().Is4In6() { + from = netip.AddrPortFrom(netip.AddrFrom4(from.Addr().As4()), from.Port()) + } select { case t.packetInCh <- ReadPacket{content, from}: return true @@ -667,40 +738,48 @@ func (t *UDPv5) dispatchReadPacket(from *net.UDPAddr, content []byte) bool { } // handlePacket decodes and processes an incoming packet from the network. -func (t *UDPv5) handlePacket(rawpacket []byte, fromAddr *net.UDPAddr) error { +func (t *UDPv5) handlePacket(rawpacket []byte, fromAddr netip.AddrPort) error { addr := fromAddr.String() fromID, fromNode, packet, err := t.codec.Decode(rawpacket, addr) if err != nil { - t.log.Trace("Bad discv5 packet", "id", fromID, "addr", addr, "err", err) + if t.unhandled != nil && v5wire.IsInvalidHeader(err) { + // The packet seems unrelated to discv5, send it to the next protocol. + // t.log.Trace("Unhandled discv5 packet", "id", fromID, "addr", addr, "err", err) + up := ReadPacket{Data: make([]byte, len(rawpacket)), Addr: fromAddr} + copy(up.Data, rawpacket) + t.unhandled <- up + return nil + } + t.log.Debug("Bad discv5 packet", "id", fromID, "addr", addr, "err", err) return err } if fromNode != nil { // Handshake succeeded, add to table. - t.tab.addSeenNode(wrapNode(fromNode)) + t.tab.addInboundNode(fromNode) } if packet.Kind() != v5wire.WhoareyouPacket { // WHOAREYOU logged separately to report errors. - if t.trace { - t.log.Trace("<< "+packet.Name(), "id", fromID, "addr", addr) - } + t.logcontext = append(t.logcontext[:0], "id", fromID, "addr", addr) + t.logcontext = packet.AppendLogInfo(t.logcontext) + t.log.Trace("<< "+packet.Name(), t.logcontext...) } t.handle(packet, fromID, fromAddr) return nil } // handleCallResponse dispatches a response packet to the call waiting for it. -func (t *UDPv5) handleCallResponse(fromID enode.ID, fromAddr *net.UDPAddr, p v5wire.Packet) bool { +func (t *UDPv5) handleCallResponse(fromID enode.ID, fromAddr netip.AddrPort, p v5wire.Packet) bool { ac := t.activeCallByNode[fromID] if ac == nil || !bytes.Equal(p.RequestID(), ac.reqid) { - t.log.Trace(fmt.Sprintf("Unsolicited/late %s response", p.Name()), "id", fromID, "addr", fromAddr) + t.log.Debug(fmt.Sprintf("Unsolicited/late %s response", p.Name()), "id", fromID, "addr", fromAddr) return false } - if !fromAddr.IP.Equal(ac.node.IP()) || fromAddr.Port != ac.node.UDP() { - t.log.Trace(p.Name()+" from wrong endpoint", "id", fromID, "addr", fromAddr) + if fromAddr != ac.addr { + t.log.Debug(fmt.Sprintf("%s from wrong endpoint", p.Name()), "id", fromID, "addr", fromAddr) return false } if p.Kind() != ac.responseType { - t.log.Trace("Wrong discv5 response type "+p.Name(), "id", fromID, "addr", fromAddr) + t.log.Debug(fmt.Sprintf("Wrong discv5 response type %s", p.Name()), "id", fromID, "addr", fromAddr) return false } t.startResponseTimeout(ac) @@ -708,8 +787,8 @@ func (t *UDPv5) handleCallResponse(fromID enode.ID, fromAddr *net.UDPAddr, p v5w return true } -// getNode looks for a node record in table and database. -func (t *UDPv5) getNode(id enode.ID) *enode.Node { +// GetNode looks for a node record in table and database. +func (t *UDPv5) GetNode(id enode.ID) *enode.Node { if n := t.tab.getNode(id); n != nil { return n } @@ -719,8 +798,13 @@ func (t *UDPv5) getNode(id enode.ID) *enode.Node { return nil } +// Nodes returns the nodes in the routing table. +func (t *UDPv5) Nodes() [][]BucketNode { + return t.tab.Nodes() +} + // handle processes incoming packets according to their message type. -func (t *UDPv5) handle(p v5wire.Packet, fromID enode.ID, fromAddr *net.UDPAddr) { +func (t *UDPv5) handle(p v5wire.Packet, fromID enode.ID, fromAddr netip.AddrPort) { switch p := p.(type) { case *v5wire.Unknown: t.handleUnknown(p, fromID, fromAddr) @@ -730,24 +814,38 @@ func (t *UDPv5) handle(p v5wire.Packet, fromID enode.ID, fromAddr *net.UDPAddr) t.handlePing(p, fromID, fromAddr) case *v5wire.Pong: if t.handleCallResponse(fromID, fromAddr, p) { - t.localNode.UDPEndpointStatement(fromAddr, &net.UDPAddr{IP: p.ToIP, Port: int(p.ToPort)}) + toAddr := netip.AddrPortFrom(netutil.IPToAddr(p.ToIP), p.ToPort) + t.localNode.UDPEndpointStatement(fromAddr, toAddr) } case *v5wire.Findnode: t.handleFindnode(p, fromID, fromAddr) case *v5wire.Nodes: t.handleCallResponse(fromID, fromAddr, p) case *v5wire.TalkRequest: - t.handleTalkRequest(p, fromID, fromAddr) + t.talk.handleRequest(fromID, fromAddr, p) case *v5wire.TalkResponse: t.handleCallResponse(fromID, fromAddr, p) } } // handleUnknown initiates a handshake by responding with WHOAREYOU. -func (t *UDPv5) handleUnknown(p *v5wire.Unknown, fromID enode.ID, fromAddr *net.UDPAddr) { +func (t *UDPv5) handleUnknown(p *v5wire.Unknown, fromID enode.ID, fromAddr netip.AddrPort) { + currentChallenge := t.codec.CurrentChallenge(fromID, fromAddr.String()) + if currentChallenge != nil { + // This case happens when the sender issues multiple concurrent requests. + // Since we only support one in-progress handshake at a time, we need to tell + // them which handshake attempt they need to complete. We tell them to use the + // existing handshake attempt since the response to that one might still be in + // transit. + t.log.Debug("Repeating discv5 handshake challenge", "id", fromID, "addr", fromAddr) + t.sendResponse(fromID, fromAddr, currentChallenge) + return + } + + // Send a fresh challenge. challenge := &v5wire.Whoareyou{Nonce: p.Nonce} crand.Read(challenge.IDNonce[:]) - if n := t.getNode(fromID); n != nil { + if n := t.GetNode(fromID); n != nil { challenge.Node = n challenge.RecordSeq = n.Seq() } @@ -760,17 +858,21 @@ var ( ) // handleWhoareyou resends the active call as a handshake packet. -func (t *UDPv5) handleWhoareyou(p *v5wire.Whoareyou, fromID enode.ID, fromAddr *net.UDPAddr) { +func (t *UDPv5) handleWhoareyou(p *v5wire.Whoareyou, fromID enode.ID, fromAddr netip.AddrPort) { c, err := t.matchWithCall(fromID, p.Nonce) if err != nil { - t.log.Trace("Invalid "+p.Name(), "addr", fromAddr, "err", err) + t.log.Debug("Invalid "+p.Name(), "addr", fromAddr, "err", err) return } - // Resend the call that was answered by WHOAREYOU. - if t.trace { - t.log.Trace("<< "+p.Name(), "id", c.node.ID(), "addr", fromAddr) + if c.node == nil { + // Can't perform handshake because we don't have the ENR. + t.log.Debug("Can't handle "+p.Name(), "addr", fromAddr, "err", "call has no ENR") + c.err <- errors.New("remote wants handshake, but call has no ENR") + return } + // Resend the call that was answered by WHOAREYOU. + t.log.Trace("<< "+p.Name(), "id", c.node.ID(), "addr", fromAddr) c.handshakeCount++ c.challenge = p p.Node = c.node @@ -778,8 +880,6 @@ func (t *UDPv5) handleWhoareyou(p *v5wire.Whoareyou, fromID enode.ID, fromAddr * } // matchWithCall checks whether a handshake attempt matches the active call. -// -//nolint:unparam func (t *UDPv5) matchWithCall(fromID enode.ID, nonce v5wire.Nonce) (*callV5, error) { c := t.activeCallByAuth[nonce] if c == nil { @@ -792,27 +892,36 @@ func (t *UDPv5) matchWithCall(fromID enode.ID, nonce v5wire.Nonce) (*callV5, err } // handlePing sends a PONG response. -func (t *UDPv5) handlePing(p *v5wire.Ping, fromID enode.ID, fromAddr *net.UDPAddr) { - //nolint:errcheck +func (t *UDPv5) handlePing(p *v5wire.Ping, fromID enode.ID, fromAddr netip.AddrPort) { + var remoteIP net.IP + // Handle IPv4 mapped IPv6 addresses in the event the local node is binded + // to an ipv6 interface. + if fromAddr.Addr().Is4() || fromAddr.Addr().Is4In6() { + ip4 := fromAddr.Addr().As4() + remoteIP = ip4[:] + } else { + remoteIP = fromAddr.Addr().AsSlice() + } t.sendResponse(fromID, fromAddr, &v5wire.Pong{ ReqID: p.ReqID, - ToIP: fromAddr.IP, - ToPort: uint16(fromAddr.Port), + ToIP: remoteIP, + ToPort: fromAddr.Port(), ENRSeq: t.localNode.Node().Seq(), }) } // handleFindnode returns nodes to the requester. -func (t *UDPv5) handleFindnode(p *v5wire.Findnode, fromID enode.ID, fromAddr *net.UDPAddr) { - nodes := t.collectTableNodes(fromAddr.IP, p.Distances, findnodeResultLimit) +func (t *UDPv5) handleFindnode(p *v5wire.Findnode, fromID enode.ID, fromAddr netip.AddrPort) { + nodes := t.collectTableNodes(fromAddr.Addr(), p.Distances, findnodeResultLimit) for _, resp := range packNodes(p.ReqID, nodes) { - t.sendResponse(fromID, fromAddr, resp) //nolint:errcheck + t.sendResponse(fromID, fromAddr, resp) } } // collectTableNodes creates a FINDNODE result set for the given distances. -func (t *UDPv5) collectTableNodes(rip net.IP, distances []uint, limit int) []*enode.Node { - nodes := make([]*enode.Node, 0, len(distances)) +func (t *UDPv5) collectTableNodes(rip netip.Addr, distances []uint, limit int) []*enode.Node { + var bn []*enode.Node + var nodes []*enode.Node var processed = make(map[uint]struct{}) for _, dist := range distances { // Reject duplicate / invalid distances. @@ -820,22 +929,13 @@ func (t *UDPv5) collectTableNodes(rip net.IP, distances []uint, limit int) []*en if seen || dist > 256 { continue } - - // Get the nodes. - var bn []*enode.Node - if dist == 0 { - bn = []*enode.Node{t.Self()} - } else if dist <= 256 { - t.tab.mutex.Lock() - bn = unwrapNodes(t.tab.bucketAtDistance(int(dist)).entries) - t.tab.mutex.Unlock() - } processed[dist] = struct{}{} - // Apply some pre-checks to avoid sending invalid nodes. - for _, n := range bn { - // TODO livenessChecks > 1 - if netutil.CheckRelayIP(rip, n.IP()) != nil { + checkLive := !t.tab.cfg.NoFindnodeLivenessCheck + for _, n := range t.tab.appendBucketNodes(dist, bn[:0], checkLive) { + // Apply some pre-checks to avoid sending invalid nodes. + // Note liveness is checked by appendLiveNodes. + if netutil.CheckRelayAddr(rip, n.IPAddr()) != nil { continue } nodes = append(nodes, n) @@ -850,33 +950,31 @@ func (t *UDPv5) collectTableNodes(rip net.IP, distances []uint, limit int) []*en // packNodes creates NODES response packets for the given node list. func packNodes(reqid []byte, nodes []*enode.Node) []*v5wire.Nodes { if len(nodes) == 0 { - return []*v5wire.Nodes{{ReqID: reqid, Total: 1}} + return []*v5wire.Nodes{{ReqID: reqid, RespCount: 1}} } - total := uint8(math.Ceil(float64(len(nodes)) / nodesResponseItemLimit)) + // This limit represents the available space for nodes in output packets. Maximum + // packet size is 1280, and out of this ~80 bytes will be taken up by the packet + // frame. So limiting to 1000 bytes here leaves 200 bytes for other fields of the + // NODES message, which is a lot. + const sizeLimit = 1000 + var resp []*v5wire.Nodes for len(nodes) > 0 { - p := &v5wire.Nodes{ReqID: reqid, Total: total} - items := min(nodesResponseItemLimit, len(nodes)) - for i := 0; i < items; i++ { - p.Nodes = append(p.Nodes, nodes[i].Record()) + p := &v5wire.Nodes{ReqID: reqid} + size := uint64(0) + for len(nodes) > 0 { + r := nodes[0].Record() + if size += r.Size(); size > sizeLimit { + break + } + p.Nodes = append(p.Nodes, r) + nodes = nodes[1:] } - nodes = nodes[items:] resp = append(resp, p) } - return resp -} - -// handleTalkRequest runs the talk request handler of the requested protocol. -func (t *UDPv5) handleTalkRequest(p *v5wire.TalkRequest, fromID enode.ID, fromAddr *net.UDPAddr) { - t.trlock.Lock() - handler := t.trhandlers[p.Protocol] - t.trlock.Unlock() - - var response []byte - if handler != nil { - response = handler(fromID, fromAddr, p.Message) + for _, msg := range resp { + msg.RespCount = uint8(len(resp)) } - resp := &v5wire.TalkResponse{ReqID: p.ReqID, Message: response} - t.sendResponse(fromID, fromAddr, resp) //nolint:errcheck + return resp } diff --git a/p2p/discover/v5_udp_integration_test.go b/p2p/discover/v5_udp_integration_test.go deleted file mode 100644 index e9d550ad12c..00000000000 --- a/p2p/discover/v5_udp_integration_test.go +++ /dev/null @@ -1,121 +0,0 @@ -// Copyright 2024 The Erigon Authors -// This file is part of Erigon. -// -// Erigon is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// Erigon is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with Erigon. If not, see . - -package discover - -import ( - "context" - "net" - "runtime" - "testing" - "time" - - "github.com/erigontech/erigon/common/log/v3" - "github.com/erigontech/erigon/p2p/discover/v5wire" -) - -// This test checks that pending calls are re-sent when a handshake happens. -func TestUDPv5_callResend(t *testing.T) { - if runtime.GOOS != "linux" { - t.Skip("fix me on win please") - } - t.Parallel() - logger := log.New() - test := newUDPV5Test(t, logger) - t.Cleanup(test.close) - - remote := test.getNode(test.remotekey, test.remoteaddr, logger).Node() - done := make(chan error, 2) - go func() { - _, err := test.udp.ping(remote) - done <- err - }() - go func() { - _, err := test.udp.ping(remote) - done <- err - }() - - // Ping answered by WHOAREYOU. - test.waitPacketOut(func(p *v5wire.Ping, addr *net.UDPAddr, nonce v5wire.Nonce) { - test.packetIn(&v5wire.Whoareyou{Nonce: nonce}, logger) - }) - // Ping should be re-sent. - test.waitPacketOut(func(p *v5wire.Ping, addr *net.UDPAddr, _ v5wire.Nonce) { - test.packetIn(&v5wire.Pong{ReqID: p.ReqID}, logger) - }) - // Answer the other ping. - test.waitPacketOut(func(p *v5wire.Ping, addr *net.UDPAddr, _ v5wire.Nonce) { - test.packetIn(&v5wire.Pong{ReqID: p.ReqID}, logger) - }) - if err := <-done; err != nil { - t.Fatalf("unexpected ping error: %v", err) - } - if err := <-done; err != nil { - t.Fatalf("unexpected ping error: %v", err) - } -} - -// This test checks that calls with n replies may take up to n * respTimeout. -func TestUDPv5_callTimeoutReset(t *testing.T) { - if runtime.GOOS != "linux" { - t.Skip("fix me on win please") - } - t.Parallel() - logger := log.New() - - replyTimeout := respTimeoutV5 - // This must be significantly lower than replyTimeout to not get "RPC timeout" error. - singleReplyDelay := replyTimeout / (totalNodesResponseLimit - 1) - if singleReplyDelay*totalNodesResponseLimit < replyTimeout { - t.Fatalf("The total delay of all replies must exceed an individual reply timeout.") - } - if replyTimeout-singleReplyDelay < 50*time.Millisecond { - t.Errorf("50ms is sometimes not enough on a slow CI to process a reply.") - } - - ctx := context.Background() - ctx = contextWithReplyTimeout(ctx, replyTimeout) - - test := newUDPV5TestContext(ctx, t, logger) - t.Cleanup(test.close) - - // Launch the request: - var ( - distance = uint(230) - remote = test.getNode(test.remotekey, test.remoteaddr, logger).Node() - nodes = nodesAtDistance(remote.ID(), int(distance), totalNodesResponseLimit) - done = make(chan error, 1) - ) - go func() { - _, err := test.udp.findnode(remote, []uint{distance}) - done <- err - }() - - // Serve two responses, slowly. - test.waitPacketOut(func(p *v5wire.Findnode, addr *net.UDPAddr, _ v5wire.Nonce) { - for i := 0; i < totalNodesResponseLimit; i++ { - time.Sleep(singleReplyDelay) - test.packetIn(&v5wire.Nodes{ - ReqID: p.ReqID, - Total: totalNodesResponseLimit, - Nodes: nodesToRecords(nodes[i : i+1]), - }, logger) - } - }) - if err := <-done; err != nil { - t.Fatalf("unexpected error: %q", err) - } -} diff --git a/p2p/discover/v5_udp_test.go b/p2p/discover/v5_udp_test.go index 863920e9511..8fea29df6dc 100644 --- a/p2p/discover/v5_udp_test.go +++ b/p2p/discover/v5_udp_test.go @@ -1,60 +1,87 @@ -// Copyright 2019 The go-ethereum Authors -// (original work) -// Copyright 2024 The Erigon Authors -// (modifications) -// This file is part of Erigon. +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. // -// Erigon is free software: you can redistribute it and/or modify +// The go-ethereum library is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // -// Erigon is distributed in the hope that it will be useful, +// The go-ethereum library is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Lesser General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License -// along with Erigon. If not, see . +// along with the go-ethereum library. If not, see . package discover import ( "bytes" - "context" "crypto/ecdsa" "encoding/binary" - "errors" "fmt" + "math/rand" "net" + "net/netip" "reflect" - "runtime" + "slices" "testing" "time" "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/common/testlog" "github.com/erigontech/erigon/execution/rlp" + "github.com/erigontech/erigon/p2p/discover/v4wire" "github.com/erigontech/erigon/p2p/discover/v5wire" "github.com/erigontech/erigon/p2p/enode" "github.com/erigontech/erigon/p2p/enr" + "github.com/stretchr/testify/require" ) -func startLocalhostV5(t *testing.T, cfg Config, logger log.Logger) *UDPv5 { - cfg.PrivateKey = newkey() - tmpDir := t.TempDir() - db, err := enode.OpenDB(context.Background(), "", tmpDir, logger) - if err != nil { - panic(err) +// Real sockets, real crypto: this test checks end-to-end connectivity for UDPv5. +func TestUDPv5_lookupE2E(t *testing.T) { + t.Parallel() + + const N = 5 + var nodes []*UDPv5 + for i := 0; i < N; i++ { + var cfg Config + if len(nodes) > 0 { + bn := nodes[0].Self() + cfg.Bootnodes = []*enode.Node{bn} + } + node := startLocalhostV5(t, cfg) + nodes = append(nodes, node) + defer node.Close() + } + last := nodes[N-1] + target := nodes[rand.Intn(N-2)].Self() + + // It is expected that all nodes can be found. + expectedResult := make([]*enode.Node, len(nodes)) + for i := range nodes { + expectedResult[i] = nodes[i].Self() } - t.Cleanup(db.Close) - ln := enode.NewLocalNode(db, cfg.PrivateKey, logger) + slices.SortFunc(expectedResult, func(a, b *enode.Node) int { + return enode.DistCmp(target.ID(), a.ID(), b.ID()) + }) + + // Do the lookup. + results := last.Lookup(target.ID()) + if err := checkNodesEqual(results, expectedResult); err != nil { + t.Fatalf("lookup returned wrong results: %v", err) + } +} + +func startLocalhostV5(t *testing.T, cfg Config) *UDPv5 { + cfg.PrivateKey = newkey() + db, _ := enode.OpenDB("") + ln := enode.NewLocalNode(db, cfg.PrivateKey) // Prefix logs with node ID. lprefix := fmt.Sprintf("(%s)", ln.ID().TerminalString()) - lfmt := log.TerminalFormat() - cfg.Log = testlog.Logger(t, log.LvlError) - cfg.Log.SetHandler(testLogHandler{lprefix: lprefix, lfmt: lfmt, t: t}) + cfg.Log = testlog.Logger(t, log.LvlTrace).New("node-id", lprefix) // Listen. socket, err := net.ListenUDP("udp4", &net.UDPAddr{IP: net.IP{127, 0, 0, 1}}) @@ -63,10 +90,8 @@ func startLocalhostV5(t *testing.T, cfg Config, logger log.Logger) *UDPv5 { } realaddr := socket.LocalAddr().(*net.UDPAddr) ln.SetStaticIP(realaddr.IP) - ln.SetFallbackUDP(realaddr.Port) - ctx := context.Background() - ctx = disableLookupSlowdown(ctx) - udp, err := ListenV5(ctx, "test", socket, ln, cfg) + ln.Set(enr.UDP(realaddr.Port)) + udp, err := ListenV5(socket, ln, cfg) if err != nil { t.Fatal(err) } @@ -75,16 +100,12 @@ func startLocalhostV5(t *testing.T, cfg Config, logger log.Logger) *UDPv5 { // This test checks that incoming PING calls are handled correctly. func TestUDPv5_pingHandling(t *testing.T) { - if runtime.GOOS != "linux" { - t.Skip("fix me on win please") - } t.Parallel() - logger := log.New() - test := newUDPV5Test(t, logger) - t.Cleanup(test.close) + test := newUDPV5Test(t) + defer test.close() - test.packetIn(&v5wire.Ping{ReqID: []byte("foo")}, logger) - test.waitPacketOut(func(p *v5wire.Pong, addr *net.UDPAddr, _ v5wire.Nonce) { + test.packetIn(&v5wire.Ping{ReqID: []byte("foo")}) + test.waitPacketOut(func(p *v5wire.Pong, addr netip.AddrPort, _ v5wire.Nonce) { if !bytes.Equal(p.ReqID, []byte("foo")) { t.Error("wrong request ID in response:", p.ReqID) } @@ -96,13 +117,9 @@ func TestUDPv5_pingHandling(t *testing.T) { // This test checks that incoming 'unknown' packets trigger the handshake. func TestUDPv5_unknownPacket(t *testing.T) { - if runtime.GOOS != "linux" { - t.Skip("fix me on win please") - } t.Parallel() - logger := log.New() - test := newUDPV5Test(t, logger) - t.Cleanup(test.close) + test := newUDPV5Test(t) + defer test.close() nonce := v5wire.Nonce{1, 2, 3} check := func(p *v5wire.Whoareyou, wantSeq uint64) { @@ -119,93 +136,142 @@ func TestUDPv5_unknownPacket(t *testing.T) { } // Unknown packet from unknown node. - test.packetIn(&v5wire.Unknown{Nonce: nonce}, logger) - test.waitPacketOut(func(p *v5wire.Whoareyou, addr *net.UDPAddr, _ v5wire.Nonce) { + test.packetIn(&v5wire.Unknown{Nonce: nonce}) + test.waitPacketOut(func(p *v5wire.Whoareyou, addr netip.AddrPort, _ v5wire.Nonce) { check(p, 0) }) +} + +func TestUDPv5_unknownPacketKnownNode(t *testing.T) { + t.Parallel() + test := newUDPV5Test(t) + defer test.close() + + nonce := v5wire.Nonce{1, 2, 3} + check := func(p *v5wire.Whoareyou, wantSeq uint64) { + t.Helper() + if p.Nonce != nonce { + t.Error("wrong nonce in WHOAREYOU:", p.Nonce, nonce) + } + if p.IDNonce == ([16]byte{}) { + t.Error("all zero ID nonce") + } + if p.RecordSeq != wantSeq { + t.Errorf("wrong record seq %d in WHOAREYOU, want %d", p.RecordSeq, wantSeq) + } + } // Make node known. - n := test.getNode(test.remotekey, test.remoteaddr, logger).Node() - test.table.addSeenNode(wrapNode(n)) + n := test.getNode(test.remotekey, test.remoteaddr).Node() + test.table.addFoundNode(n, false) - test.packetIn(&v5wire.Unknown{Nonce: nonce}, logger) - test.waitPacketOut(func(p *v5wire.Whoareyou, addr *net.UDPAddr, _ v5wire.Nonce) { + test.packetIn(&v5wire.Unknown{Nonce: nonce}) + test.waitPacketOut(func(p *v5wire.Whoareyou, addr netip.AddrPort, _ v5wire.Nonce) { check(p, n.Seq()) }) } +// This test checks that, when multiple 'unknown' packets are received during a handshake, +// the node sticks to the first handshake attempt. +func TestUDPv5_handshakeRepeatChallenge(t *testing.T) { + t.Parallel() + test := newUDPV5Test(t) + defer test.close() + + nonce1 := v5wire.Nonce{1} + nonce2 := v5wire.Nonce{2} + nonce3 := v5wire.Nonce{3} + var firstAuthTag *v5wire.Nonce + check := func(p *v5wire.Whoareyou, authTag, wantNonce v5wire.Nonce) { + t.Helper() + if p.Nonce != wantNonce { + t.Error("wrong nonce in WHOAREYOU:", p.Nonce, "want:", wantNonce) + } + if firstAuthTag == nil { + firstAuthTag = &authTag + } else if authTag != *firstAuthTag { + t.Error("wrong auth tag in WHOAREYOU header:", authTag, "want:", *firstAuthTag) + } + } + + // Unknown packet from unknown node. + test.packetIn(&v5wire.Unknown{Nonce: nonce1}) + test.waitPacketOut(func(p *v5wire.Whoareyou, addr netip.AddrPort, authTag v5wire.Nonce) { + check(p, authTag, nonce1) + }) + + // Second unknown packet. Here we expect the response to reference the + // first unknown packet. + test.packetIn(&v5wire.Unknown{Nonce: nonce2}) + test.waitPacketOut(func(p *v5wire.Whoareyou, addr netip.AddrPort, authTag v5wire.Nonce) { + check(p, authTag, nonce1) + }) + // Third unknown packet. This should still return the first nonce. + test.packetIn(&v5wire.Unknown{Nonce: nonce3}) + test.waitPacketOut(func(p *v5wire.Whoareyou, addr netip.AddrPort, authTag v5wire.Nonce) { + check(p, authTag, nonce1) + }) +} + // This test checks that incoming FINDNODE calls are handled correctly. func TestUDPv5_findnodeHandling(t *testing.T) { - if runtime.GOOS != "linux" { - t.Skip("fix me on win please") - } t.Parallel() - logger := log.New() - test := newUDPV5Test(t, logger) - t.Cleanup(test.close) + test := newUDPV5Test(t) + defer test.close() // Create test nodes and insert them into the table. - nodes253 := nodesAtDistance(test.table.self().ID(), 253, 10) + nodes253 := nodesAtDistance(test.table.self().ID(), 253, 16) nodes249 := nodesAtDistance(test.table.self().ID(), 249, 4) nodes248 := nodesAtDistance(test.table.self().ID(), 248, 10) - fillTable(test.table, wrapNodes(nodes253)) - fillTable(test.table, wrapNodes(nodes249)) - fillTable(test.table, wrapNodes(nodes248)) + fillTable(test.table, nodes253, true) + fillTable(test.table, nodes249, true) + fillTable(test.table, nodes248, true) // Requesting with distance zero should return the node's own record. - test.packetIn(&v5wire.Findnode{ReqID: []byte{0}, Distances: []uint{0}}, logger) + test.packetIn(&v5wire.Findnode{ReqID: []byte{0}, Distances: []uint{0}}) test.expectNodes([]byte{0}, 1, []*enode.Node{test.udp.Self()}) // Requesting with distance > 256 shouldn't crash. - test.packetIn(&v5wire.Findnode{ReqID: []byte{1}, Distances: []uint{4234098}}, logger) + test.packetIn(&v5wire.Findnode{ReqID: []byte{1}, Distances: []uint{4234098}}) test.expectNodes([]byte{1}, 1, nil) // Requesting with empty distance list shouldn't crash either. - test.packetIn(&v5wire.Findnode{ReqID: []byte{2}, Distances: []uint{}}, logger) + test.packetIn(&v5wire.Findnode{ReqID: []byte{2}, Distances: []uint{}}) test.expectNodes([]byte{2}, 1, nil) // This request gets no nodes because the corresponding bucket is empty. - test.packetIn(&v5wire.Findnode{ReqID: []byte{3}, Distances: []uint{254}}, logger) + test.packetIn(&v5wire.Findnode{ReqID: []byte{3}, Distances: []uint{254}}) test.expectNodes([]byte{3}, 1, nil) // This request gets all the distance-253 nodes. - test.packetIn(&v5wire.Findnode{ReqID: []byte{4}, Distances: []uint{253}}, logger) - test.expectNodes([]byte{4}, 4, nodes253) + test.packetIn(&v5wire.Findnode{ReqID: []byte{4}, Distances: []uint{253}}) + test.expectNodes([]byte{4}, 2, nodes253) // This request gets all the distance-249 nodes and some more at 248 because // the bucket at 249 is not full. - test.packetIn(&v5wire.Findnode{ReqID: []byte{5}, Distances: []uint{249, 248}}, logger) - nodes := make([]*enode.Node, 0, len(nodes249)+len(nodes248[:10])) + test.packetIn(&v5wire.Findnode{ReqID: []byte{5}, Distances: []uint{249, 248}}) + var nodes []*enode.Node nodes = append(nodes, nodes249...) nodes = append(nodes, nodes248[:10]...) - test.expectNodes([]byte{5}, 5, nodes) + test.expectNodes([]byte{5}, 1, nodes) } func (test *udpV5Test) expectNodes(wantReqID []byte, wantTotal uint8, wantNodes []*enode.Node) { - nodeSet := make(map[enode.ID]*enr.Record) + nodeSet := make(map[enode.ID]*enr.Record, len(wantNodes)) for _, n := range wantNodes { nodeSet[n.ID()] = n.Record() } for { - test.waitPacketOut(func(p *v5wire.Nodes, addr *net.UDPAddr, _ v5wire.Nonce) { + test.waitPacketOut(func(p *v5wire.Nodes, addr netip.AddrPort, _ v5wire.Nonce) { if !bytes.Equal(p.ReqID, wantReqID) { test.t.Fatalf("wrong request ID %v in response, want %v", p.ReqID, wantReqID) } - if len(p.Nodes) > 3 { - test.t.Fatalf("too many nodes in response") - } - if p.Total != wantTotal { - test.t.Fatalf("wrong total response count %d, want %d", p.Total, wantTotal) - } - if !bytes.Equal(p.ReqID, wantReqID) { - test.t.Fatalf("wrong request ID in response: %v", p.ReqID) + if p.RespCount != wantTotal { + test.t.Fatalf("wrong total response count %d, want %d", p.RespCount, wantTotal) } for _, record := range p.Nodes { - n, err := enode.New(enode.ValidSchemesForTesting, record) - if err != nil { - panic(err) - } + n, _ := enode.New(enode.ValidSchemesForTesting, record) want := nodeSet[n.ID()] if want == nil { test.t.Fatalf("unexpected node in response: %v", n) @@ -224,15 +290,11 @@ func (test *udpV5Test) expectNodes(wantReqID []byte, wantTotal uint8, wantNodes // This test checks that outgoing PING calls work. func TestUDPv5_pingCall(t *testing.T) { - if runtime.GOOS != "linux" { - t.Skip("fix me on win please") - } t.Parallel() - logger := log.New() - test := newUDPV5Test(t, logger) - t.Cleanup(test.close) + test := newUDPV5Test(t) + defer test.close() - remote := test.getNode(test.remotekey, test.remoteaddr, logger).Node() + remote := test.getNode(test.remotekey, test.remoteaddr).Node() done := make(chan error, 1) // This ping times out. @@ -240,7 +302,7 @@ func TestUDPv5_pingCall(t *testing.T) { _, err := test.udp.ping(remote) done <- err }() - test.waitPacketOut(func(p *v5wire.Ping, addr *net.UDPAddr, _ v5wire.Nonce) {}) + test.waitPacketOut(func(p *v5wire.Ping, addr netip.AddrPort, _ v5wire.Nonce) {}) if err := <-done; err != errTimeout { t.Fatalf("want errTimeout, got %q", err) } @@ -250,8 +312,8 @@ func TestUDPv5_pingCall(t *testing.T) { _, err := test.udp.ping(remote) done <- err }() - test.waitPacketOut(func(p *v5wire.Ping, addr *net.UDPAddr, _ v5wire.Nonce) { - test.packetInFrom(test.remotekey, test.remoteaddr, &v5wire.Pong{ReqID: p.ReqID}, logger) + test.waitPacketOut(func(p *v5wire.Ping, addr netip.AddrPort, _ v5wire.Nonce) { + test.packetInFrom(test.remotekey, test.remoteaddr, &v5wire.Pong{ReqID: p.ReqID}) }) if err := <-done; err != nil { t.Fatal(err) @@ -262,9 +324,9 @@ func TestUDPv5_pingCall(t *testing.T) { _, err := test.udp.ping(remote) done <- err }() - test.waitPacketOut(func(p *v5wire.Ping, addr *net.UDPAddr, _ v5wire.Nonce) { - wrongAddr := &net.UDPAddr{IP: net.IP{33, 44, 55, 22}, Port: 10101} - test.packetInFrom(test.remotekey, wrongAddr, &v5wire.Pong{ReqID: p.ReqID}, logger) + test.waitPacketOut(func(p *v5wire.Ping, addr netip.AddrPort, _ v5wire.Nonce) { + wrongAddr := netip.MustParseAddrPort("33.44.55.22:10101") + test.packetInFrom(test.remotekey, wrongAddr, &v5wire.Pong{ReqID: p.ReqID}) }) if err := <-done; err != errTimeout { t.Fatalf("want errTimeout for reply from wrong IP, got %q", err) @@ -274,43 +336,39 @@ func TestUDPv5_pingCall(t *testing.T) { // This test checks that outgoing FINDNODE calls work and multiple NODES // replies are aggregated. func TestUDPv5_findnodeCall(t *testing.T) { - if runtime.GOOS != "linux" { - t.Skip("fix me on win please") - } t.Parallel() - logger := log.New() - test := newUDPV5Test(t, logger) - t.Cleanup(test.close) + test := newUDPV5Test(t) + defer test.close() // Launch the request: var ( distances = []uint{230} - remote = test.getNode(test.remotekey, test.remoteaddr, logger).Node() + remote = test.getNode(test.remotekey, test.remoteaddr).Node() nodes = nodesAtDistance(remote.ID(), int(distances[0]), 8) done = make(chan error, 1) response []*enode.Node ) go func() { var err error - response, err = test.udp.findnode(remote, distances) + response, err = test.udp.Findnode(remote, distances) done <- err }() // Serve the responses: - test.waitPacketOut(func(p *v5wire.Findnode, addr *net.UDPAddr, _ v5wire.Nonce) { + test.waitPacketOut(func(p *v5wire.Findnode, addr netip.AddrPort, _ v5wire.Nonce) { if !reflect.DeepEqual(p.Distances, distances) { t.Fatalf("wrong distances in request: %v", p.Distances) } test.packetIn(&v5wire.Nodes{ - ReqID: p.ReqID, - Total: 2, - Nodes: nodesToRecords(nodes[:4]), - }, logger) + ReqID: p.ReqID, + RespCount: 2, + Nodes: nodesToRecords(nodes[:4]), + }) test.packetIn(&v5wire.Nodes{ - ReqID: p.ReqID, - Total: 2, - Nodes: nodesToRecords(nodes[4:]), - }, logger) + ReqID: p.ReqID, + RespCount: 2, + Nodes: nodesToRecords(nodes[4:]), + }) }) // Check results: @@ -320,22 +378,139 @@ func TestUDPv5_findnodeCall(t *testing.T) { if !reflect.DeepEqual(response, nodes) { t.Fatalf("wrong nodes in response") } +} + +// BadIdentityScheme mocks an identity scheme not supported by the test node. +type BadIdentityScheme struct{} - // TODO: check invalid IPs - // TODO: check invalid/unsigned record +func (s BadIdentityScheme) Verify(r *enr.Record, sig []byte) error { return nil } +func (s BadIdentityScheme) NodeAddr(r *enr.Record) []byte { + var id enode.ID + r.Load(enr.WithEntry("badaddr", &id)) + return id[:] +} + +// This test covers invalid NODES responses for the FINDNODE call in a single table-driven test. +func TestUDPv5_findnodeCall_InvalidNodes(t *testing.T) { + t.Parallel() + test := newUDPV5Test(t) + defer test.close() + + for i, tt := range []struct { + name string + ip enr.Entry + port enr.Entry + sign func(r *enr.Record, id enode.ID) *enode.Node + }{ + { + name: "invalid ip (unspecified 0.0.0.0)", + ip: enr.IP(net.IPv4zero), + }, + { + name: "invalid udp port (<=1024)", + port: enr.UDP(1024), + }, + { + name: "invalid record, no signature", + sign: func(r *enr.Record, id enode.ID) *enode.Node { + r.Set(enr.ID("bad")) + r.Set(enr.WithEntry("badaddr", id)) + r.SetSig(BadIdentityScheme{}, []byte{}) + n, _ := enode.New(BadIdentityScheme{}, r) + return n + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + // Build ENR node for test. + var ( + distance = 230 + remote = test.getNode(test.remotekey, test.remoteaddr).Node() + id = idAtDistance(remote.ID(), distance) + r enr.Record + ) + r.Set(enr.IP(intIP(i))) + if tt.ip != nil { + r.Set(tt.ip) + } + r.Set(enr.UDP(30303)) + if tt.port != nil { + r.Set(tt.port) + } + r = *enode.SignNull(&r, id).Record() + if tt.sign != nil { + r = *tt.sign(&r, id).Record() + } + + // Launch findnode request. + var ( + done = make(chan error, 1) + got []*enode.Node + ) + go func() { + var err error + got, err = test.udp.Findnode(remote, []uint{uint(distance)}) + done <- err + }() + + // Handle request. + test.waitPacketOut(func(p *v5wire.Findnode, _ netip.AddrPort, _ v5wire.Nonce) { + test.packetIn(&v5wire.Nodes{ReqID: p.ReqID, RespCount: 1, Nodes: []*enr.Record{&r}}) + }) + if err := <-done; err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(got) != 0 { + t.Fatalf("expected 0 nodes, got %d", len(got)) + } + }) + } +} + +// This test checks that pending calls are re-sent when a handshake happens. +func TestUDPv5_callResend(t *testing.T) { + t.Parallel() + test := newUDPV5Test(t) + defer test.close() + + remote := test.getNode(test.remotekey, test.remoteaddr).Node() + done := make(chan error, 2) + go func() { + _, err := test.udp.ping(remote) + done <- err + }() + go func() { + _, err := test.udp.ping(remote) + done <- err + }() + + // Ping answered by WHOAREYOU. + test.waitPacketOut(func(p *v5wire.Ping, addr netip.AddrPort, nonce v5wire.Nonce) { + test.packetIn(&v5wire.Whoareyou{Nonce: nonce}) + }) + // Ping should be re-sent. + test.waitPacketOut(func(p *v5wire.Ping, addr netip.AddrPort, _ v5wire.Nonce) { + test.packetIn(&v5wire.Pong{ReqID: p.ReqID}) + }) + // Answer the other ping. + test.waitPacketOut(func(p *v5wire.Ping, addr netip.AddrPort, _ v5wire.Nonce) { + test.packetIn(&v5wire.Pong{ReqID: p.ReqID}) + }) + if err := <-done; err != nil { + t.Fatalf("unexpected ping error: %v", err) + } + if err := <-done; err != nil { + t.Fatalf("unexpected ping error: %v", err) + } } // This test ensures we don't allow multiple rounds of WHOAREYOU for a single call. func TestUDPv5_multipleHandshakeRounds(t *testing.T) { - if runtime.GOOS != "linux" { - t.Skip("fix me on win please") - } t.Parallel() - logger := log.New() - test := newUDPV5Test(t, logger) - t.Cleanup(test.close) + test := newUDPV5Test(t) + defer test.close() - remote := test.getNode(test.remotekey, test.remoteaddr, logger).Node() + remote := test.getNode(test.remotekey, test.remoteaddr).Node() done := make(chan error, 1) go func() { _, err := test.udp.ping(remote) @@ -343,30 +518,65 @@ func TestUDPv5_multipleHandshakeRounds(t *testing.T) { }() // Ping answered by WHOAREYOU. - test.waitPacketOut(func(p *v5wire.Ping, addr *net.UDPAddr, nonce v5wire.Nonce) { - test.packetIn(&v5wire.Whoareyou{Nonce: nonce}, logger) + test.waitPacketOut(func(p *v5wire.Ping, addr netip.AddrPort, nonce v5wire.Nonce) { + test.packetIn(&v5wire.Whoareyou{Nonce: nonce}) }) // Ping answered by WHOAREYOU again. - test.waitPacketOut(func(p *v5wire.Ping, addr *net.UDPAddr, nonce v5wire.Nonce) { - test.packetIn(&v5wire.Whoareyou{Nonce: nonce}, logger) + test.waitPacketOut(func(p *v5wire.Ping, addr netip.AddrPort, nonce v5wire.Nonce) { + test.packetIn(&v5wire.Whoareyou{Nonce: nonce}) }) if err := <-done; err != errTimeout { t.Fatalf("unexpected ping error: %q", err) } } +// This test checks that calls with n replies may take up to n * respTimeout. +func TestUDPv5_callTimeoutReset(t *testing.T) { + t.Parallel() + test := newUDPV5Test(t) + defer test.close() + + // Launch the request: + var ( + distance = uint(230) + remote = test.getNode(test.remotekey, test.remoteaddr).Node() + nodes = nodesAtDistance(remote.ID(), int(distance), 8) + done = make(chan error, 1) + ) + go func() { + _, err := test.udp.Findnode(remote, []uint{distance}) + done <- err + }() + + // Serve two responses, slowly. + test.waitPacketOut(func(p *v5wire.Findnode, addr netip.AddrPort, _ v5wire.Nonce) { + time.Sleep(respTimeout - 50*time.Millisecond) + test.packetIn(&v5wire.Nodes{ + ReqID: p.ReqID, + RespCount: 2, + Nodes: nodesToRecords(nodes[:4]), + }) + + time.Sleep(respTimeout - 50*time.Millisecond) + test.packetIn(&v5wire.Nodes{ + ReqID: p.ReqID, + RespCount: 2, + Nodes: nodesToRecords(nodes[4:]), + }) + }) + if err := <-done; err != nil { + t.Fatalf("unexpected error: %q", err) + } +} + // This test checks that TALKREQ calls the registered handler function. func TestUDPv5_talkHandling(t *testing.T) { - if runtime.GOOS != "linux" { - t.Skip("fix me on win please") - } t.Parallel() - logger := log.New() - test := newUDPV5Test(t, logger) - t.Cleanup(test.close) + test := newUDPV5Test(t) + defer test.close() var recvMessage []byte - test.udp.RegisterTalkHandler("test", func(id enode.ID, addr *net.UDPAddr, message []byte) []byte { + test.udp.RegisterTalkHandler("test", func(n *enode.Node, addr *net.UDPAddr, message []byte) []byte { recvMessage = message return []byte("test response") }) @@ -376,15 +586,15 @@ func TestUDPv5_talkHandling(t *testing.T) { ReqID: []byte("foo"), Protocol: "test", Message: []byte("test request"), - }, logger) - test.waitPacketOut(func(p *v5wire.TalkResponse, addr *net.UDPAddr, _ v5wire.Nonce) { + }) + test.waitPacketOut(func(p *v5wire.TalkResponse, addr netip.AddrPort, _ v5wire.Nonce) { if !bytes.Equal(p.ReqID, []byte("foo")) { t.Error("wrong request ID in response:", p.ReqID) } if string(p.Message) != "test response" { t.Errorf("wrong talk response message: %q", p.Message) } - if string(recvMessage) != "test request" { //nolint:goconst + if string(recvMessage) != "test request" { t.Errorf("wrong message received in handler: %q", recvMessage) } }) @@ -395,8 +605,8 @@ func TestUDPv5_talkHandling(t *testing.T) { ReqID: []byte("2"), Protocol: "wrong", Message: []byte("test request"), - }, logger) - test.waitPacketOut(func(p *v5wire.TalkResponse, addr *net.UDPAddr, _ v5wire.Nonce) { + }) + test.waitPacketOut(func(p *v5wire.TalkResponse, addr netip.AddrPort, _ v5wire.Nonce) { if !bytes.Equal(p.ReqID, []byte("2")) { t.Error("wrong request ID in response:", p.ReqID) } @@ -411,15 +621,11 @@ func TestUDPv5_talkHandling(t *testing.T) { // This test checks that outgoing TALKREQ calls work. func TestUDPv5_talkRequest(t *testing.T) { - if runtime.GOOS != "linux" { - t.Skip("fix me on win please") - } t.Parallel() - logger := log.New() - test := newUDPV5Test(t, logger) - t.Cleanup(test.close) + test := newUDPV5Test(t) + defer test.close() - remote := test.getNode(test.remotekey, test.remoteaddr, logger).Node() + remote := test.getNode(test.remotekey, test.remoteaddr).Node() done := make(chan error, 1) // This request times out. @@ -427,8 +633,8 @@ func TestUDPv5_talkRequest(t *testing.T) { _, err := test.udp.TalkRequest(remote, "test", []byte("test request")) done <- err }() - test.waitPacketOut(func(p *v5wire.TalkRequest, addr *net.UDPAddr, _ v5wire.Nonce) {}) - if err := <-done; !errors.Is(err, errTimeout) { + test.waitPacketOut(func(p *v5wire.TalkRequest, addr netip.AddrPort, _ v5wire.Nonce) {}) + if err := <-done; err != errTimeout { t.Fatalf("want errTimeout, got %q", err) } @@ -437,7 +643,7 @@ func TestUDPv5_talkRequest(t *testing.T) { _, err := test.udp.TalkRequest(remote, "test", []byte("test request")) done <- err }() - test.waitPacketOut(func(p *v5wire.TalkRequest, addr *net.UDPAddr, _ v5wire.Nonce) { + test.waitPacketOut(func(p *v5wire.TalkRequest, addr netip.AddrPort, _ v5wire.Nonce) { if p.Protocol != "test" { t.Errorf("wrong protocol ID in talk request: %q", p.Protocol) } @@ -447,22 +653,132 @@ func TestUDPv5_talkRequest(t *testing.T) { test.packetInFrom(test.remotekey, test.remoteaddr, &v5wire.TalkResponse{ ReqID: p.ReqID, Message: []byte("test response"), - }, logger) + }) }) if err := <-done; err != nil { t.Fatal(err) } + + // Also check requesting without ENR. + go func() { + _, err := test.udp.TalkRequestToID(remote.ID(), test.remoteaddr, "test", []byte("test request 2")) + done <- err + }() + test.waitPacketOut(func(p *v5wire.TalkRequest, addr netip.AddrPort, _ v5wire.Nonce) { + if p.Protocol != "test" { + t.Errorf("wrong protocol ID in talk request: %q", p.Protocol) + } + if string(p.Message) != "test request 2" { + t.Errorf("wrong message talk request: %q", p.Message) + } + test.packetInFrom(test.remotekey, test.remoteaddr, &v5wire.TalkResponse{ + ReqID: p.ReqID, + Message: []byte("test response 2"), + }) + }) + if err := <-done; err != nil { + t.Fatal(err) + } +} + +// This test checks that lookupDistances works. +func TestUDPv5_lookupDistances(t *testing.T) { + test := newUDPV5Test(t) + lnID := test.table.self().ID() + + t.Run("target distance of 1", func(t *testing.T) { + node := nodeAtDistance(lnID, 1, intIP(0)) + dists := lookupDistances(lnID, node.ID()) + require.Equal(t, []uint{1, 2, 3}, dists) + }) + + t.Run("target distance of 2", func(t *testing.T) { + node := nodeAtDistance(lnID, 2, intIP(0)) + dists := lookupDistances(lnID, node.ID()) + require.Equal(t, []uint{2, 3, 1}, dists) + }) + + t.Run("target distance of 128", func(t *testing.T) { + node := nodeAtDistance(lnID, 128, intIP(0)) + dists := lookupDistances(lnID, node.ID()) + require.Equal(t, []uint{128, 129, 127}, dists) + }) + + t.Run("target distance of 255", func(t *testing.T) { + node := nodeAtDistance(lnID, 255, intIP(0)) + dists := lookupDistances(lnID, node.ID()) + require.Equal(t, []uint{255, 256, 254}, dists) + }) + + t.Run("target distance of 256", func(t *testing.T) { + node := nodeAtDistance(lnID, 256, intIP(0)) + dists := lookupDistances(lnID, node.ID()) + require.Equal(t, []uint{256, 255, 254}, dists) + }) +} + +// This test checks that lookup works. +func TestUDPv5_lookup(t *testing.T) { + t.Parallel() + test := newUDPV5Test(t) + + // Lookup on empty table returns no nodes. + if results := test.udp.Lookup(lookupTestnet.target.ID()); len(results) > 0 { + t.Fatalf("lookup on empty table returned %d results: %#v", len(results), results) + } + + // Ensure the tester knows all nodes in lookupTestnet by IP. + for d, nn := range lookupTestnet.dists { + for i, key := range nn { + n := lookupTestnet.node(d, i) + addr, _ := n.UDPEndpoint() + test.getNode(key, addr) + } + } + + // Seed table with initial node. + initialNode := lookupTestnet.node(256, 0) + fillTable(test.table, []*enode.Node{initialNode}, true) + + // Start the lookup. + resultC := make(chan []*enode.Node, 1) + go func() { + resultC <- test.udp.Lookup(lookupTestnet.target.ID()) + test.close() + }() + + // Answer lookup packets. + asked := make(map[enode.ID]bool) + for done := false; !done; { + done = test.waitPacketOut(func(p v5wire.Packet, to netip.AddrPort, _ v5wire.Nonce) { + recipient, key := lookupTestnet.nodeByAddr(to) + switch p := p.(type) { + case *v5wire.Ping: + test.packetInFrom(key, to, &v5wire.Pong{ReqID: p.ReqID}) + case *v5wire.Findnode: + if asked[recipient.ID()] { + t.Error("Asked node", recipient.ID(), "twice") + } + asked[recipient.ID()] = true + nodes := lookupTestnet.neighborsAtDistances(recipient, p.Distances, 16) + t.Logf("Got FINDNODE for %v, returning %d nodes", p.Distances, len(nodes)) + for _, resp := range packNodes(p.ReqID, nodes) { + test.packetInFrom(key, to, resp) + } + } + }) + } + + // Verify result nodes. + results := <-resultC + checkLookupResults(t, lookupTestnet, results) } // This test checks the local node can be utilised to set key-values. func TestUDPv5_LocalNode(t *testing.T) { - if runtime.GOOS != "linux" { - t.Skip("fix me on win please") - } t.Parallel() - logger := log.New() var cfg Config - node := startLocalhostV5(t, cfg, logger) + node := startLocalhostV5(t, cfg) defer node.Close() localNd := node.LocalNode() @@ -480,6 +796,35 @@ func TestUDPv5_LocalNode(t *testing.T) { } } +func TestUDPv5_PingWithIPV4MappedAddress(t *testing.T) { + t.Parallel() + test := newUDPV5Test(t) + defer test.close() + + rawIP := netip.AddrFrom4([4]byte{0xFF, 0x12, 0x33, 0xE5}) + test.remoteaddr = netip.AddrPortFrom(netip.AddrFrom16(rawIP.As16()), 0) + remote := test.getNode(test.remotekey, test.remoteaddr).Node() + done := make(chan struct{}, 1) + + // This handler will truncate the ipv4-mapped in ipv6 address. + go func() { + test.udp.handlePing(&v5wire.Ping{ENRSeq: 1}, remote.ID(), test.remoteaddr) + done <- struct{}{} + }() + test.waitPacketOut(func(p *v5wire.Pong, addr netip.AddrPort, _ v5wire.Nonce) { + if len(p.ToIP) == net.IPv6len { + t.Error("Received untruncated ip address") + } + if len(p.ToIP) != net.IPv4len { + t.Errorf("Received ip address with incorrect length: %d", len(p.ToIP)) + } + if !p.ToIP.Equal(rawIP.AsSlice()) { + t.Errorf("Received incorrect ip address: wanted %s but received %s", rawIP.String(), p.ToIP.String()) + } + }) + <-done +} + // udpV5Test is the framework for all tests above. // It runs the UDPv5 transport on a virtual socket and allows testing outgoing packets. type udpV5Test struct { @@ -489,9 +834,9 @@ type udpV5Test struct { db *enode.DB udp *UDPv5 localkey, remotekey *ecdsa.PrivateKey - remoteaddr *net.UDPAddr + remoteaddr netip.AddrPort nodesByID map[enode.ID]*enode.LocalNode - nodesByIP map[string]*enode.LocalNode + nodesByIP map[netip.Addr]*enode.LocalNode } // testCodec is the packet encoding used by protocol tests. This codec does not perform encryption. @@ -499,6 +844,8 @@ type testCodec struct { test *udpV5Test id enode.ID ctr uint64 + + sentChallenges map[enode.ID]*v5wire.Whoareyou } type testCodecFrame struct { @@ -509,18 +856,37 @@ type testCodecFrame struct { } func (c *testCodec) Encode(toID enode.ID, addr string, p v5wire.Packet, _ *v5wire.Whoareyou) ([]byte, v5wire.Nonce, error) { + // To match the behavior of v5wire.Codec, we return the cached encoding of + // WHOAREYOU challenges. + if wp, ok := p.(*v5wire.Whoareyou); ok && len(wp.Encoded) > 0 { + return wp.Encoded, wp.Nonce, nil + } + c.ctr++ var authTag v5wire.Nonce binary.BigEndian.PutUint64(authTag[:], c.ctr) - - penc, err := rlp.EncodeToBytes(p) + penc, _ := rlp.EncodeToBytes(p) + frame, err := rlp.EncodeToBytes(testCodecFrame{c.id, authTag, p.Kind(), penc}) if err != nil { - panic(err) + return frame, authTag, err + } + + // Store recently sent challenges. + if w, ok := p.(*v5wire.Whoareyou); ok { + w.Nonce = authTag + w.Encoded = frame + if c.sentChallenges == nil { + c.sentChallenges = make(map[enode.ID]*v5wire.Whoareyou) + } + c.sentChallenges[toID] = w } - frame, err := rlp.EncodeToBytes(testCodecFrame{c.id, authTag, p.Kind(), penc}) return frame, authTag, err } +func (c *testCodec) CurrentChallenge(id enode.ID, addr string) *v5wire.Whoareyou { + return c.sentChallenges[id] +} + func (c *testCodec) Decode(input []byte, addr string) (enode.ID, *enode.Node, v5wire.Packet, error) { frame, p, err := c.decodeFrame(input) if err != nil { @@ -529,9 +895,13 @@ func (c *testCodec) Decode(input []byte, addr string) (enode.ID, *enode.Node, v5 return frame.NodeID, nil, p, nil } +func (c *testCodec) SessionNode(id enode.ID, addr string) *enode.Node { + return c.test.nodesByID[id].Node() +} + func (c *testCodec) decodeFrame(input []byte) (frame testCodecFrame, p v5wire.Packet, err error) { if err = rlp.DecodeBytes(input, &frame); err != nil { - return frame, nil, fmt.Errorf("invalid frame: %w", err) + return frame, nil, fmt.Errorf("invalid frame: %v", err) } switch frame.Ptype { case v5wire.UnknownPacket: @@ -548,49 +918,25 @@ func (c *testCodec) decodeFrame(input []byte) (frame testCodecFrame, p v5wire.Pa return frame, p, err } -func newUDPV5Test(t *testing.T, logger log.Logger) *udpV5Test { - return newUDPV5TestContext(context.Background(), t, logger) -} - -func newUDPV5TestContext(ctx context.Context, t *testing.T, logger log.Logger) *udpV5Test { - ctx = disableLookupSlowdown(ctx) - - replyTimeout := contextGetReplyTimeout(ctx) - if replyTimeout == 0 { - replyTimeout = 50 * time.Millisecond - } - +func newUDPV5Test(t *testing.T) *udpV5Test { test := &udpV5Test{ t: t, pipe: newpipe(), localkey: newkey(), remotekey: newkey(), - remoteaddr: &net.UDPAddr{IP: net.IP{10, 0, 1, 99}, Port: 30303}, + remoteaddr: netip.MustParseAddrPort("10.0.1.99:30303"), nodesByID: make(map[enode.ID]*enode.LocalNode), - nodesByIP: make(map[string]*enode.LocalNode), - } - t.Cleanup(test.close) - var err error - tmpDir := t.TempDir() - test.db, err = enode.OpenDB(context.Background(), "", tmpDir, logger) - if err != nil { - panic(err) + nodesByIP: make(map[netip.Addr]*enode.LocalNode), } - - ln := enode.NewLocalNode(test.db, test.localkey, logger) + test.db, _ = enode.OpenDB("") + ln := enode.NewLocalNode(test.db, test.localkey) ln.SetStaticIP(net.IP{10, 0, 0, 1}) ln.Set(enr.UDP(30303)) - test.udp, err = ListenV5(ctx, "test", test.pipe, ln, Config{ + test.udp, _ = ListenV5(test.pipe, ln, Config{ PrivateKey: test.localkey, - Log: testlog.Logger(t, log.LvlError), + Log: testlog.Logger(t, log.LvlTrace), ValidSchemes: enode.ValidSchemesForTesting, - ReplyTimeout: replyTimeout, - - TableRevalidateInterval: time.Hour, }) - if err != nil { - panic(err) - } test.udp.codec = &testCodec{test: test, id: ln.ID()} test.table = test.udp.tab test.nodesByID[ln.ID()] = ln @@ -600,16 +946,16 @@ func newUDPV5TestContext(ctx context.Context, t *testing.T, logger log.Logger) * } // handles a packet as if it had been sent to the transport. -func (test *udpV5Test) packetIn(packet v5wire.Packet, logger log.Logger) { +func (test *udpV5Test) packetIn(packet v5wire.Packet) { test.t.Helper() - test.packetInFrom(test.remotekey, test.remoteaddr, packet, logger) + test.packetInFrom(test.remotekey, test.remoteaddr, packet) } -// handles a packet as if it had been sent to the transport by the key/endpoint. -func (test *udpV5Test) packetInFrom(key *ecdsa.PrivateKey, addr *net.UDPAddr, packet v5wire.Packet, logger log.Logger) { +// packetInFrom handles a packet as if it had been sent to the transport by the key/endpoint. +func (test *udpV5Test) packetInFrom(key *ecdsa.PrivateKey, addr netip.AddrPort, packet v5wire.Packet) { test.t.Helper() - ln := test.getNode(key, addr, logger) + ln := test.getNode(key, addr) codec := &testCodec{test: test, id: ln.ID()} enc, _, err := codec.Encode(test.udp.Self().ID(), addr.String(), packet, nil) if err != nil { @@ -621,28 +967,22 @@ func (test *udpV5Test) packetInFrom(key *ecdsa.PrivateKey, addr *net.UDPAddr, pa } // getNode ensures the test knows about a node at the given endpoint. -func (test *udpV5Test) getNode(key *ecdsa.PrivateKey, addr *net.UDPAddr, logger log.Logger) *enode.LocalNode { - id := enode.PubkeyToIDV4(&key.PublicKey) +func (test *udpV5Test) getNode(key *ecdsa.PrivateKey, addr netip.AddrPort) *enode.LocalNode { + id := v4wire.EncodePubkey(&key.PublicKey).ID() ln := test.nodesByID[id] if ln == nil { - tmpDir := test.t.TempDir() - db, err := enode.OpenDB(context.Background(), "", tmpDir, logger) - if err != nil { - panic(err) - } - test.t.Cleanup(db.Close) - - ln = enode.NewLocalNode(db, key, logger) - ln.SetStaticIP(addr.IP) - ln.Set(enr.UDP(addr.Port)) + db, _ := enode.OpenDB("") + ln = enode.NewLocalNode(db, key) + ln.SetStaticIP(addr.Addr().AsSlice()) + ln.Set(enr.UDP(addr.Port())) test.nodesByID[id] = ln } - test.nodesByIP[string(addr.IP)] = ln + test.nodesByIP[addr.Addr()] = ln return ln } // waitPacketOut waits for the next output packet and handles it using the given 'validate' -// function. The function must be of type func (X, *net.UDPAddr, v5wire.Nonce) where X is +// function. The function must be of type func (X, netip.AddrPort, v5wire.Nonce) where X is // assignable to packetV5. func (test *udpV5Test) waitPacketOut(validate interface{}) (closed bool) { test.t.Helper() @@ -658,14 +998,8 @@ func (test *udpV5Test) waitPacketOut(validate interface{}) (closed bool) { test.t.Fatalf("timed out waiting for %v", exptype) return false } - ln := test.nodesByIP[string(dgram.to.IP)] + ln := test.nodesByIP[dgram.to.Addr()] if ln == nil { - _, _, packet, err := test.udp.codec.Decode(dgram.data, test.pipe.LocalAddr().String()) - if err != nil { - test.t.Errorf("failed to decode a UDP packet: %v", err) - } else { - test.t.Errorf("attempt to send UDP packet: %v", packet.Name()) - } test.t.Fatalf("attempt to send to non-existing node %v", &dgram.to) return false } @@ -679,7 +1013,7 @@ func (test *udpV5Test) waitPacketOut(validate interface{}) (closed bool) { test.t.Errorf("sent packet type mismatch, got: %v, want: %v", reflect.TypeOf(p), exptype) return false } - fn.Call([]reflect.Value{reflect.ValueOf(p), reflect.ValueOf(&dgram.to), reflect.ValueOf(frame.AuthTag)}) + fn.Call([]reflect.Value{reflect.ValueOf(p), reflect.ValueOf(dgram.to), reflect.ValueOf(frame.AuthTag)}) return false } @@ -693,23 +1027,7 @@ func (test *udpV5Test) close() { n.Database().Close() } } - - unmatchedCount := len(test.pipe.queue) - if (unmatchedCount > 0) && !test.t.Failed() { - test.t.Errorf("%d unmatched UDP packets in queue", unmatchedCount) - - for len(test.pipe.queue) > 0 { - dgram, err := test.pipe.receive() - if err != nil { - test.t.Errorf("Failed to receive remaining UDP packets: %v", err) - break - } - _, _, packet, err := test.udp.codec.Decode(dgram.data, test.pipe.LocalAddr().String()) - if err != nil { - test.t.Errorf("Failed to decode a remaining UDP packet: %v", err) - } else { - test.t.Errorf("Remaining UDP packet: %v", packet.Name()) - } - } + if len(test.pipe.queue) != 0 { + test.t.Fatalf("%d unmatched UDP packets in queue", len(test.pipe.queue)) } } diff --git a/p2p/discover/v5wire/crypto.go b/p2p/discover/v5wire/crypto.go index dcc1c8cd703..d946207a0f9 100644 --- a/p2p/discover/v5wire/crypto.go +++ b/p2p/discover/v5wire/crypto.go @@ -1,21 +1,18 @@ // Copyright 2020 The go-ethereum Authors -// (original work) -// Copyright 2024 The Erigon Authors -// (modifications) -// This file is part of Erigon. +// This file is part of the go-ethereum library. // -// Erigon is free software: you can redistribute it and/or modify +// The go-ethereum library is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // -// Erigon is distributed in the hope that it will be useful, +// The go-ethereum library is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Lesser General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License -// along with Erigon. If not, see . +// along with the go-ethereum library. If not, see . package v5wire @@ -28,11 +25,10 @@ import ( "fmt" "hash" - "golang.org/x/crypto/hkdf" - - "github.com/erigontech/erigon/common/crypto" "github.com/erigontech/erigon/common/math" + "github.com/erigontech/erigon/common/crypto" "github.com/erigontech/erigon/p2p/enode" + "golang.org/x/crypto/hkdf" ) const ( @@ -70,10 +66,10 @@ func DecodePubkey(curve elliptic.Curve, e []byte) (*ecdsa.PublicKey, error) { // idNonceHash computes the ID signature hash used in the handshake. func idNonceHash(h hash.Hash, challenge, ephkey []byte, destID enode.ID) []byte { h.Reset() - h.Write([]byte("discovery v5 identity proof")) //nolint:errcheck - h.Write(challenge) //nolint:errcheck - h.Write(ephkey) //nolint:errcheck - h.Write(destID[:]) //nolint:errcheck + h.Write([]byte("discovery v5 identity proof")) + h.Write(challenge) + h.Write(ephkey) + h.Write(destID[:]) return h.Sum(nil) } @@ -131,8 +127,8 @@ func deriveKeys(hash hashFn, priv *ecdsa.PrivateKey, pub *ecdsa.PublicKey, n1, n } kdf := hkdf.New(hash, eph, challenge, info) sec := session{writeKey: make([]byte, aesKeySize), readKey: make([]byte, aesKeySize)} - kdf.Read(sec.writeKey) //nolint:errcheck - kdf.Read(sec.readKey) //nolint:errcheck + kdf.Read(sec.writeKey) + kdf.Read(sec.readKey) clear(eph) return &sec } @@ -152,15 +148,14 @@ func ecdh(privkey *ecdsa.PrivateKey, pubkey *ecdsa.PublicKey) []byte { // encryptGCM encrypts pt using AES-GCM with the given key and nonce. The ciphertext is // appended to dest, which must not overlap with plaintext. The resulting ciphertext is 16 // bytes longer than plaintext because it contains an authentication tag. -// nolint:unparam func encryptGCM(dest, key, nonce, plaintext, authData []byte) ([]byte, error) { block, err := aes.NewCipher(key) if err != nil { - panic(fmt.Errorf("can't create block cipher: %w", err)) + panic(fmt.Errorf("can't create block cipher: %v", err)) } aesgcm, err := cipher.NewGCMWithNonceSize(block, gcmNonceSize) if err != nil { - panic(fmt.Errorf("can't create GCM: %w", err)) + panic(fmt.Errorf("can't create GCM: %v", err)) } return aesgcm.Seal(dest, nonce, plaintext, authData), nil } diff --git a/p2p/discover/v5wire/crypto_test.go b/p2p/discover/v5wire/crypto_test.go index 3256884190d..771112db1fb 100644 --- a/p2p/discover/v5wire/crypto_test.go +++ b/p2p/discover/v5wire/crypto_test.go @@ -1,21 +1,18 @@ // Copyright 2020 The go-ethereum Authors -// (original work) -// Copyright 2024 The Erigon Authors -// (modifications) -// This file is part of Erigon. +// This file is part of the go-ethereum library. // -// Erigon is free software: you can redistribute it and/or modify +// The go-ethereum library is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // -// Erigon is distributed in the hope that it will be useful, +// The go-ethereum library is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Lesser General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License -// along with Erigon. If not, see . +// along with the go-ethereum library. If not, see . package v5wire @@ -28,9 +25,8 @@ import ( "strings" "testing" - "github.com/erigontech/erigon/common/crypto" "github.com/erigontech/erigon/common/hexutil" - "github.com/erigontech/erigon/common/log/v3" + "github.com/erigontech/erigon/common/crypto" "github.com/erigontech/erigon/p2p/enode" ) @@ -45,11 +41,10 @@ func TestVector_ECDH(t *testing.T) { } func TestVector_KDF(t *testing.T) { - logger := log.New() var ( ephKey = hexPrivkey("0xfb757dc581730490a1d7a00deea65e9b1936924caaea8f44d476014856b68736") cdata = hexutil.MustDecode("0x000000000000000000000000000000006469736376350001010102030405060708090a0b0c00180102030405060708090a0b0c0d0e0f100000000000000000") - net = newHandshakeTest(t.TempDir(), logger) + net = newHandshakeTest() ) defer net.close() diff --git a/p2p/discover/v5wire/encoding.go b/p2p/discover/v5wire/encoding.go index 2fad47b369e..03ee4c627ac 100644 --- a/p2p/discover/v5wire/encoding.go +++ b/p2p/discover/v5wire/encoding.go @@ -1,21 +1,18 @@ -// Copyright 2019 The go-ethereum Authors -// (original work) -// Copyright 2024 The Erigon Authors -// (modifications) -// This file is part of Erigon. +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. // -// Erigon is free software: you can redistribute it and/or modify +// The go-ethereum library is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // -// Erigon is distributed in the hope that it will be useful, +// The go-ethereum library is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Lesser General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License -// along with Erigon. If not, see . +// along with the go-ethereum library. If not, see . package v5wire @@ -30,12 +27,12 @@ import ( "errors" "fmt" "hash" + "slices" - "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/common/mclock" - "github.com/erigontech/erigon/execution/rlp" "github.com/erigontech/erigon/p2p/enode" "github.com/erigontech/erigon/p2p/enr" + "github.com/erigontech/erigon/execution/rlp" ) // TODO concurrent WHOAREYOU tie-breaker @@ -69,7 +66,7 @@ type ( handshakeAuthData struct { h struct { SrcID enode.ID - SigSize byte // ignature data + SigSize byte // signature data PubkeySize byte // offset of } // Trailing variable-size data. @@ -94,11 +91,17 @@ const ( minVersion = 1 sizeofMaskingIV = 16 + // The minimum size of any Discovery v5 packet is 63 bytes. + // Should reject packets smaller than minPacketSize. + minPacketSize = 63 + + maxPacketSize = 1280 + minMessageSize = 48 // this refers to data after static headers randomPacketMsgSize = 20 ) -var protocolID = [6]byte{'d', 'i', 's', 'c', 'v', '5'} +var DefaultProtocolID = [6]byte{'d', 'i', 's', 'c', 'v', '5'} // Errors. var ( @@ -118,9 +121,17 @@ var ( // Public errors. var ( + // ErrInvalidReqID represents error when the ID is invalid. ErrInvalidReqID = errors.New("request ID larger than 8 bytes") ) +// IsInvalidHeader reports whether 'err' is related to an invalid packet header. When it +// returns false, it is pretty certain that the packet causing the error does not belong +// to discv5. +func IsInvalidHeader(err error) bool { + return err == errTooShort || err == errInvalidHeader || err == errMsgTooShort +} + // Packet sizes. var ( sizeofStaticHeader = binary.Size(StaticHeader{}) @@ -133,10 +144,11 @@ var ( // Codec encodes and decodes Discovery v5 packets. // This type is not safe for concurrent use. type Codec struct { - sha256 hash.Hash - localnode *enode.LocalNode - privkey *ecdsa.PrivateKey - sc *SessionCache + sha256 hash.Hash + localnode *enode.LocalNode + privkey *ecdsa.PrivateKey + sc *SessionCache + protocolID [6]byte // encoder buffers buf bytes.Buffer // whole packet @@ -145,16 +157,22 @@ type Codec struct { msgctbuf []byte // message data ciphertext // decoder buffer + decbuf []byte reader bytes.Reader } // NewCodec creates a wire codec. -func NewCodec(ln *enode.LocalNode, key *ecdsa.PrivateKey, clock mclock.Clock) *Codec { +func NewCodec(ln *enode.LocalNode, key *ecdsa.PrivateKey, clock mclock.Clock, protocolID *[6]byte) *Codec { c := &Codec{ - sha256: sha256.New(), - localnode: ln, - privkey: key, - sc: NewSessionCache(1024, clock), + sha256: sha256.New(), + localnode: ln, + privkey: key, + sc: NewSessionCache(1024, clock), + protocolID: DefaultProtocolID, + decbuf: make([]byte, maxPacketSize), + } + if protocolID != nil { + c.protocolID = *protocolID } return c } @@ -172,6 +190,11 @@ func (c *Codec) Encode(id enode.ID, addr string, packet Packet, challenge *Whoar ) switch { case packet.Kind() == WhoareyouPacket: + // just send the WHOAREYOU packet raw again, rather than the re-encoded challenge data + w := packet.(*Whoareyou) + if len(w.Encoded) > 0 { + return w.Encoded, w.Nonce, nil + } head, err = c.encodeWhoareyou(id, packet.(*Whoareyou)) case challenge != nil: // We have an unanswered challenge, send handshake. @@ -191,8 +214,8 @@ func (c *Codec) Encode(id enode.ID, addr string, packet Packet, challenge *Whoar } // Generate masking IV. - if err = c.sc.maskingIVGen(head.IV[:]); err != nil { - return nil, Nonce{}, fmt.Errorf("can't generate masking IV: %w", err) + if err := c.sc.maskingIVGen(head.IV[:]); err != nil { + return nil, Nonce{}, fmt.Errorf("can't generate masking IV: %v", err) } // Encode header data. @@ -200,16 +223,23 @@ func (c *Codec) Encode(id enode.ID, addr string, packet Packet, challenge *Whoar // Store sent WHOAREYOU challenges. if challenge, ok := packet.(*Whoareyou); ok { - challenge.ChallengeData = common.Copy(c.buf.Bytes()) + challenge.ChallengeData = slices.Clone(c.buf.Bytes()) + enc, err := c.EncodeRaw(id, head, msgData) + if err != nil { + return nil, Nonce{}, err + } + challenge.Encoded = bytes.Clone(enc) c.sc.storeSentHandshake(id, addr, challenge) - } else if msgData == nil { + return enc, head.Nonce, err + } + + if msgData == nil { headerData := c.buf.Bytes() msgData, err = c.encryptMessage(session, packet, &head, headerData) if err != nil { return nil, Nonce{}, err } } - enc, err := c.EncodeRaw(id, head, msgData) return enc, head.Nonce, err } @@ -221,23 +251,27 @@ func (c *Codec) EncodeRaw(id enode.ID, head Header, msgdata []byte) ([]byte, err // Apply masking. masked := c.buf.Bytes()[sizeofMaskingIV:] mask := head.mask(id) - mask.XORKeyStream(masked, masked) + mask.XORKeyStream(masked[:], masked[:]) // Write message data. c.buf.Write(msgdata) return c.buf.Bytes(), nil } +// CurrentChallenge returns the latest challenge sent to the given node. +// This will return non-nil while a handshake is in progress. +func (c *Codec) CurrentChallenge(id enode.ID, addr string) *Whoareyou { + return c.sc.getHandshake(id, addr) +} + func (c *Codec) writeHeaders(head *Header) { c.buf.Reset() c.buf.Write(head.IV[:]) - binary.Write(&c.buf, binary.BigEndian, &head.StaticHeader) //nolint:errcheck + binary.Write(&c.buf, binary.BigEndian, &head.StaticHeader) c.buf.Write(head.AuthData) } // makeHeader creates a packet header. -// -//nolint:unparam func (c *Codec) makeHeader(toID enode.ID, flag byte, authsizeExtra int) Header { var authsize int switch flag { @@ -256,7 +290,7 @@ func (c *Codec) makeHeader(toID enode.ID, flag byte, authsizeExtra int) Header { } return Header{ StaticHeader: StaticHeader{ - ProtocolID: protocolID, + ProtocolID: c.protocolID, Version: version, Flag: flag, AuthSize: uint16(authsize), @@ -271,21 +305,19 @@ func (c *Codec) encodeRandom(toID enode.ID) (Header, []byte, error) { // Encode auth data. auth := messageAuthData{SrcID: c.localnode.ID()} if _, err := crand.Read(head.Nonce[:]); err != nil { - return head, nil, fmt.Errorf("can't get random data: %w", err) + return head, nil, fmt.Errorf("can't get random data: %v", err) } c.headbuf.Reset() - binary.Write(&c.headbuf, binary.BigEndian, auth) //nolint:errcheck + binary.Write(&c.headbuf, binary.BigEndian, auth) head.AuthData = c.headbuf.Bytes() // Fill message ciphertext buffer with random bytes. c.msgctbuf = append(c.msgctbuf[:0], make([]byte, randomPacketMsgSize)...) - crand.Read(c.msgctbuf) //nolint:errcheck + crand.Read(c.msgctbuf) return head, c.msgctbuf, nil } // encodeWhoareyou encodes a WHOAREYOU packet. -// -//nolint:unparam func (c *Codec) encodeWhoareyou(toID enode.ID, packet *Whoareyou) (Header, error) { // Sanity check node field to catch misbehaving callers. if packet.RecordSeq > 0 && packet.Node == nil { @@ -294,7 +326,6 @@ func (c *Codec) encodeWhoareyou(toID enode.ID, packet *Whoareyou) (Header, error // Create header. head := c.makeHeader(toID, flagWhoareyou, 0) - head.AuthData = common.Copy(c.buf.Bytes()) head.Nonce = packet.Nonce // Encode auth data. @@ -303,7 +334,7 @@ func (c *Codec) encodeWhoareyou(toID enode.ID, packet *Whoareyou) (Header, error RecordSeq: packet.RecordSeq, } c.headbuf.Reset() - binary.Write(&c.headbuf, binary.BigEndian, auth) //nolint:errcheck + binary.Write(&c.headbuf, binary.BigEndian, auth) head.AuthData = c.headbuf.Bytes() return head, nil } @@ -324,11 +355,11 @@ func (c *Codec) encodeHandshakeHeader(toID enode.ID, addr string, challenge *Who // Generate nonce for message. nonce, err := c.sc.nextNonce(session) if err != nil { - return Header{}, nil, fmt.Errorf("can't generate nonce: %w", err) + return Header{}, nil, fmt.Errorf("can't generate nonce: %v", err) } // TODO: this should happen when the first authenticated message is received - c.sc.storeNewSession(toID, addr, session) + c.sc.storeNewSession(toID, addr, session, challenge.Node) // Encode the auth header. var ( @@ -336,7 +367,7 @@ func (c *Codec) encodeHandshakeHeader(toID enode.ID, addr string, challenge *Who head = c.makeHeader(toID, flagHandshake, authsizeExtra) ) c.headbuf.Reset() - binary.Write(&c.headbuf, binary.BigEndian, &auth.h) //nolint:errcheck + binary.Write(&c.headbuf, binary.BigEndian, &auth.h) c.headbuf.Write(auth.signature) c.headbuf.Write(auth.pubkey) c.headbuf.Write(auth.record) @@ -345,9 +376,7 @@ func (c *Codec) encodeHandshakeHeader(toID enode.ID, addr string, challenge *Who return head, session, err } -// encodeAuthHeader creates the auth header on a request packet following WHOAREYOU. -// -//nolint:unparam +// makeHandshakeAuth creates the auth header on a request packet following WHOAREYOU. func (c *Codec) makeHandshakeAuth(toID enode.ID, addr string, challenge *Whoareyou) (*handshakeAuthData, *session, error) { auth := new(handshakeAuthData) auth.h.SrcID = c.localnode.ID() @@ -363,14 +392,14 @@ func (c *Codec) makeHandshakeAuth(toID enode.ID, addr string, challenge *Whoarey return nil, nil, errors.New("can't generate ephemeral key") } ephpubkey := EncodePubkey(&ephkey.PublicKey) - auth.pubkey = ephpubkey + auth.pubkey = ephpubkey[:] auth.h.PubkeySize = byte(len(auth.pubkey)) // Add ID nonce signature to response. cdata := challenge.ChallengeData - idsig, err := makeIDSignature(c.sha256, c.privkey, cdata, ephpubkey, toID) + idsig, err := makeIDSignature(c.sha256, c.privkey, cdata, ephpubkey[:], toID) if err != nil { - return nil, nil, fmt.Errorf("can't sign: %w", err) + return nil, nil, fmt.Errorf("can't sign: %v", err) } auth.signature = idsig auth.h.SigSize = byte(len(auth.signature)) @@ -389,19 +418,19 @@ func (c *Codec) makeHandshakeAuth(toID enode.ID, addr string, challenge *Whoarey return auth, sec, err } -// encodeMessage encodes an encrypted message packet. +// encodeMessageHeader encodes an encrypted message packet. func (c *Codec) encodeMessageHeader(toID enode.ID, s *session) (Header, error) { head := c.makeHeader(toID, flagMessage, 0) // Create the header. nonce, err := c.sc.nextNonce(s) if err != nil { - return Header{}, fmt.Errorf("can't generate nonce: %w", err) + return Header{}, fmt.Errorf("can't generate nonce: %v", err) } auth := messageAuthData{SrcID: c.localnode.ID()} c.buf.Reset() - binary.Write(&c.buf, binary.BigEndian, &auth) //nolint:errcheck - head.AuthData = common.Copy(c.buf.Bytes()) + binary.Write(&c.buf, binary.BigEndian, &auth) + head.AuthData = slices.Clone(c.buf.Bytes()) head.Nonce = nonce return head, err } @@ -424,11 +453,14 @@ func (c *Codec) encryptMessage(s *session, p Packet, head *Header, headerData [] } // Decode decodes a discovery packet. -func (c *Codec) Decode(input []byte, addr string) (src enode.ID, n *enode.Node, p Packet, err error) { - // Unmask the static header. - if len(input) < sizeofStaticPacketData { +func (c *Codec) Decode(inputData []byte, addr string) (src enode.ID, n *enode.Node, p Packet, err error) { + if len(inputData) < minPacketSize { return enode.ID{}, nil, nil, errTooShort } + // Copy the packet to a tmp buffer to avoid modifying it. + c.decbuf = append(c.decbuf[:0], inputData...) + input := c.decbuf + // Unmask the static header. var head Header copy(head.IV[:], input[:sizeofMaskingIV]) mask := head.mask(c.localnode.ID()) @@ -437,9 +469,9 @@ func (c *Codec) Decode(input []byte, addr string) (src enode.ID, n *enode.Node, // Decode and verify the static header. c.reader.Reset(staticHeader) - binary.Read(&c.reader, binary.BigEndian, &head.StaticHeader) //nolint:errcheck + binary.Read(&c.reader, binary.BigEndian, &head.StaticHeader) remainingInput := len(input) - sizeofStaticPacketData - if err = head.checkValid(remainingInput); err != nil { + if err := head.checkValid(remainingInput, c.protocolID); err != nil { return enode.ID{}, nil, nil, err } @@ -476,7 +508,7 @@ func (c *Codec) decodeWhoareyou(head *Header, headerData []byte) (Packet, error) } var auth whoareyouAuthData c.reader.Reset(head.AuthData) - binary.Read(&c.reader, binary.BigEndian, &auth) //nolint:errcheck + binary.Read(&c.reader, binary.BigEndian, &auth) p := &Whoareyou{ Nonce: head.Nonce, IDNonce: auth.IDNonce, @@ -502,7 +534,7 @@ func (c *Codec) decodeHandshakeMessage(fromAddr string, head *Header, headerData } // Handshake OK, drop the challenge and store the new session keys. - c.sc.storeNewSession(auth.h.SrcID, fromAddr, session) + c.sc.storeNewSession(auth.h.SrcID, fromAddr, session, node) c.sc.deleteHandshake(auth.h.SrcID, fromAddr) return node, msg, nil } @@ -534,7 +566,7 @@ func (c *Codec) decodeHandshake(fromAddr string, head *Header) (n *enode.Node, a if err != nil { return nil, auth, nil, errInvalidAuthKey } - // Derive sesssion keys. + // Derive session keys. session := deriveKeys(sha256.New, c.privkey, ephkey, auth.h.SrcID, c.localnode.ID(), cdata) session = session.keysFlipped() return n, auth, session, nil @@ -547,7 +579,7 @@ func (c *Codec) decodeHandshakeAuthData(head *Header) (auth handshakeAuthData, e return auth, fmt.Errorf("header authsize %d too low for handshake", head.AuthSize) } c.reader.Reset(head.AuthData) - binary.Read(&c.reader, binary.BigEndian, &auth.h) //nolint:errcheck + binary.Read(&c.reader, binary.BigEndian, &auth.h) head.src = auth.h.SrcID // Decode variable-size part. @@ -579,7 +611,7 @@ func (c *Codec) decodeHandshakeRecord(local *enode.Node, wantID enode.ID, remote if local == nil || local.Seq() < record.Seq() { n, err := enode.New(enode.ValidSchemes, &record) if err != nil { - return nil, fmt.Errorf("invalid node record: %w", err) + return nil, fmt.Errorf("invalid node record: %v", err) } if n.ID() != wantID { return nil, fmt.Errorf("record in handshake has wrong ID: %v", n.ID()) @@ -600,7 +632,7 @@ func (c *Codec) decodeMessage(fromAddr string, head *Header, headerData, msgData } var auth messageAuthData c.reader.Reset(head.AuthData) - binary.Read(&c.reader, binary.BigEndian, &auth) //nolint:errcheck + binary.Read(&c.reader, binary.BigEndian, &auth) head.src = auth.SrcID // Try decrypting the message. @@ -624,9 +656,13 @@ func (c *Codec) decryptMessage(input, nonce, headerData, readKey []byte) (Packet return DecodeMessage(msgdata[0], msgdata[1:]) } +func (c *Codec) SessionNode(id enode.ID, addr string) *enode.Node { + return c.sc.readNode(id, addr) +} + // checkValid performs some basic validity checks on the header. // The packetLen here is the length remaining after the static header. -func (h *StaticHeader) checkValid(packetLen int) error { +func (h *StaticHeader) checkValid(packetLen int, protocolID [6]byte) error { if h.ProtocolID != protocolID { return errInvalidHeader } @@ -642,7 +678,7 @@ func (h *StaticHeader) checkValid(packetLen int) error { return nil } -// headerMask returns a cipher for 'masking' / 'unmasking' packet headers. +// mask returns a cipher for 'masking' / 'unmasking' packet headers. func (h *Header) mask(destID enode.ID) cipher.Stream { block, err := aes.NewCipher(destID[:16]) if err != nil { diff --git a/p2p/discover/v5wire/encoding_test.go b/p2p/discover/v5wire/encoding_test.go index 41a4bc692cc..7cf5ae8909e 100644 --- a/p2p/discover/v5wire/encoding_test.go +++ b/p2p/discover/v5wire/encoding_test.go @@ -1,45 +1,39 @@ -// Copyright 2019 The go-ethereum Authors -// (original work) -// Copyright 2024 The Erigon Authors -// (modifications) -// This file is part of Erigon. +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. // -// Erigon is free software: you can redistribute it and/or modify +// The go-ethereum library is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // -// Erigon is distributed in the hope that it will be useful, +// The go-ethereum library is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Lesser General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License -// along with Erigon. If not, see . +// along with the go-ethereum library. If not, see . package v5wire import ( "bytes" - "context" "crypto/ecdsa" "encoding/hex" + "errors" "flag" "fmt" "net" "os" "path/filepath" - "reflect" - "runtime" "strings" "testing" "github.com/davecgh/go-spew/spew" - "github.com/erigontech/erigon/common/crypto" "github.com/erigontech/erigon/common/hexutil" - "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/common/mclock" + "github.com/erigontech/erigon/common/crypto" "github.com/erigontech/erigon/p2p/enode" ) @@ -75,9 +69,7 @@ func TestMinSizes(t *testing.T) { // This test checks the basic handshake flow where A talks to B and A has no secrets. func TestHandshake(t *testing.T) { t.Parallel() - logger := log.New() - tmpDir := t.TempDir() - net := newHandshakeTest(tmpDir, logger) + net := newHandshakeTest() defer net.close() // A -> B RANDOM PACKET @@ -101,16 +93,14 @@ func TestHandshake(t *testing.T) { } // A <- B NODES - nodes, _ := net.nodeB.encode(t, net.nodeA, &Nodes{Total: 1}) + nodes, _ := net.nodeB.encode(t, net.nodeA, &Nodes{RespCount: 1}) net.nodeA.expectDecode(t, NodesMsg, nodes) } // This test checks that handshake attempts are removed within the timeout. func TestHandshake_timeout(t *testing.T) { t.Parallel() - logger := log.New() - tmpDir := t.TempDir() - net := newHandshakeTest(tmpDir, logger) + net := newHandshakeTest() defer net.close() // A -> B RANDOM PACKET @@ -135,9 +125,7 @@ func TestHandshake_timeout(t *testing.T) { // This test checks handshake behavior when no record is sent in the auth response. func TestHandshake_norecord(t *testing.T) { t.Parallel() - logger := log.New() - tmpDir := t.TempDir() - net := newHandshakeTest(tmpDir, logger) + net := newHandshakeTest() defer net.close() // A -> B RANDOM PACKET @@ -163,30 +151,22 @@ func TestHandshake_norecord(t *testing.T) { net.nodeB.expectDecode(t, FindnodeMsg, findnode) // A <- B NODES - nodes, _ := net.nodeB.encode(t, net.nodeA, &Nodes{Total: 1}) + nodes, _ := net.nodeB.encode(t, net.nodeA, &Nodes{RespCount: 1}) net.nodeA.expectDecode(t, NodesMsg, nodes) } // In this test, A tries to send FINDNODE with existing secrets but B doesn't know // anything about A. func TestHandshake_rekey(t *testing.T) { - // runtime: setevent failed; errno=6 - // fatal error: runtime.semawakeup - if runtime.GOOS != "linux" { - t.Skip("fix me on win please") - } - t.Parallel() - tmpDir := t.TempDir() - logger := log.New() - net := newHandshakeTest(tmpDir, logger) + net := newHandshakeTest() defer net.close() session := &session{ readKey: []byte("BBBBBBBBBBBBBBBB"), writeKey: []byte("AAAAAAAAAAAAAAAA"), } - net.nodeA.c.sc.storeNewSession(net.nodeB.id(), net.nodeB.addr(), session) + net.nodeA.c.sc.storeNewSession(net.nodeB.id(), net.nodeB.addr(), session, net.nodeB.n()) // A -> B FINDNODE (encrypted with zero keys) findnode, authTag := net.nodeA.encode(t, net.nodeB, &Findnode{}) @@ -211,16 +191,14 @@ func TestHandshake_rekey(t *testing.T) { net.nodeB.expectDecode(t, FindnodeMsg, findnode) // A <- B NODES - nodes, _ := net.nodeB.encode(t, net.nodeA, &Nodes{Total: 1}) + nodes, _ := net.nodeB.encode(t, net.nodeA, &Nodes{RespCount: 1}) net.nodeA.expectDecode(t, NodesMsg, nodes) } // In this test A and B have different keys before the handshake. func TestHandshake_rekey2(t *testing.T) { t.Parallel() - tmpDir := t.TempDir() - logger := log.New() - net := newHandshakeTest(tmpDir, logger) + net := newHandshakeTest() defer net.close() initKeysA := &session{ @@ -231,8 +209,8 @@ func TestHandshake_rekey2(t *testing.T) { readKey: []byte("CCCCCCCCCCCCCCCC"), writeKey: []byte("DDDDDDDDDDDDDDDD"), } - net.nodeA.c.sc.storeNewSession(net.nodeB.id(), net.nodeB.addr(), initKeysA) - net.nodeB.c.sc.storeNewSession(net.nodeA.id(), net.nodeA.addr(), initKeysB) + net.nodeA.c.sc.storeNewSession(net.nodeB.id(), net.nodeB.addr(), initKeysA, net.nodeB.n()) + net.nodeB.c.sc.storeNewSession(net.nodeA.id(), net.nodeA.addr(), initKeysB, net.nodeA.n()) // A -> B FINDNODE encrypted with initKeysA findnode, authTag := net.nodeA.encode(t, net.nodeB, &Findnode{Distances: []uint{3}}) @@ -248,15 +226,13 @@ func TestHandshake_rekey2(t *testing.T) { net.nodeB.expectDecode(t, FindnodeMsg, findnode) // A <- B NODES - nodes, _ := net.nodeB.encode(t, net.nodeA, &Nodes{Total: 1}) + nodes, _ := net.nodeB.encode(t, net.nodeA, &Nodes{RespCount: 1}) net.nodeA.expectDecode(t, NodesMsg, nodes) } func TestHandshake_BadHandshakeAttack(t *testing.T) { t.Parallel() - tmpDir := t.TempDir() - logger := log.New() - net := newHandshakeTest(tmpDir, logger) + net := newHandshakeTest() defer net.close() // A -> B RANDOM PACKET @@ -273,20 +249,20 @@ func TestHandshake_BadHandshakeAttack(t *testing.T) { net.nodeA.expectDecode(t, WhoareyouPacket, whoareyou) // A -> B FINDNODE - incorrect_challenge := &Whoareyou{ + incorrectChallenge := &Whoareyou{ IDNonce: [16]byte{5, 6, 7, 8, 9, 6, 11, 12}, RecordSeq: challenge.RecordSeq, Node: challenge.Node, sent: challenge.sent, } - incorrect_findnode, _ := net.nodeA.encodeWithChallenge(t, net.nodeB, incorrect_challenge, &Findnode{}) - incorrect_findnode2 := make([]byte, len(incorrect_findnode)) - copy(incorrect_findnode2, incorrect_findnode) + incorrectFindNode, _ := net.nodeA.encodeWithChallenge(t, net.nodeB, incorrectChallenge, &Findnode{}) + incorrectFindNode2 := make([]byte, len(incorrectFindNode)) + copy(incorrectFindNode2, incorrectFindNode) - net.nodeB.expectDecodeErr(t, errInvalidNonceSig, incorrect_findnode) + net.nodeB.expectDecodeErr(t, errInvalidNonceSig, incorrectFindNode) // Reject new findnode as previous handshake is now deleted. - net.nodeB.expectDecodeErr(t, errUnexpectedHandshake, incorrect_findnode2) + net.nodeB.expectDecodeErr(t, errUnexpectedHandshake, incorrectFindNode2) // The findnode packet is again rejected even with a valid challenge this time. findnode, _ := net.nodeA.encodeWithChallenge(t, net.nodeB, challenge, &Findnode{}) @@ -296,20 +272,54 @@ func TestHandshake_BadHandshakeAttack(t *testing.T) { // This test checks some malformed packets. func TestDecodeErrorsV5(t *testing.T) { t.Parallel() - tmpDir := t.TempDir() - logger := log.New() - net := newHandshakeTest(tmpDir, logger) + net := newHandshakeTest() defer net.close() - net.nodeA.expectDecodeErr(t, errTooShort, []byte{}) - // TODO some more tests would be nice :) - // - check invalid authdata sizes - // - check invalid handshake data sizes + b := make([]byte, 0) + net.nodeA.expectDecodeErr(t, errTooShort, b) + + b = make([]byte, 62) + net.nodeA.expectDecodeErr(t, errTooShort, b) + + b = make([]byte, 63) + net.nodeA.expectDecodeErr(t, errInvalidHeader, b) + + t.Run("invalid-handshake-datasize", func(t *testing.T) { + requiredNumber := 108 + + testDataFile := filepath.Join("testdata", "v5.1-ping-handshake"+".txt") + enc := hexFile(testDataFile) + //delete some byte from handshake to make it invalid + enc = enc[:len(enc)-requiredNumber] + net.nodeB.expectDecodeErr(t, errMsgTooShort, enc) + }) + + t.Run("invalid-auth-datasize", func(t *testing.T) { + testPacket := []byte{} + testDataFiles := []string{"v5.1-whoareyou", "v5.1-ping-handshake"} + for counter, name := range testDataFiles { + file := filepath.Join("testdata", name+".txt") + enc := hexFile(file) + if counter == 0 { + //make whoareyou header + testPacket = enc[:sizeofStaticPacketData-1] + testPacket = append(testPacket, 255) + } + if counter == 1 { + //append invalid auth size + testPacket = append(testPacket, enc[sizeofStaticPacketData:]...) + } + } + + wantErr := "invalid auth size" + if _, err := net.nodeB.decode(testPacket); strings.HasSuffix(err.Error(), wantErr) { + t.Fatal(fmt.Errorf("(%s) got err %q, want %q", net.nodeB.ln.ID().TerminalString(), err, wantErr)) + } + }) } // This test checks that all test vectors can be decoded. func TestTestVectorsV5(t *testing.T) { - logger := log.New() var ( idA = enode.PubkeyToIDV4(&testKeyA.PublicKey) idB = enode.PubkeyToIDV4(&testKeyB.PublicKey) @@ -328,8 +338,7 @@ func TestTestVectorsV5(t *testing.T) { } challenge0A, challenge1A, challenge0B = c, c, c challenge1A.RecordSeq = 1 - tmpDir := t.TempDir() - net := newHandshakeTest(tmpDir, logger) + net := newHandshakeTest() challenge0A.Node = net.nodeA.n() challenge0B.Node = net.nodeB.n() challenge1A.Node = net.nodeA.n() @@ -353,8 +362,8 @@ func TestTestVectorsV5(t *testing.T) { ENRSeq: 2, }, prep: func(net *handshakeTest) { - net.nodeA.c.sc.storeNewSession(idB, addr, session) - net.nodeB.c.sc.storeNewSession(idA, addr, session.keysFlipped()) + net.nodeA.c.sc.storeNewSession(idB, addr, session, net.nodeB.n()) + net.nodeB.c.sc.storeNewSession(idA, addr, session.keysFlipped(), net.nodeA.n()) }, }, { @@ -366,7 +375,7 @@ func TestTestVectorsV5(t *testing.T) { challenge: &challenge0A, prep: func(net *handshakeTest) { // Update challenge.Header.AuthData. - net.nodeA.c.Encode(idB, "", &challenge0A, nil) //nolint:errcheck + net.nodeA.c.Encode(idB, "", &challenge0A, nil) net.nodeB.c.sc.storeSentHandshake(idA, addr, &challenge0A) }, }, @@ -379,7 +388,7 @@ func TestTestVectorsV5(t *testing.T) { challenge: &challenge1A, prep: func(net *handshakeTest) { // Update challenge data. - net.nodeA.c.Encode(idB, "", &challenge1A, nil) //nolint:errcheck + net.nodeA.c.Encode(idB, "", &challenge1A, nil) net.nodeB.c.sc.storeSentHandshake(idA, addr, &challenge1A) }, }, @@ -387,7 +396,7 @@ func TestTestVectorsV5(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - net := newHandshakeTest(tmpDir, logger) + net := newHandshakeTest() defer net.close() // Override all random inputs. @@ -455,9 +464,7 @@ func testVectorComment(net *handshakeTest, p Packet, challenge *Whoareyou, nonce // This benchmark checks performance of handshake packet decoding. func BenchmarkV5_DecodeHandshakePingSecp256k1(b *testing.B) { - tmpDir := b.TempDir() - logger := log.New() - net := newHandshakeTest(tmpDir, logger) + net := newHandshakeTest() defer net.close() var ( @@ -484,17 +491,15 @@ func BenchmarkV5_DecodeHandshakePingSecp256k1(b *testing.B) { // This benchmark checks how long it takes to decode an encrypted ping packet. func BenchmarkV5_DecodePing(b *testing.B) { - tmpDir := b.TempDir() - logger := log.New() - net := newHandshakeTest(tmpDir, logger) + net := newHandshakeTest() defer net.close() session := &session{ readKey: []byte{233, 203, 93, 195, 86, 47, 177, 186, 227, 43, 2, 141, 244, 230, 120, 17}, writeKey: []byte{79, 145, 252, 171, 167, 216, 252, 161, 208, 190, 176, 106, 214, 39, 178, 134}, } - net.nodeA.c.sc.storeNewSession(net.nodeB.id(), net.nodeB.addr(), session) - net.nodeB.c.sc.storeNewSession(net.nodeA.id(), net.nodeA.addr(), session.keysFlipped()) + net.nodeA.c.sc.storeNewSession(net.nodeB.id(), net.nodeB.addr(), session, net.nodeB.n()) + net.nodeB.c.sc.storeNewSession(net.nodeA.id(), net.nodeA.addr(), session.keysFlipped(), net.nodeA.n()) addrB := net.nodeA.addr() ping := &Ping{ReqID: []byte("reqid"), ENRSeq: 5} enc, _, err := net.nodeA.c.Encode(net.nodeB.id(), addrB, ping, nil) @@ -524,10 +529,10 @@ type handshakeTestNode struct { c *Codec } -func newHandshakeTest(tmpDir string, logger log.Logger) *handshakeTest { +func newHandshakeTest() *handshakeTest { t := new(handshakeTest) - t.nodeA.init(testKeyA, net.IP{127, 0, 0, 1}, &t.clock, tmpDir, logger) - t.nodeB.init(testKeyB, net.IP{127, 0, 0, 1}, &t.clock, tmpDir, logger) + t.nodeA.init(testKeyA, net.IP{127, 0, 0, 1}, &t.clock, DefaultProtocolID) + t.nodeB.init(testKeyB, net.IP{127, 0, 0, 1}, &t.clock, DefaultProtocolID) return t } @@ -536,26 +541,20 @@ func (t *handshakeTest) close() { t.nodeB.ln.Database().Close() } -func (n *handshakeTestNode) init(key *ecdsa.PrivateKey, ip net.IP, clock mclock.Clock, tmpDir string, logger log.Logger) { - db, err := enode.OpenDB(context.Background(), "", tmpDir, logger) - if err != nil { - panic(err) - } - n.ln = enode.NewLocalNode(db, key, logger) +func (n *handshakeTestNode) init(key *ecdsa.PrivateKey, ip net.IP, clock mclock.Clock, protocolID [6]byte) { + db, _ := enode.OpenDB("") + n.ln = enode.NewLocalNode(db, key) n.ln.SetStaticIP(ip) - if n.ln.Node().Seq() != 1 { - panic(fmt.Errorf("unexpected seq %d", n.ln.Node().Seq())) - } - n.c = NewCodec(n.ln, key, clock) + n.c = NewCodec(n.ln, key, clock, nil) } -func (n *handshakeTestNode) encode(tb testing.TB, to handshakeTestNode, p Packet) ([]byte, Nonce) { - tb.Helper() - return n.encodeWithChallenge(tb, to, nil, p) +func (n *handshakeTestNode) encode(t testing.TB, to handshakeTestNode, p Packet) ([]byte, Nonce) { + t.Helper() + return n.encodeWithChallenge(t, to, nil, p) } -func (n *handshakeTestNode) encodeWithChallenge(tb testing.TB, to handshakeTestNode, c *Whoareyou, p Packet) ([]byte, Nonce) { - tb.Helper() +func (n *handshakeTestNode) encodeWithChallenge(t testing.TB, to handshakeTestNode, c *Whoareyou, p Packet) ([]byte, Nonce) { + t.Helper() // Copy challenge and add destination node. This avoids sharing 'c' among the two codecs. var challenge *Whoareyou @@ -567,9 +566,9 @@ func (n *handshakeTestNode) encodeWithChallenge(tb testing.TB, to handshakeTestN // Encode to destination. enc, nonce, err := n.c.Encode(to.id(), to.addr(), p, challenge) if err != nil { - tb.Fatal(fmt.Errorf("(%s) %w", n.ln.ID().TerminalString(), err)) + t.Fatal(fmt.Errorf("(%s) %v", n.ln.ID().TerminalString(), err)) } - //tb.Logf("(%s) -> (%s) %s\n%s", n.ln.ID().TerminalString(), to.id().TerminalString(), p.Name(), hex.Dump(enc)) + t.Logf("(%s) -> (%s) %s\n%s", n.ln.ID().TerminalString(), to.id().TerminalString(), p.Name(), hex.Dump(enc)) return enc, nonce } @@ -578,9 +577,9 @@ func (n *handshakeTestNode) expectDecode(t *testing.T, ptype byte, p []byte) Pac dec, err := n.decode(p) if err != nil { - t.Fatal(fmt.Errorf("(%s) %w", n.ln.ID().TerminalString(), err)) + t.Fatal(fmt.Errorf("(%s) %v", n.ln.ID().TerminalString(), err)) } - //t.Logf("(%s) %#v", n.ln.ID().TerminalString(), pp.NewFormatter(dec)) + t.Logf("(%s) %#v", n.ln.ID().TerminalString(), pp.NewFormatter(dec)) if dec.Kind() != ptype { t.Fatalf("expected packet type %d, got %d", ptype, dec.Kind()) } @@ -589,7 +588,7 @@ func (n *handshakeTestNode) expectDecode(t *testing.T, ptype byte, p []byte) Pac func (n *handshakeTestNode) expectDecodeErr(t *testing.T, wantErr error, p []byte) { t.Helper() - if _, err := n.decode(p); !reflect.DeepEqual(err, wantErr) { + if _, err := n.decode(p); !errors.Is(err, wantErr) { t.Fatal(fmt.Errorf("(%s) got err %q, want %q", n.ln.ID().TerminalString(), err, wantErr)) } } @@ -604,7 +603,7 @@ func (n *handshakeTestNode) n() *enode.Node { } func (n *handshakeTestNode) addr() string { - return n.ln.Node().IP().String() + return n.ln.Node().IPAddr().String() } func (n *handshakeTestNode) id() enode.ID { @@ -620,8 +619,8 @@ func hexFile(file string) []byte { } // Gather hex data, ignore comments. - var text []byte //nolint:prealloc - for line := range bytes.SplitSeq(fileContent, []byte("\n")) { + var text []byte + for _, line := range bytes.Split(fileContent, []byte("\n")) { line = bytes.TrimSpace(line) if len(line) > 0 && line[0] == '#' { continue @@ -649,7 +648,7 @@ func writeTestVector(file, comment string, data []byte) { defer fd.Close() if len(comment) > 0 { - for line := range strings.SplitSeq(strings.TrimSpace(comment), "\n") { + for _, line := range strings.Split(strings.TrimSpace(comment), "\n") { fmt.Fprintf(fd, "# %s\n", line) } fmt.Fprintln(fd) diff --git a/p2p/discover/v5wire/msg.go b/p2p/discover/v5wire/msg.go index 9634da90b22..3c74cbb4bf4 100644 --- a/p2p/discover/v5wire/msg.go +++ b/p2p/discover/v5wire/msg.go @@ -1,21 +1,18 @@ -// Copyright 2019 The go-ethereum Authors -// (original work) -// Copyright 2024 The Erigon Authors -// (modifications) -// This file is part of Erigon. +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. // -// Erigon is free software: you can redistribute it and/or modify +// The go-ethereum library is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // -// Erigon is distributed in the hope that it will be useful, +// The go-ethereum library is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Lesser General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License -// along with Erigon. If not, see . +// along with the go-ethereum library. If not, see . package v5wire @@ -23,10 +20,11 @@ import ( "fmt" "net" + "github.com/erigontech/erigon/common/hexutil" "github.com/erigontech/erigon/common/mclock" - "github.com/erigontech/erigon/execution/rlp" "github.com/erigontech/erigon/p2p/enode" "github.com/erigontech/erigon/p2p/enr" + "github.com/erigontech/erigon/execution/rlp" ) // Packet is implemented by all message types. @@ -35,6 +33,10 @@ type Packet interface { Kind() byte // Kind returns the message type. RequestID() []byte // Returns the request ID. SetRequestID([]byte) // Sets the request ID. + + // AppendLogInfo returns its argument 'ctx' with additional fields + // appended for logging purposes. + AppendLogInfo(ctx []interface{}) []interface{} } // Message types. @@ -47,9 +49,6 @@ const ( TalkResponseMsg RequestTicketMsg TicketMsg - RegtopicMsg - RegconfirmationMsg - TopicQueryMsg UnknownPacket = byte(255) // any non-decryptable packet WhoareyouPacket = byte(254) // the WHOAREYOU packet @@ -74,6 +73,9 @@ type ( Node *enode.Node sent mclock.AbsTime // for handshake GC. + + // Encoded is packet raw data for sending out, but should not be include in the RLP encoding. + Encoded []byte `rlp:"-"` } // PING is sent during liveness checks. @@ -94,13 +96,17 @@ type ( Findnode struct { ReqID []byte Distances []uint + + // OpID is for debugging purposes and is not part of the packet encoding. + // It identifies the 'operation' on behalf of which the request was sent. + OpID uint64 `rlp:"-"` } - // NODES is the reply to FINDNODE and TOPICQUERY. + // NODES is a response to FINDNODE. Nodes struct { - ReqID []byte - Total uint8 - Nodes []*enr.Record + ReqID []byte + RespCount uint8 // total number of responses to the request + Nodes []*enr.Record } // TALKREQ is an application-level request. @@ -115,37 +121,6 @@ type ( ReqID []byte Message []byte } - - // REQUESTTICKET requests a ticket for a topic queue. - RequestTicket struct { - ReqID []byte - Topic []byte - } - - // TICKET is the response to REQUESTTICKET. - Ticket struct { - ReqID []byte - Ticket []byte - } - - // REGTOPIC registers the sender in a topic queue using a ticket. - Regtopic struct { - ReqID []byte - Ticket []byte - ENR *enr.Record - } - - // REGCONFIRMATION is the reply to REGTOPIC. - Regconfirmation struct { - ReqID []byte - Registered bool - } - - // TOPICQUERY asks for nodes with the given topic. - TopicQuery struct { - ReqID []byte - Topic []byte - } ) // DecodeMessage decodes the message body of a packet. @@ -164,16 +139,6 @@ func DecodeMessage(ptype byte, body []byte) (Packet, error) { dec = new(TalkRequest) case TalkResponseMsg: dec = new(TalkResponse) - case RequestTicketMsg: - dec = new(RequestTicket) - case TicketMsg: - dec = new(Ticket) - case RegtopicMsg: - dec = new(Regtopic) - case RegconfirmationMsg: - dec = new(Regconfirmation) - case TopicQueryMsg: - dec = new(TopicQuery) default: return nil, fmt.Errorf("unknown packet type %d", ptype) } @@ -191,62 +156,77 @@ func (*Whoareyou) Kind() byte { return WhoareyouPacket } func (*Whoareyou) RequestID() []byte { return nil } func (*Whoareyou) SetRequestID([]byte) {} +func (*Whoareyou) AppendLogInfo(ctx []interface{}) []interface{} { + return ctx +} + func (*Unknown) Name() string { return "UNKNOWN/v5" } func (*Unknown) Kind() byte { return UnknownPacket } func (*Unknown) RequestID() []byte { return nil } func (*Unknown) SetRequestID([]byte) {} +func (*Unknown) AppendLogInfo(ctx []interface{}) []interface{} { + return ctx +} + func (*Ping) Name() string { return "PING/v5" } func (*Ping) Kind() byte { return PingMsg } func (p *Ping) RequestID() []byte { return p.ReqID } func (p *Ping) SetRequestID(id []byte) { p.ReqID = id } +func (p *Ping) AppendLogInfo(ctx []interface{}) []interface{} { + return append(ctx, "req", hexutil.Bytes(p.ReqID), "enrseq", p.ENRSeq) +} + func (*Pong) Name() string { return "PONG/v5" } func (*Pong) Kind() byte { return PongMsg } func (p *Pong) RequestID() []byte { return p.ReqID } func (p *Pong) SetRequestID(id []byte) { p.ReqID = id } -func (*Findnode) Name() string { return "FINDNODE/v5" } -func (*Findnode) Kind() byte { return FindnodeMsg } +func (p *Pong) AppendLogInfo(ctx []interface{}) []interface{} { + return append(ctx, "req", hexutil.Bytes(p.ReqID), "enrseq", p.ENRSeq) +} + +func (p *Findnode) Name() string { return "FINDNODE/v5" } +func (p *Findnode) Kind() byte { return FindnodeMsg } func (p *Findnode) RequestID() []byte { return p.ReqID } func (p *Findnode) SetRequestID(id []byte) { p.ReqID = id } +func (p *Findnode) AppendLogInfo(ctx []interface{}) []interface{} { + ctx = append(ctx, "req", hexutil.Bytes(p.ReqID)) + if p.OpID != 0 { + ctx = append(ctx, "opid", p.OpID) + } + return ctx +} + func (*Nodes) Name() string { return "NODES/v5" } func (*Nodes) Kind() byte { return NodesMsg } func (p *Nodes) RequestID() []byte { return p.ReqID } func (p *Nodes) SetRequestID(id []byte) { p.ReqID = id } +func (p *Nodes) AppendLogInfo(ctx []interface{}) []interface{} { + return append(ctx, + "req", hexutil.Bytes(p.ReqID), + "tot", p.RespCount, + "n", len(p.Nodes), + ) +} + func (*TalkRequest) Name() string { return "TALKREQ/v5" } func (*TalkRequest) Kind() byte { return TalkRequestMsg } func (p *TalkRequest) RequestID() []byte { return p.ReqID } func (p *TalkRequest) SetRequestID(id []byte) { p.ReqID = id } +func (p *TalkRequest) AppendLogInfo(ctx []interface{}) []interface{} { + return append(ctx, "proto", p.Protocol, "req", hexutil.Bytes(p.ReqID), "len", len(p.Message)) +} + func (*TalkResponse) Name() string { return "TALKRESP/v5" } func (*TalkResponse) Kind() byte { return TalkResponseMsg } func (p *TalkResponse) RequestID() []byte { return p.ReqID } func (p *TalkResponse) SetRequestID(id []byte) { p.ReqID = id } -func (*RequestTicket) Name() string { return "REQTICKET/v5" } -func (*RequestTicket) Kind() byte { return RequestTicketMsg } -func (p *RequestTicket) RequestID() []byte { return p.ReqID } -func (p *RequestTicket) SetRequestID(id []byte) { p.ReqID = id } - -func (*Regtopic) Name() string { return "REGTOPIC/v5" } -func (*Regtopic) Kind() byte { return RegtopicMsg } -func (p *Regtopic) RequestID() []byte { return p.ReqID } -func (p *Regtopic) SetRequestID(id []byte) { p.ReqID = id } - -func (*Ticket) Name() string { return "TICKET/v5" } -func (*Ticket) Kind() byte { return TicketMsg } -func (p *Ticket) RequestID() []byte { return p.ReqID } -func (p *Ticket) SetRequestID(id []byte) { p.ReqID = id } - -func (*Regconfirmation) Name() string { return "REGCONFIRMATION/v5" } -func (*Regconfirmation) Kind() byte { return RegconfirmationMsg } -func (p *Regconfirmation) RequestID() []byte { return p.ReqID } -func (p *Regconfirmation) SetRequestID(id []byte) { p.ReqID = id } - -func (*TopicQuery) Name() string { return "TOPICQUERY/v5" } -func (*TopicQuery) Kind() byte { return TopicQueryMsg } -func (p *TopicQuery) RequestID() []byte { return p.ReqID } -func (p *TopicQuery) SetRequestID(id []byte) { p.ReqID = id } +func (p *TalkResponse) AppendLogInfo(ctx []interface{}) []interface{} { + return append(ctx, "req", hexutil.Bytes(p.ReqID), "len", len(p.Message)) +} diff --git a/p2p/discover/v5wire/session.go b/p2p/discover/v5wire/session.go index 7ec1a51d45d..ed3bc490f9b 100644 --- a/p2p/discover/v5wire/session.go +++ b/p2p/discover/v5wire/session.go @@ -1,21 +1,18 @@ // Copyright 2020 The go-ethereum Authors -// (original work) -// Copyright 2024 The Erigon Authors -// (modifications) -// This file is part of Erigon. +// This file is part of the go-ethereum library. // -// Erigon is free software: you can redistribute it and/or modify +// The go-ethereum library is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // -// Erigon is distributed in the hope that it will be useful, +// The go-ethereum library is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Lesser General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License -// along with Erigon. If not, see . +// along with the go-ethereum library. If not, see . package v5wire @@ -25,10 +22,9 @@ import ( "encoding/binary" "time" - "github.com/hashicorp/golang-lru/v2/simplelru" - - "github.com/erigontech/erigon/common/crypto" + "github.com/erigontech/erigon/common/lru" "github.com/erigontech/erigon/common/mclock" + "github.com/erigontech/erigon/common/crypto" "github.com/erigontech/erigon/p2p/enode" ) @@ -37,7 +33,7 @@ const handshakeTimeout = time.Second // The SessionCache keeps negotiated encryption keys and // state for in-progress handshakes in the Discovery v5 wire protocol. type SessionCache struct { - sessions *simplelru.LRU[sessionID, *session] + sessions lru.BasicLRU[sessionID, *session] handshakes map[sessionID]*Whoareyou clock mclock.Clock @@ -58,20 +54,17 @@ type session struct { writeKey []byte readKey []byte nonceCounter uint32 + node *enode.Node } // keysFlipped returns a copy of s with the read and write keys flipped. func (s *session) keysFlipped() *session { - return &session{s.readKey, s.writeKey, s.nonceCounter} + return &session{s.readKey, s.writeKey, s.nonceCounter, s.node} } func NewSessionCache(maxItems int, clock mclock.Clock) *SessionCache { - cache, err := simplelru.NewLRU[sessionID, *session](maxItems, nil) - if err != nil { - panic("can't create session cache") - } return &SessionCache{ - sessions: cache, + sessions: lru.NewBasicLRU[sessionID, *session](maxItems), handshakes: make(map[sessionID]*Whoareyou), clock: clock, nonceGen: generateNonce, @@ -99,10 +92,7 @@ func (sc *SessionCache) nextNonce(s *session) (Nonce, error) { // session returns the current session for the given node, if any. func (sc *SessionCache) session(id enode.ID, addr string) *session { - item, ok := sc.sessions.Get(sessionID{id, addr}) - if !ok { - return nil - } + item, _ := sc.sessions.Get(sessionID{id, addr}) return item } @@ -114,8 +104,19 @@ func (sc *SessionCache) readKey(id enode.ID, addr string) []byte { return nil } +func (sc *SessionCache) readNode(id enode.ID, addr string) *enode.Node { + if s := sc.session(id, addr); s != nil { + return s.node + } + return nil +} + // storeNewSession stores new encryption keys in the cache. -func (sc *SessionCache) storeNewSession(id enode.ID, addr string, s *session) { +func (sc *SessionCache) storeNewSession(id enode.ID, addr string, s *session, n *enode.Node) { + if n == nil { + panic("nil node in storeNewSession") + } + s.node = n sc.sessions.Add(sessionID{id, addr}, s) } diff --git a/p2p/enode/idscheme.go b/p2p/enode/idscheme.go index 4b3a183493a..21d0e181dae 100644 --- a/p2p/enode/idscheme.go +++ b/p2p/enode/idscheme.go @@ -1,6 +1,6 @@ // Copyright 2018 The go-ethereum Authors // (original work) -// Copyright 2024 The Erigon Authors +// Copyright 2026 The Erigon Authors // (modifications) // This file is part of Erigon. // @@ -27,16 +27,17 @@ import ( "golang.org/x/crypto/sha3" "github.com/erigontech/erigon/common/crypto" + "github.com/erigontech/erigon/common/math" "github.com/erigontech/erigon/execution/rlp" - "github.com/erigontech/erigon/p2p/discover/v4wire" "github.com/erigontech/erigon/p2p/enr" ) -// List of known secure identity schemes. +// ValidSchemes is a List of known secure identity schemes. var ValidSchemes = enr.SchemeMap{ "v4": V4ID{}, } +// ValidSchemesForTesting is a List of identity schemes for testing. var ValidSchemesForTesting = enr.SchemeMap{ "v4": V4ID{}, "null": NullID{}, @@ -87,20 +88,10 @@ func (V4ID) NodeAddr(r *enr.Record) []byte { if err != nil { return nil } - id := PubkeyToIDV4((*ecdsa.PublicKey)(&pubkey)) - return id[:] -} - -// PubkeyToIDV4 derives the v4 node address from the given public key. -func PubkeyToIDV4(key *ecdsa.PublicKey) ID { - return PubkeyEncoded(v4wire.EncodePubkey(key)).ID() -} - -type PubkeyEncoded v4wire.Pubkey - -// ID returns the node ID corresponding to the public key. -func (e PubkeyEncoded) ID() ID { - return ID(crypto.Keccak256Hash(e[:])) + buf := make([]byte, 64) + math.ReadBits(pubkey.X, buf[:32]) + math.ReadBits(pubkey.Y, buf[32:]) + return crypto.Keccak256(buf) } // Secp256k1 is the "secp256k1" key, which holds a public key. diff --git a/p2p/enode/localnode.go b/p2p/enode/localnode.go index 5c1bc6a4a9e..f673b1775c5 100644 --- a/p2p/enode/localnode.go +++ b/p2p/enode/localnode.go @@ -23,8 +23,8 @@ import ( "crypto/ecdsa" "fmt" "net" + "net/netip" "reflect" - "strconv" "sync" "sync/atomic" "time" @@ -39,34 +39,38 @@ const ( iptrackMinStatements = 10 iptrackWindow = 5 * time.Minute iptrackContactWindow = 10 * time.Minute + + // time needed to wait between two updates to the local ENR + recordUpdateThrottle = time.Millisecond ) // LocalNode produces the signed node record of a local node, i.e. a node run in the // current process. Setting ENR entries via the Set method updates the record. A new version // of the record is signed on demand when the Node method is called. type LocalNode struct { - cur atomic.Pointer[Node] // holds a non-nil node pointer while the record is up-to-date. + cur atomic.Pointer[Node] // holds a non-nil node pointer while the record is up-to-date + id ID key *ecdsa.PrivateKey db *DB // everything below is protected by a lock - mu sync.Mutex + mu sync.RWMutex seq uint64 + update time.Time // timestamp when the record was last updated entries map[string]enr.Entry endpoint4 lnEndpoint endpoint6 lnEndpoint - logger log.Logger } type lnEndpoint struct { track *netutil.IPTracker staticIP, fallbackIP net.IP - fallbackUDP int + fallbackUDP uint16 // port } // NewLocalNode creates a local node. -func NewLocalNode(db *DB, key *ecdsa.PrivateKey, logger log.Logger) *LocalNode { +func NewLocalNode(db *DB, key *ecdsa.PrivateKey) *LocalNode { ln := &LocalNode{ id: PubkeyToIDV4(&key.PublicKey), db: db, @@ -78,10 +82,10 @@ func NewLocalNode(db *DB, key *ecdsa.PrivateKey, logger log.Logger) *LocalNode { endpoint6: lnEndpoint{ track: netutil.NewIPTracker(iptrackWindow, iptrackContactWindow, iptrackMinStatements), }, - logger: logger, } ln.seq = db.localSeq(ln.id) - ln.invalidate() + ln.update = time.Now() + ln.cur.Store(nil) return ln } @@ -92,14 +96,34 @@ func (ln *LocalNode) Database() *DB { // Node returns the current version of the local node record. func (ln *LocalNode) Node() *Node { + // If we have a valid record, return that n := ln.cur.Load() if n != nil { return n } + // Record was invalidated, sign a new copy. ln.mu.Lock() defer ln.mu.Unlock() + + // Double check the current record, since multiple goroutines might be waiting + // on the write mutex. + if n = ln.cur.Load(); n != nil { + return n + } + + // The initial sequence number is the current timestamp in milliseconds. To ensure + // that the initial sequence number will always be higher than any previous sequence + // number (assuming the clock is correct), we want to avoid updating the record faster + // than once per ms. So we need to sleep here until the next possible update time has + // arrived. + lastChange := time.Since(ln.update) + if lastChange < recordUpdateThrottle { + time.Sleep(recordUpdateThrottle - lastChange) + } + ln.sign() + ln.update = time.Now() return ln.cur.Load() } @@ -119,6 +143,10 @@ func (ln *LocalNode) ID() ID { // Set puts the given entry into the local record, overwriting any existing value. // Use Set*IP and SetFallbackUDP to set IP addresses and UDP port, otherwise they'll // be overwritten by the endpoint predictor. +// +// Since node record updates are throttled to one per second, Set is asynchronous. +// Any update will be queued up and published when at least one second passes from +// the last change. func (ln *LocalNode) Set(e enr.Entry) { ln.mu.Lock() defer ln.mu.Unlock() @@ -150,8 +178,8 @@ func (ln *LocalNode) delete(e enr.Entry) { } } -func (ln *LocalNode) endpointForIP(ip net.IP) *lnEndpoint { - if ip.To4() != nil { +func (ln *LocalNode) endpointForIP(ip netip.Addr) *lnEndpoint { + if ip.Is4() { return &ln.endpoint4 } return &ln.endpoint6 @@ -163,7 +191,7 @@ func (ln *LocalNode) SetStaticIP(ip net.IP) { ln.mu.Lock() defer ln.mu.Unlock() - ln.endpointForIP(ip).staticIP = ip + ln.endpointForIP(netutil.IPToAddr(ip)).staticIP = ip ln.updateEndpoints() } @@ -173,7 +201,7 @@ func (ln *LocalNode) SetFallbackIP(ip net.IP) { ln.mu.Lock() defer ln.mu.Unlock() - ln.endpointForIP(ip).fallbackIP = ip + ln.endpointForIP(netutil.IPToAddr(ip)).fallbackIP = ip ln.updateEndpoints() } @@ -183,28 +211,28 @@ func (ln *LocalNode) SetFallbackUDP(port int) { ln.mu.Lock() defer ln.mu.Unlock() - ln.endpoint4.fallbackUDP = port - ln.endpoint6.fallbackUDP = port + ln.endpoint4.fallbackUDP = uint16(port) + ln.endpoint6.fallbackUDP = uint16(port) ln.updateEndpoints() } // UDPEndpointStatement should be called whenever a statement about the local node's // UDP endpoint is received. It feeds the local endpoint predictor. -func (ln *LocalNode) UDPEndpointStatement(fromaddr, endpoint *net.UDPAddr) { +func (ln *LocalNode) UDPEndpointStatement(fromaddr, endpoint netip.AddrPort) { ln.mu.Lock() defer ln.mu.Unlock() - ln.endpointForIP(endpoint.IP).track.AddStatement(fromaddr.String(), endpoint.String()) + ln.endpointForIP(endpoint.Addr()).track.AddStatement(fromaddr.Addr(), endpoint) ln.updateEndpoints() } // UDPContact should be called whenever the local node has announced itself to another node // via UDP. It feeds the local endpoint predictor. -func (ln *LocalNode) UDPContact(toaddr *net.UDPAddr) { +func (ln *LocalNode) UDPContact(toaddr netip.AddrPort) { ln.mu.Lock() defer ln.mu.Unlock() - ln.endpointForIP(toaddr.IP).track.AddContact(toaddr.String()) + ln.endpointForIP(toaddr.Addr()).track.AddContact(toaddr.Addr()) ln.updateEndpoints() } @@ -236,33 +264,20 @@ func (ln *LocalNode) updateEndpoints() { } // get returns the endpoint with highest precedence. -func (e *lnEndpoint) get() (newIP net.IP, newPort int) { +func (e *lnEndpoint) get() (newIP net.IP, newPort uint16) { newPort = e.fallbackUDP if e.fallbackIP != nil { newIP = e.fallbackIP } if e.staticIP != nil { newIP = e.staticIP - } else if ip, port := predictAddr(e.track); ip != nil { - newIP = ip - newPort = port + } else if ap := e.track.PredictEndpoint(); ap.IsValid() { + newIP = ap.Addr().AsSlice() + newPort = ap.Port() } return newIP, newPort } -// predictAddr wraps IPTracker.PredictEndpoint, converting from its string-based -// endpoint representation to IP and port types. -func predictAddr(t *netutil.IPTracker) (net.IP, int) { - ep := t.PredictEndpoint() - if ep == "" { - return nil, 0 - } - ipString, portString, _ := net.SplitHostPort(ep) - ip := net.ParseIP(ipString) - port, _ := strconv.Atoi(portString) - return ip, port -} - func (ln *LocalNode) invalidate() { ln.cur.Store(nil) } @@ -279,14 +294,14 @@ func (ln *LocalNode) sign() { ln.bumpSeq() r.SetSeq(ln.seq) if err := SignV4(&r, ln.key); err != nil { - panic(fmt.Errorf("enode: can't sign record: %w", err)) + panic(fmt.Errorf("enode: can't sign record: %v", err)) } n, err := New(ValidSchemes, &r) if err != nil { - panic(fmt.Errorf("enode: can't verify local record: %w", err)) + panic(fmt.Errorf("enode: can't verify local record: %v", err)) } ln.cur.Store(n) - ln.logger.Trace("New local node record", "seq", ln.seq, "id", n.ID(), "ip", n.IP(), "udp", n.UDP(), "tcp", n.TCP()) + log.Info("New local node record", "seq", ln.seq, "id", n.ID(), "ip", n.IPAddr(), "udp", n.UDP(), "tcp", n.TCP()) } func (ln *LocalNode) bumpSeq() { diff --git a/p2p/enode/node.go b/p2p/enode/node.go index e095ba03a7a..72de555633b 100644 --- a/p2p/enode/node.go +++ b/p2p/enode/node.go @@ -22,11 +22,13 @@ package enode import ( "crypto/ecdsa" "encoding/base64" + "encoding/binary" "encoding/hex" "errors" "fmt" "math/bits" "net" + "net/netip" "strings" "github.com/erigontech/erigon/execution/rlp" @@ -39,6 +41,14 @@ var errMissingPrefix = errors.New("missing 'enr:' prefix for base64-encoded reco type Node struct { r enr.Record id ID + + // hostname tracks the DNS name of the node. + hostname string + + // endpoint information + ip netip.Addr + udp uint16 + tcp uint16 } // New wraps a node record. The record must be valid according to the given @@ -47,11 +57,82 @@ func New(validSchemes enr.IdentityScheme, r *enr.Record) (*Node, error) { if err := r.VerifySignature(validSchemes); err != nil { return nil, err } - node := &Node{r: *r} - if n := copy(node.id[:], validSchemes.NodeAddr(&node.r)); n != len(ID{}) { - return nil, fmt.Errorf("invalid node ID length %d, need %d", n, len(ID{})) + var id ID + if n := copy(id[:], validSchemes.NodeAddr(r)); n != len(id) { + return nil, fmt.Errorf("invalid node ID length %d, need %d", n, len(id)) + } + return newNodeWithID(r, id), nil +} + +func newNodeWithID(r *enr.Record, id ID) *Node { + n := &Node{r: *r, id: id} + // Set the preferred endpoint. + // Here we decide between IPv4 and IPv6, choosing the 'most global' address. + var ip4 netip.Addr + var ip6 netip.Addr + n.Load((*enr.IPv4Addr)(&ip4)) + n.Load((*enr.IPv6Addr)(&ip6)) + valid4 := validIP(ip4) + valid6 := validIP(ip6) + switch { + case valid4 && valid6: + if localityScore(ip4) >= localityScore(ip6) { + n.setIP4(ip4) + } else { + n.setIP6(ip6) + } + case valid4: + n.setIP4(ip4) + case valid6: + n.setIP6(ip6) + default: + n.setIPv4Ports() + } + return n +} + +// validIP reports whether 'ip' is a valid node endpoint IP address. +func validIP(ip netip.Addr) bool { + return ip.IsValid() && !ip.IsMulticast() +} + +func localityScore(ip netip.Addr) int { + switch { + case ip.IsUnspecified(): + return 0 + case ip.IsLoopback(): + return 1 + case ip.IsLinkLocalUnicast(): + return 2 + case ip.IsPrivate(): + return 3 + default: + return 4 + } +} + +func (n *Node) setIP4(ip netip.Addr) { + n.ip = ip + n.setIPv4Ports() +} + +func (n *Node) setIPv4Ports() { + n.Load((*enr.UDP)(&n.udp)) + n.Load((*enr.TCP)(&n.tcp)) +} + +func (n *Node) setIP6(ip netip.Addr) { + if ip.Is4In6() { + n.setIP4(ip) + return + } + n.ip = ip + if err := n.Load((*enr.UDP6)(&n.udp)); err != nil { + n.Load((*enr.UDP)(&n.udp)) + } + if err := n.Load((*enr.TCP6)(&n.tcp)); err != nil { + n.Load((*enr.TCP)(&n.tcp)) } - return node, nil } // MustParse parses a node record or enode:// URL. It panics if the input is invalid. @@ -82,21 +163,6 @@ func Parse(validSchemes enr.IdentityScheme, input string) (*Node, error) { return New(validSchemes, &r) } -func ParseNodesFromURLs(urls []string) ([]*Node, error) { - nodes := make([]*Node, 0, len(urls)) - for _, url := range urls { - if url == "" { - continue - } - n, err := Parse(ValidSchemes, url) - if err != nil { - return nil, fmt.Errorf("invalid node URL %s: %w", url, err) - } - nodes = append(nodes, n) - } - return nodes, nil -} - // ID returns the node identifier. func (n *Node) ID() ID { return n.id @@ -107,43 +173,71 @@ func (n *Node) Seq() uint64 { return n.r.Seq() } -// Incomplete returns true for nodes with no IP address. -func (n *Node) Incomplete() bool { - return n.IP() == nil -} - // Load retrieves an entry from the underlying record. func (n *Node) Load(k enr.Entry) error { return n.r.Load(k) } -// IP returns the IP address of the node. This prefers IPv4 addresses. +// IP returns the IP address of the node. func (n *Node) IP() net.IP { - var ( - ip4 enr.IPv4 - ip6 enr.IPv6 - ) - if n.Load(&ip4) == nil { - return net.IP(ip4) - } - if n.Load(&ip6) == nil { - return net.IP(ip6) - } - return nil + return net.IP(n.ip.AsSlice()) +} + +// IPAddr returns the IP address of the node. +func (n *Node) IPAddr() netip.Addr { + return n.ip } // UDP returns the UDP port of the node. func (n *Node) UDP() int { - var port enr.UDP - n.Load(&port) - return int(port) + return int(n.udp) } -// UDP returns the TCP port of the node. +// TCP returns the TCP port of the node. func (n *Node) TCP() int { - var port enr.TCP - n.Load(&port) - return int(port) + return int(n.tcp) +} + +// WithHostname adds a DNS hostname to the node. +func (n *Node) WithHostname(hostname string) *Node { + cpy := *n + cpy.hostname = hostname + return &cpy +} + +// Hostname returns the DNS name assigned by WithHostname. +func (n *Node) Hostname() string { + return n.hostname +} + +// UDPEndpoint returns the announced UDP endpoint. +func (n *Node) UDPEndpoint() (netip.AddrPort, bool) { + if !n.ip.IsValid() || n.ip.IsUnspecified() || n.udp == 0 { + return netip.AddrPort{}, false + } + return netip.AddrPortFrom(n.ip, n.udp), true +} + +// TCPEndpoint returns the announced TCP endpoint. +func (n *Node) TCPEndpoint() (netip.AddrPort, bool) { + if !n.ip.IsValid() || n.ip.IsUnspecified() || n.tcp == 0 { + return netip.AddrPort{}, false + } + return netip.AddrPortFrom(n.ip, n.tcp), true +} + +// QUICEndpoint returns the announced QUIC endpoint. +func (n *Node) QUICEndpoint() (netip.AddrPort, bool) { + var quic uint16 + if n.ip.Is4() || n.ip.Is4In6() { + n.Load((*enr.QUIC)(&quic)) + } else if n.ip.Is6() { + n.Load((*enr.QUIC6)(&quic)) + } + if !n.ip.IsValid() || n.ip.IsUnspecified() || quic == 0 { + return netip.AddrPort{}, false + } + return netip.AddrPortFrom(n.ip, quic), true } // Pubkey returns the secp256k1 public key of the node, if present. @@ -163,19 +257,17 @@ func (n *Node) Record() *enr.Record { } // ValidateComplete checks whether n has a valid IP and UDP port. -// // Deprecated: don't use this method. func (n *Node) ValidateComplete() error { - if n.Incomplete() { + if !n.ip.IsValid() { return errors.New("missing IP address") } - if n.UDP() == 0 { - return errors.New("missing UDP port") - } - ip := n.IP() - if ip.IsMulticast() || ip.IsUnspecified() { + if n.ip.IsMulticast() || n.ip.IsUnspecified() { return errors.New("invalid IP (multicast/unspecified)") } + if n.udp == 0 { + return errors.New("missing UDP port") + } // Validate the node key (on curve, etc.). var key Secp256k1 return n.Load(&key) @@ -215,10 +307,10 @@ func (n ID) Bytes() []byte { // ID prints as a long hexadecimal number. func (n ID) String() string { - return hex.EncodeToString(n[:]) + return fmt.Sprintf("%x", n[:]) } -// The Go syntax representation of a ID is a call to HexID. +// GoString returns the Go syntax representation of a ID is a call to HexID. func (n ID) GoString() string { return fmt.Sprintf("enode.HexID(\"%x\")", n[:]) } @@ -270,9 +362,10 @@ func ParseID(in string) (ID, error) { // Returns -1 if a is closer to target, 1 if b is closer to target // and 0 if they are equal. func DistCmp(target, a, b ID) int { - for i := range target { - da := a[i] ^ target[i] - db := b[i] ^ target[i] + for i := 0; i < len(target); i += 8 { + tn := binary.BigEndian.Uint64(target[i : i+8]) + da := tn ^ binary.BigEndian.Uint64(a[i:i+8]) + db := tn ^ binary.BigEndian.Uint64(b[i:i+8]) if da > db { return 1 } else if da < db { @@ -285,12 +378,14 @@ func DistCmp(target, a, b ID) int { // LogDist returns the logarithmic distance between a and b, log2(a ^ b). func LogDist(a, b ID) int { lz := 0 - for i := range a { - x := a[i] ^ b[i] + for i := 0; i < len(a); i += 8 { + ai := binary.BigEndian.Uint64(a[i : i+8]) + bi := binary.BigEndian.Uint64(b[i : i+8]) + x := ai ^ bi if x == 0 { - lz += 8 + lz += 64 } else { - lz += bits.LeadingZeros8(x) + lz += bits.LeadingZeros64(x) break } } diff --git a/p2p/enode/nodedb.go b/p2p/enode/nodedb.go index ef8c639af74..a2cab0b44c0 100644 --- a/p2p/enode/nodedb.go +++ b/p2p/enode/nodedb.go @@ -26,7 +26,7 @@ import ( "encoding/binary" "errors" "fmt" - "net" + "net/netip" "sync" "time" @@ -74,7 +74,7 @@ var ( errInvalidIP = errors.New("invalid IP") ) -var zeroIP = make(net.IP, 16) +var zeroIP = netip.IPv6Unspecified() // DB is the node database, storing previously seen nodes and any collected metadata about // them for QoS purposes. @@ -86,9 +86,14 @@ type DB struct { ctxCancel func() } +// Deprecated: Use OpenDBEx. Wraps OpenDBEx for compatibility with geth code. +func OpenDB(path string) (*DB, error) { + return OpenDBEx(context.Background(), path, "", log.Root()) +} + // OpenDB opens a node database for storing and retrieving infos about known peers in the // network. If no path is given an in-memory, temporary database is constructed. -func OpenDB(ctx context.Context, path string, tmpDir string, logger log.Logger) (*DB, error) { +func OpenDBEx(ctx context.Context, path string, tmpDir string, logger log.Logger) (*DB, error) { if path == "" { return newMemoryDB(ctx, logger, tmpDir) } @@ -200,39 +205,37 @@ func splitNodeKey(key []byte) (id ID, rest []byte) { } // nodeItemKey returns the database key for a node metadata field. -func nodeItemKey(id ID, ip net.IP, field string) []byte { - ip16 := ip.To16() - if ip16 == nil { - panic(fmt.Errorf("invalid IP (length %d)", len(ip))) +func nodeItemKey(id ID, ip netip.Addr, field string) []byte { + if !ip.IsValid() { + panic("invalid IP") } - return bytes.Join([][]byte{nodeKey(id), ip16, []byte(field)}, []byte{':'}) + ip16 := ip.As16() + return bytes.Join([][]byte{nodeKey(id), ip16[:], []byte(field)}, []byte{':'}) } // splitNodeItemKey returns the components of a key created by nodeItemKey. -func splitNodeItemKey(key []byte) (id ID, ip net.IP, field string) { +func splitNodeItemKey(key []byte) (id ID, ip netip.Addr, field string) { id, key = splitNodeKey(key) // Skip discover root. if string(key) == dbDiscoverRoot { - return id, nil, "" + return id, netip.Addr{}, "" } key = key[len(dbDiscoverRoot)+1:] // Split out the IP. - ip = key[:16] - if ip4 := ip.To4(); ip4 != nil { - ip = ip4 - } + ip, _ = netip.AddrFromSlice(key[:16]) key = key[16+1:] // Field is the remainder of key. field = string(key) return id, ip, field } -func v5Key(id ID, ip net.IP, field string) []byte { +func v5Key(id ID, ip netip.Addr, field string) []byte { + ip16 := ip.As16() return bytes.Join([][]byte{ []byte(dbNodePrefix), id[:], []byte(dbDiscv5Root), - ip.To16(), + ip16[:], []byte(field), }, []byte{':'}) } @@ -488,24 +491,24 @@ func (db *DB) expireNodes() { // LastPingReceived retrieves the time of the last ping packet received from // a remote node. -func (db *DB) LastPingReceived(id ID, ip net.IP) time.Time { - if ip = ip.To16(); ip == nil { +func (db *DB) LastPingReceived(id ID, ip netip.Addr) time.Time { + if !ip.IsValid() { return time.Time{} } return time.Unix(db.fetchInt64(nodeItemKey(id, ip, dbNodePing)), 0) } // UpdateLastPingReceived updates the last time we tried contacting a remote node. -func (db *DB) UpdateLastPingReceived(id ID, ip net.IP, instance time.Time) error { - if ip = ip.To16(); ip == nil { +func (db *DB) UpdateLastPingReceived(id ID, ip netip.Addr, instance time.Time) error { + if !ip.IsValid() { return errInvalidIP } return db.storeInt64(nodeItemKey(id, ip, dbNodePing), instance.Unix()) } // LastPongReceived retrieves the time of the last successful pong from remote node. -func (db *DB) LastPongReceived(id ID, ip net.IP) time.Time { - if ip = ip.To16(); ip == nil { +func (db *DB) LastPongReceived(id ID, ip netip.Addr) time.Time { + if !ip.IsValid() { return time.Time{} } // Launch expirer @@ -514,48 +517,53 @@ func (db *DB) LastPongReceived(id ID, ip net.IP) time.Time { } // UpdateLastPongReceived updates the last pong time of a node. -func (db *DB) UpdateLastPongReceived(id ID, ip net.IP, instance time.Time) error { - if ip = ip.To16(); ip == nil { +func (db *DB) UpdateLastPongReceived(id ID, ip netip.Addr, instance time.Time) error { + if !ip.IsValid() { return errInvalidIP } return db.storeInt64(nodeItemKey(id, ip, dbNodePong), instance.Unix()) } // FindFails retrieves the number of findnode failures since bonding. -func (db *DB) FindFails(id ID, ip net.IP) int { - if ip = ip.To16(); ip == nil { +func (db *DB) FindFails(id ID, ip netip.Addr) int { + if !ip.IsValid() { return 0 } return int(db.fetchInt64(nodeItemKey(id, ip, dbNodeFindFails))) } // UpdateFindFails updates the number of findnode failures since bonding. -func (db *DB) UpdateFindFails(id ID, ip net.IP, fails int) error { - if ip = ip.To16(); ip == nil { +func (db *DB) UpdateFindFails(id ID, ip netip.Addr, fails int) error { + if !ip.IsValid() { return errInvalidIP } return db.storeInt64(nodeItemKey(id, ip, dbNodeFindFails), int64(fails)) } // FindFailsV5 retrieves the discv5 findnode failure counter. -func (db *DB) FindFailsV5(id ID, ip net.IP) int { - if ip = ip.To16(); ip == nil { +func (db *DB) FindFailsV5(id ID, ip netip.Addr) int { + if !ip.IsValid() { return 0 } return int(db.fetchInt64(v5Key(id, ip, dbNodeFindFails))) } // UpdateFindFailsV5 stores the discv5 findnode failure counter. -func (db *DB) UpdateFindFailsV5(id ID, ip net.IP, fails int) error { - if ip = ip.To16(); ip == nil { +func (db *DB) UpdateFindFailsV5(id ID, ip netip.Addr, fails int) error { + if !ip.IsValid() { return errInvalidIP } return db.storeInt64(v5Key(id, ip, dbNodeFindFails), int64(fails)) } -// LocalSeq retrieves the local record sequence counter. +// localSeq retrieves the local record sequence counter, defaulting to the current +// timestamp if no previous exists. This ensures that wiping all data associated +// with a node (apart from its key) will not generate already used sequence nums. func (db *DB) localSeq(id ID) uint64 { - return db.fetchUint64(localItemKey(id, dbLocalSeq)) + if seq := db.fetchUint64(localItemKey(id, dbLocalSeq)); seq > 0 { + return seq + } + return uint64(time.Now().UnixMilli()) } // storeLocalSeq stores the local record sequence counter. @@ -601,7 +609,7 @@ func (db *DB) QuerySeeds(n int, maxAge time.Duration) []*Node { continue // iterator exhausted } db.ensureExpirer() - pongKey := nodeItemKey(n.ID(), n.IP(), dbNodePong) + pongKey := nodeItemKey(n.ID(), n.IPAddr(), dbNodePong) var lastPongReceived int64 blob, errGet := tx.GetOne(kv.Inodes, pongKey) if errGet != nil { diff --git a/p2p/enode/nodedb_test.go b/p2p/enode/nodedb_test.go index 7a57da80d5d..07a5219097c 100644 --- a/p2p/enode/nodedb_test.go +++ b/p2p/enode/nodedb_test.go @@ -215,7 +215,7 @@ var nodeDBInt64Tests = []struct { func TestDBInt64(t *testing.T) { tmpDir := t.TempDir() - db, err := OpenDB(context.Background(), "", tmpDir, log.Root()) + db, err := OpenDB("") if err != nil { panic(err) } @@ -251,7 +251,7 @@ func TestDBFetchStore(t *testing.T) { inst := time.Now() num := 314 - db, err := OpenDB(context.Background(), "", tmpDir, log.Root()) + db, err := OpenDB("") if err != nil { panic(err) } @@ -394,7 +394,7 @@ func TestDBSeedQuery(t *testing.T) { } func testSeedQuery(tmpDir string) error { - db, err := OpenDB(context.Background(), "", tmpDir, log.Root()) + db, err := OpenDB("") if err != nil { panic(err) } @@ -558,7 +558,7 @@ var nodeDBExpirationNodes = []struct { func TestDBExpiration(t *testing.T) { tmpDir := t.TempDir() - db, err := OpenDB(context.Background(), "", tmpDir, log.Root()) + db, err := OpenDB("") if err != nil { panic(err) } @@ -605,7 +605,7 @@ func TestDBExpiration(t *testing.T) { // in the database. func TestDBExpireV5(t *testing.T) { tmpDir := t.TempDir() - db, err := OpenDB(context.Background(), "", tmpDir, log.Root()) + db, err := OpenDB("") if err != nil { panic(err) } diff --git a/p2p/enode/urlv4.go b/p2p/enode/urlv4.go index e10c69a9ec6..031bd9b6392 100644 --- a/p2p/enode/urlv4.go +++ b/p2p/enode/urlv4.go @@ -1,6 +1,6 @@ // Copyright 2018 The go-ethereum Authors // (original work) -// Copyright 2024 The Erigon Authors +// Copyright 2026 The Erigon Authors // (modifications) // This file is part of Erigon. // @@ -30,6 +30,7 @@ import ( "strconv" "github.com/erigontech/erigon/common/crypto" + "github.com/erigontech/erigon/common/math" "github.com/erigontech/erigon/p2p/enr" ) @@ -128,20 +129,9 @@ func parseComplete(rawurl string) (*Node, error) { if id, err = parsePubkey(u.User.String()); err != nil { return nil, fmt.Errorf("invalid public key (%w)", err) } - // Parse the IP address. + + // Parse the IP and ports. ip := net.ParseIP(u.Hostname()) - if ip == nil { - ips, err := lookupIPFunc(u.Hostname()) - if err != nil { - return nil, err - } - ip = ips[0] - } - // Ensure the IP is 4 bytes long for IPv4 addresses. - if ipv4 := ip.To4(); ipv4 != nil { - ip = ipv4 - } - // Parse the port numbers. if tcpPort, err = strconv.ParseUint(u.Port(), 10, 16); err != nil { return nil, errors.New("invalid port") } @@ -153,7 +143,13 @@ func parseComplete(rawurl string) (*Node, error) { return nil, errors.New("invalid discport in query") } } - return NewV4(id, ip, int(tcpPort), int(udpPort)), nil + + // Create the node. + node := NewV4(id, ip, int(tcpPort), int(udpPort)) + if ip == nil && u.Hostname() != "" { + node = node.WithHostname(u.Hostname()) + } + return node, nil } // parsePubkey parses a hex-encoded secp256k1 public key. @@ -164,6 +160,7 @@ func parsePubkey(in string) (*ecdsa.PublicKey, error) { } else if len(b) != 64 { return nil, fmt.Errorf("wrong length, want %d hex chars", 128) } + b = append([]byte{0x4}, b...) return crypto.UnmarshalPubkey(b) } @@ -177,20 +174,36 @@ func (n *Node) URLv4() string { n.Load((*Secp256k1)(&key)) switch { case scheme == "v4" || key != ecdsa.PublicKey{}: - nodeid = hex.EncodeToString(crypto.MarshalPubkey(&key)) + nodeid = fmt.Sprintf("%x", crypto.FromECDSAPub(&key)[1:]) default: nodeid = fmt.Sprintf("%s.%x", scheme, n.id[:]) } u := url.URL{Scheme: "enode"} - if n.Incomplete() { - u.Host = nodeid - } else { + if n.Hostname() != "" { + // For nodes with a DNS name: include DNS name, TCP port, and optional UDP port + u.User = url.User(nodeid) + u.Host = fmt.Sprintf("%s:%d", n.Hostname(), n.TCP()) + if n.UDP() != n.TCP() { + u.RawQuery = "discport=" + strconv.Itoa(n.UDP()) + } + } else if n.ip.IsValid() { + // For IP-based nodes: include IP address, TCP port, and optional UDP port addr := net.TCPAddr{IP: n.IP(), Port: n.TCP()} u.User = url.User(nodeid) u.Host = addr.String() if n.UDP() != n.TCP() { u.RawQuery = "discport=" + strconv.Itoa(n.UDP()) } + } else { + u.Host = nodeid } return u.String() } + +// PubkeyToIDV4 derives the v4 node address from the given public key. +func PubkeyToIDV4(key *ecdsa.PublicKey) ID { + e := make([]byte, 64) + math.ReadBits(key.X, e[:len(e)/2]) + math.ReadBits(key.Y, e[len(e)/2:]) + return ID(crypto.Keccak256Hash(e)) +} diff --git a/p2p/enr/enr.go b/p2p/enr/enr.go index 50b2ca4ef75..42e8a22adfb 100644 --- a/p2p/enr/enr.go +++ b/p2p/enr/enr.go @@ -53,6 +53,7 @@ var ( errNotSorted = errors.New("record key/value pairs are not sorted by key") errDuplicateKey = errors.New("record contains duplicate key") errIncompletePair = errors.New("record contains incomplete k/v pair") + errIncompleteList = errors.New("record contains less than two list elements") errTooBig = fmt.Errorf("record bigger than %d bytes", SizeLimit) errEncodeUnsigned = errors.New("can't encode unsigned record") errNotFound = errors.New("no such key in record") @@ -98,6 +99,24 @@ type pair struct { v rlp.RawValue } +// Size returns the encoded size of the record. +func (r *Record) Size() uint64 { + if r.raw != nil { + return uint64(len(r.raw)) + } + return computeSize(r) +} + +func computeSize(r *Record) uint64 { + size := uint64(rlp.IntSize(r.seq)) + size += rlp.BytesSize(r.signature) + for _, p := range r.pairs { + size += rlp.StringSize(p.k) + size += uint64(len(p.v)) + } + return rlp.ListSize(size) +} + // Seq returns the sequence number. func (r *Record) Seq() uint64 { return r.seq @@ -134,7 +153,7 @@ func (r *Record) Load(e Entry) error { func (r *Record) Set(e Entry) { blob, err := rlp.EncodeToBytes(e) if err != nil { - panic(fmt.Errorf("enr: can't encode %s: %w", e.ENRKey(), err)) + panic(fmt.Errorf("enr: can't encode %s: %v", e.ENRKey(), err)) } r.invalidate() @@ -208,27 +227,33 @@ func decodeRecord(s *rlp.Stream) (dec Record, raw []byte, err error) { // Decode the RLP container. s = rlp.NewStream(bytes.NewReader(raw), 0) - if _, e := s.List(); e != nil { - return dec, raw, e + if _, err := s.List(); err != nil { + return dec, raw, err } - if e := s.Decode(&dec.signature); e != nil { - return dec, raw, e + if err = s.Decode(&dec.signature); err != nil { + if err == rlp.EOL { + err = errIncompleteList + } + return dec, raw, err } - if e := s.Decode(&dec.seq); e != nil { - return dec, raw, e + if err = s.Decode(&dec.seq); err != nil { + if err == rlp.EOL { + err = errIncompleteList + } + return dec, raw, err } // The rest of the record contains sorted k/v pairs. var prevkey string for i := 0; ; i++ { var kv pair if err := s.Decode(&kv.k); err != nil { - if errors.Is(err, rlp.EOL) { + if err == rlp.EOL { break } return dec, raw, err } if err := s.Decode(&kv.v); err != nil { - if errors.Is(err, rlp.EOL) { + if err == rlp.EOL { return dec, raw, errIncompletePair } return dec, raw, err @@ -300,7 +325,7 @@ func (r *Record) AppendElements(list []interface{}) []interface{} { } func (r *Record) encode(sig []byte) (raw []byte, err error) { - list := make([]interface{}, 1, 2*len(r.pairs)+1) + list := make([]interface{}, 1, 2*len(r.pairs)+2) list[0] = sig list = r.AppendElements(list) if raw, err = rlp.EncodeToBytes(list); err != nil { diff --git a/p2p/enr/enr_test.go b/p2p/enr/enr_test.go index 0a5ee60c4ee..afe78606a4c 100644 --- a/p2p/enr/enr_test.go +++ b/p2p/enr/enr_test.go @@ -143,7 +143,7 @@ func TestSortedGetAndSet(t *testing.T) { for i, w := range tt.want { // set got's key from r.pair[i], so that we preserve order of pairs got := pair{k: r.pairs[i].k} - require.NoError(t, r.Load(WithEntry(w.k, &got.v))) + assert.NoError(t, r.Load(WithEntry(w.k, &got.v))) assert.Equal(t, w, got) } } @@ -162,7 +162,7 @@ func TestDirty(t *testing.T) { t.Error("record is not signed") } _, err := rlp.EncodeToBytes(r) - require.NoError(t, err) + assert.NoError(t, err) r.SetSeq(3) if len(r.signature) != 0 { @@ -173,6 +173,32 @@ func TestDirty(t *testing.T) { } } +func TestSize(t *testing.T) { + var r Record + + // Empty record size is 3 bytes. + // Unsigned records cannot be encoded, but they could, the encoding + // would be [ 0, 0 ] -> 0xC28080. + assert.Equal(t, uint64(3), r.Size()) + + // Add one attribute. The size increases to 5, the encoding + // would be [ 0, 0, "k", "v" ] -> 0xC58080C26B76. + r.Set(WithEntry("k", "v")) + assert.Equal(t, uint64(5), r.Size()) + + // Now add a signature. + nodeid := []byte{1, 2, 3, 4, 5, 6, 7, 8} + signTest(nodeid, &r) + assert.Equal(t, uint64(45), r.Size()) + enc, _ := rlp.EncodeToBytes(&r) + if r.Size() != uint64(len(enc)) { + t.Error("Size() not equal encoded length", len(enc)) + } + if r.Size() != computeSize(&r) { + t.Error("Size() not equal computed size", computeSize(&r)) + } +} + func TestSeq(t *testing.T) { var r Record @@ -235,6 +261,29 @@ func TestRecordTooBig(t *testing.T) { require.NoError(t, signTest([]byte{5}, &r)) } +// This checks that incomplete RLP inputs are handled correctly. +func TestDecodeIncomplete(t *testing.T) { + type decTest struct { + input []byte + err error + } + tests := []decTest{ + {[]byte{0xC0}, errIncompleteList}, + {[]byte{0xC1, 0x1}, errIncompleteList}, + {[]byte{0xC2, 0x1, 0x2}, nil}, + {[]byte{0xC3, 0x1, 0x2, 0x3}, errIncompletePair}, + {[]byte{0xC4, 0x1, 0x2, 0x3, 0x4}, nil}, + {[]byte{0xC5, 0x1, 0x2, 0x3, 0x4, 0x5}, errIncompletePair}, + } + for _, test := range tests { + var r Record + err := rlp.DecodeBytes(test.input, &r) + if err != test.err { + t.Errorf("wrong error for %X: %v", test.input, err) + } + } +} + // TestSignEncodeAndDecodeRandom tests encoding/decoding of records containing random key/value pairs. func TestSignEncodeAndDecodeRandom(t *testing.T) { var r Record @@ -249,8 +298,11 @@ func TestSignEncodeAndDecodeRandom(t *testing.T) { } require.NoError(t, signTest([]byte{5}, &r)) - _, err := rlp.EncodeToBytes(r) + + enc, err := rlp.EncodeToBytes(r) require.NoError(t, err) + require.Equal(t, uint64(len(enc)), r.Size()) + require.Equal(t, uint64(len(enc)), computeSize(&r)) for k, v := range pairs { desc := fmt.Sprintf("key %q", k) diff --git a/p2p/enr/entries.go b/p2p/enr/entries.go index cb9c4206a5f..03aa2eea416 100644 --- a/p2p/enr/entries.go +++ b/p2p/enr/entries.go @@ -20,9 +20,11 @@ package enr import ( + "errors" "fmt" "io" "net" + "net/netip" "github.com/erigontech/erigon/execution/rlp" ) @@ -63,7 +65,7 @@ type TCP uint16 func (v TCP) ENRKey() string { return "tcp" } -// UDP is the "udp" key, which holds the IPv6-specific UDP port of the node. +// TCP6 is the "tcp6" key, which holds the IPv6-specific tcp6 port of the node. type TCP6 uint16 func (v TCP6) ENRKey() string { return "tcp6" } @@ -73,11 +75,21 @@ type UDP uint16 func (v UDP) ENRKey() string { return "udp" } -// UDP is the "udp" key, which holds the IPv6-specific UDP port of the node. +// UDP6 is the "udp6" key, which holds the IPv6-specific UDP port of the node. type UDP6 uint16 func (v UDP6) ENRKey() string { return "udp6" } +// QUIC is the "quic" key, which holds the QUIC port of the node. +type QUIC uint16 + +func (v QUIC) ENRKey() string { return "quic" } + +// QUIC6 is the "quic6" key, which holds the IPv6-specific quic6 port of the node. +type QUIC6 uint16 + +func (v QUIC6) ENRKey() string { return "quic6" } + // ID is the "id" key, which holds the name of the identity scheme. type ID string @@ -169,6 +181,60 @@ func (v *IPv6) DecodeRLP(s *rlp.Stream) error { return nil } +// IPv4Addr is the "ip" key, which holds the IP address of the node. +type IPv4Addr netip.Addr + +func (v IPv4Addr) ENRKey() string { return "ip" } + +// EncodeRLP implements rlp.Encoder. +func (v IPv4Addr) EncodeRLP(w io.Writer) error { + addr := netip.Addr(v) + if !addr.Is4() { + return errors.New("address is not IPv4") + } + enc := rlp.NewEncoderBuffer(w) + bytes := addr.As4() + enc.WriteBytes(bytes[:]) + return enc.Flush() +} + +// DecodeRLP implements rlp.Decoder. +func (v *IPv4Addr) DecodeRLP(s *rlp.Stream) error { + var bytes [4]byte + if err := s.ReadBytes(bytes[:]); err != nil { + return err + } + *v = IPv4Addr(netip.AddrFrom4(bytes)) + return nil +} + +// IPv6Addr is the "ip6" key, which holds the IP address of the node. +type IPv6Addr netip.Addr + +func (v IPv6Addr) ENRKey() string { return "ip6" } + +// EncodeRLP implements rlp.Encoder. +func (v IPv6Addr) EncodeRLP(w io.Writer) error { + addr := netip.Addr(v) + if !addr.Is6() { + return errors.New("address is not IPv6") + } + enc := rlp.NewEncoderBuffer(w) + bytes := addr.As16() + enc.WriteBytes(bytes[:]) + return enc.Flush() +} + +// DecodeRLP implements rlp.Decoder. +func (v *IPv6Addr) DecodeRLP(s *rlp.Stream) error { + var bytes [16]byte + if err := s.ReadBytes(bytes[:]); err != nil { + return err + } + *v = IPv6Addr(netip.AddrFrom16(bytes)) + return nil +} + // KeyError is an error related to a key. type KeyError struct { Key string @@ -183,9 +249,16 @@ func (err *KeyError) Error() string { return fmt.Sprintf("ENR key %q: %v", err.Key, err.Err) } +func (err *KeyError) Unwrap() error { + return err.Err +} + // IsNotFound reports whether the given error means that a key/value pair is // missing from a record. func IsNotFound(err error) bool { - kerr, ok := err.(*KeyError) - return ok && kerr.Err == errNotFound + var ke *KeyError + if errors.As(err, &ke) { + return ke.Err == errNotFound + } + return false } diff --git a/p2p/event/feedof.go b/p2p/event/feedof.go new file mode 100644 index 00000000000..4a24e37f125 --- /dev/null +++ b/p2p/event/feedof.go @@ -0,0 +1,164 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package event + +import ( + "reflect" + "sync" +) + +// FeedOf implements one-to-many subscriptions where the carrier of events is a channel. +// Values sent to a Feed are delivered to all subscribed channels simultaneously. +// +// The zero value is ready to use. +type FeedOf[T any] struct { + once sync.Once // ensures that init only runs once + sendLock chan struct{} // sendLock has a one-element buffer and is empty when held.It protects sendCases. + removeSub chan chan<- T // interrupts Send + sendCases caseList // the active set of select cases used by Send + + // The inbox holds newly subscribed channels until they are added to sendCases. + mu sync.Mutex + inbox caseList +} + +func (f *FeedOf[T]) init() { + f.removeSub = make(chan chan<- T) + f.sendLock = make(chan struct{}, 1) + f.sendLock <- struct{}{} + f.sendCases = caseList{{Chan: reflect.ValueOf(f.removeSub), Dir: reflect.SelectRecv}} +} + +// Subscribe adds a channel to the feed. Future sends will be delivered on the channel +// until the subscription is canceled. +// +// The channel should have ample buffer space to avoid blocking other subscribers. Slow +// subscribers are not dropped. +func (f *FeedOf[T]) Subscribe(channel chan<- T) Subscription { + f.once.Do(f.init) + + chanval := reflect.ValueOf(channel) + sub := &feedOfSub[T]{feed: f, channel: channel, err: make(chan error, 1)} + + // Add the select case to the inbox. + // The next Send will add it to f.sendCases. + f.mu.Lock() + defer f.mu.Unlock() + cas := reflect.SelectCase{Dir: reflect.SelectSend, Chan: chanval} + f.inbox = append(f.inbox, cas) + return sub +} + +func (f *FeedOf[T]) remove(sub *feedOfSub[T]) { + // Delete from inbox first, which covers channels + // that have not been added to f.sendCases yet. + f.mu.Lock() + index := f.inbox.find(sub.channel) + if index != -1 { + f.inbox = f.inbox.delete(index) + f.mu.Unlock() + return + } + f.mu.Unlock() + + select { + case f.removeSub <- sub.channel: + // Send will remove the channel from f.sendCases. + case <-f.sendLock: + // No Send is in progress, delete the channel now that we have the send lock. + f.sendCases = f.sendCases.delete(f.sendCases.find(sub.channel)) + f.sendLock <- struct{}{} + } +} + +// Send delivers to all subscribed channels simultaneously. +// It returns the number of subscribers that the value was sent to. +func (f *FeedOf[T]) Send(value T) (nsent int) { + rvalue := reflect.ValueOf(value) + + f.once.Do(f.init) + <-f.sendLock + + // Add new cases from the inbox after taking the send lock. + f.mu.Lock() + f.sendCases = append(f.sendCases, f.inbox...) + f.inbox = nil + f.mu.Unlock() + + // Set the sent value on all channels. + for i := firstSubSendCase; i < len(f.sendCases); i++ { + f.sendCases[i].Send = rvalue + } + + // Send until all channels except removeSub have been chosen. 'cases' tracks a prefix + // of sendCases. When a send succeeds, the corresponding case moves to the end of + // 'cases' and it shrinks by one element. + cases := f.sendCases + for { + // Fast path: try sending without blocking before adding to the select set. + // This should usually succeed if subscribers are fast enough and have free + // buffer space. + for i := firstSubSendCase; i < len(cases); i++ { + if cases[i].Chan.TrySend(rvalue) { + nsent++ + cases = cases.deactivate(i) + i-- + } + } + if len(cases) == firstSubSendCase { + break + } + // Select on all the receivers, waiting for them to unblock. + chosen, recv, _ := reflect.Select(cases) + if chosen == 0 /* <-f.removeSub */ { + index := f.sendCases.find(recv.Interface()) + f.sendCases = f.sendCases.delete(index) + if index >= 0 && index < len(cases) { + // Shrink 'cases' too because the removed case was still active. + cases = f.sendCases[:len(cases)-1] + } + } else { + cases = cases.deactivate(chosen) + nsent++ + } + } + + // Forget about the sent value and hand off the send lock. + for i := firstSubSendCase; i < len(f.sendCases); i++ { + f.sendCases[i].Send = reflect.Value{} + } + f.sendLock <- struct{}{} + return nsent +} + +type feedOfSub[T any] struct { + feed *FeedOf[T] + channel chan<- T + errOnce sync.Once + err chan error +} + +func (sub *feedOfSub[T]) Unsubscribe() { + sub.errOnce.Do(func() { + sub.feed.remove(sub) + close(sub.err) + }) +} + +func (sub *feedOfSub[T]) Err() <-chan error { + return sub.err +} diff --git a/p2p/event/feedof_test.go b/p2p/event/feedof_test.go new file mode 100644 index 00000000000..846afc9ee19 --- /dev/null +++ b/p2p/event/feedof_test.go @@ -0,0 +1,279 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package event + +import ( + "sync" + "testing" + "time" +) + +func TestFeedOf(t *testing.T) { + var feed FeedOf[int] + var done, subscribed sync.WaitGroup + subscriber := func(i int) { + defer done.Done() + + subchan := make(chan int) + sub := feed.Subscribe(subchan) + timeout := time.NewTimer(2 * time.Second) + defer timeout.Stop() + subscribed.Done() + + select { + case v := <-subchan: + if v != 1 { + t.Errorf("%d: received value %d, want 1", i, v) + } + case <-timeout.C: + t.Errorf("%d: receive timeout", i) + } + + sub.Unsubscribe() + select { + case _, ok := <-sub.Err(): + if ok { + t.Errorf("%d: error channel not closed after unsubscribe", i) + } + case <-timeout.C: + t.Errorf("%d: unsubscribe timeout", i) + } + } + + const n = 1000 + done.Add(n) + subscribed.Add(n) + for i := 0; i < n; i++ { + go subscriber(i) + } + subscribed.Wait() + if nsent := feed.Send(1); nsent != n { + t.Errorf("first send delivered %d times, want %d", nsent, n) + } + if nsent := feed.Send(2); nsent != 0 { + t.Errorf("second send delivered %d times, want 0", nsent) + } + done.Wait() +} + +func TestFeedOfSubscribeSameChannel(t *testing.T) { + var ( + feed FeedOf[int] + done sync.WaitGroup + ch = make(chan int) + sub1 = feed.Subscribe(ch) + sub2 = feed.Subscribe(ch) + _ = feed.Subscribe(ch) + ) + expectSends := func(value, n int) { + if nsent := feed.Send(value); nsent != n { + t.Errorf("send delivered %d times, want %d", nsent, n) + } + done.Done() + } + expectRecv := func(wantValue, n int) { + for i := 0; i < n; i++ { + if v := <-ch; v != wantValue { + t.Errorf("received %d, want %d", v, wantValue) + } + } + } + + done.Add(1) + go expectSends(1, 3) + expectRecv(1, 3) + done.Wait() + + sub1.Unsubscribe() + + done.Add(1) + go expectSends(2, 2) + expectRecv(2, 2) + done.Wait() + + sub2.Unsubscribe() + + done.Add(1) + go expectSends(3, 1) + expectRecv(3, 1) + done.Wait() +} + +func TestFeedOfSubscribeBlockedPost(t *testing.T) { + var ( + feed FeedOf[int] + nsends = 2000 + ch1 = make(chan int) + ch2 = make(chan int) + wg sync.WaitGroup + ) + defer wg.Wait() + + feed.Subscribe(ch1) + wg.Add(nsends) + for i := 0; i < nsends; i++ { + go func() { + feed.Send(99) + wg.Done() + }() + } + + sub2 := feed.Subscribe(ch2) + defer sub2.Unsubscribe() + + // We're done when ch1 has received N times. + // The number of receives on ch2 depends on scheduling. + for i := 0; i < nsends; { + select { + case <-ch1: + i++ + case <-ch2: + } + } +} + +func TestFeedOfUnsubscribeBlockedPost(t *testing.T) { + var ( + feed FeedOf[int] + nsends = 200 + chans = make([]chan int, 2000) + subs = make([]Subscription, len(chans)) + bchan = make(chan int) + bsub = feed.Subscribe(bchan) + wg sync.WaitGroup + ) + for i := range chans { + chans[i] = make(chan int, nsends) + } + + // Queue up some Sends. None of these can make progress while bchan isn't read. + wg.Add(nsends) + for i := 0; i < nsends; i++ { + go func() { + feed.Send(99) + wg.Done() + }() + } + // Subscribe the other channels. + for i, ch := range chans { + subs[i] = feed.Subscribe(ch) + } + // Unsubscribe them again. + for _, sub := range subs { + sub.Unsubscribe() + } + // Unblock the Sends. + bsub.Unsubscribe() + wg.Wait() +} + +// Checks that unsubscribing a channel during Send works even if that +// channel has already been sent on. +func TestFeedOfUnsubscribeSentChan(t *testing.T) { + var ( + feed FeedOf[int] + ch1 = make(chan int) + ch2 = make(chan int) + sub1 = feed.Subscribe(ch1) + sub2 = feed.Subscribe(ch2) + wg sync.WaitGroup + ) + defer sub2.Unsubscribe() + + wg.Add(1) + go func() { + feed.Send(0) + wg.Done() + }() + + // Wait for the value on ch1. + <-ch1 + // Unsubscribe ch1, removing it from the send cases. + sub1.Unsubscribe() + + // Receive ch2, finishing Send. + <-ch2 + wg.Wait() + + // Send again. This should send to ch2 only, so the wait group will unblock + // as soon as a value is received on ch2. + wg.Add(1) + go func() { + feed.Send(0) + wg.Done() + }() + <-ch2 + wg.Wait() +} + +func TestFeedOfUnsubscribeFromInbox(t *testing.T) { + var ( + feed FeedOf[int] + ch1 = make(chan int) + ch2 = make(chan int) + sub1 = feed.Subscribe(ch1) + sub2 = feed.Subscribe(ch1) + sub3 = feed.Subscribe(ch2) + ) + if len(feed.inbox) != 3 { + t.Errorf("inbox length != 3 after subscribe") + } + if len(feed.sendCases) != 1 { + t.Errorf("sendCases is non-empty after unsubscribe") + } + + sub1.Unsubscribe() + sub2.Unsubscribe() + sub3.Unsubscribe() + if len(feed.inbox) != 0 { + t.Errorf("inbox is non-empty after unsubscribe") + } + if len(feed.sendCases) != 1 { + t.Errorf("sendCases is non-empty after unsubscribe") + } +} + +func BenchmarkFeedOfSend1000(b *testing.B) { + var ( + done sync.WaitGroup + feed FeedOf[int] + nsubs = 1000 + ) + subscriber := func(ch <-chan int) { + for i := 0; i < b.N; i++ { + <-ch + } + done.Done() + } + done.Add(nsubs) + for i := 0; i < nsubs; i++ { + ch := make(chan int, 200) + feed.Subscribe(ch) + go subscriber(ch) + } + + // The actual benchmark. + b.ResetTimer() + for i := 0; i < b.N; i++ { + if feed.Send(i) != nsubs { + panic("wrong number of sends") + } + } + + b.StopTimer() + done.Wait() +} diff --git a/p2p/netutil/addrutil.go b/p2p/netutil/addrutil.go index 0244db009fb..b73d266fc92 100644 --- a/p2p/netutil/addrutil.go +++ b/p2p/netutil/addrutil.go @@ -19,18 +19,53 @@ package netutil -import "net" +import ( + "fmt" + "math/rand" + "net" + "net/netip" +) -// AddrIP gets the IP address contained in addr. It returns nil if no address is present. -func AddrIP(addr net.Addr) net.IP { +// AddrAddr gets the IP address contained in addr. The result will be invalid if the +// address type is unsupported. +func AddrAddr(addr net.Addr) netip.Addr { switch a := addr.(type) { case *net.IPAddr: - return a.IP + return IPToAddr(a.IP) case *net.TCPAddr: - return a.IP + return IPToAddr(a.IP) case *net.UDPAddr: - return a.IP + return IPToAddr(a.IP) default: - return nil + return netip.Addr{} } } + +// IPToAddr converts net.IP to netip.Addr. Note that unlike netip.AddrFromSlice, this +// function will always ensure that the resulting Addr is IPv4 when the input is. +func IPToAddr(ip net.IP) netip.Addr { + if ip4 := ip.To4(); ip4 != nil { + addr, _ := netip.AddrFromSlice(ip4) + return addr + } else if ip6 := ip.To16(); ip6 != nil { + addr, _ := netip.AddrFromSlice(ip6) + return addr + } + return netip.Addr{} +} + +// RandomAddr creates a random IP address. +func RandomAddr(rng *rand.Rand, ipv4 bool) netip.Addr { + var bytes []byte + if ipv4 || rng.Intn(2) == 0 { + bytes = make([]byte, 4) + } else { + bytes = make([]byte, 16) + } + rng.Read(bytes) + addr, ok := netip.AddrFromSlice(bytes) + if !ok { + panic(fmt.Errorf("BUG! invalid IP %v", bytes)) + } + return addr +} diff --git a/p2p/netutil/iptrack.go b/p2p/netutil/iptrack.go index 36b3a1df5b9..598b7def608 100644 --- a/p2p/netutil/iptrack.go +++ b/p2p/netutil/iptrack.go @@ -20,6 +20,7 @@ package netutil import ( + "net/netip" "time" "github.com/erigontech/erigon/common/mclock" @@ -32,14 +33,14 @@ type IPTracker struct { contactWindow time.Duration minStatements int clock mclock.Clock - statements map[string]ipStatement - contact map[string]mclock.AbsTime + statements map[netip.Addr]ipStatement + contact map[netip.Addr]mclock.AbsTime lastStatementGC mclock.AbsTime lastContactGC mclock.AbsTime } type ipStatement struct { - endpoint string + endpoint netip.AddrPort time mclock.AbsTime } @@ -54,9 +55,9 @@ func NewIPTracker(window, contactWindow time.Duration, minStatements int) *IPTra return &IPTracker{ window: window, contactWindow: contactWindow, - statements: make(map[string]ipStatement), + statements: make(map[netip.Addr]ipStatement), minStatements: minStatements, - contact: make(map[string]mclock.AbsTime), + contact: make(map[netip.Addr]mclock.AbsTime), clock: mclock.System{}, } } @@ -77,24 +78,27 @@ func (it *IPTracker) PredictFullConeNAT() bool { } // PredictEndpoint returns the current prediction of the external endpoint. -func (it *IPTracker) PredictEndpoint() string { +func (it *IPTracker) PredictEndpoint() netip.AddrPort { it.gcStatements(it.clock.Now()) // The current strategy is simple: find the endpoint with most statements. - counts := make(map[string]int) - maxcount, _max := 0, "" + var ( + counts = make(map[netip.AddrPort]int, len(it.statements)) + maxcount int + max netip.AddrPort + ) for _, s := range it.statements { c := counts[s.endpoint] + 1 counts[s.endpoint] = c if c > maxcount && c >= it.minStatements { - maxcount, _max = c, s.endpoint + maxcount, max = c, s.endpoint } } - return _max + return max } // AddStatement records that a certain host thinks our external endpoint is the one given. -func (it *IPTracker) AddStatement(host, endpoint string) { +func (it *IPTracker) AddStatement(host netip.Addr, endpoint netip.AddrPort) { now := it.clock.Now() it.statements[host] = ipStatement{endpoint, now} if time.Duration(now-it.lastStatementGC) >= it.window { @@ -104,7 +108,7 @@ func (it *IPTracker) AddStatement(host, endpoint string) { // AddContact records that a packet containing our endpoint information has been sent to a // certain host. -func (it *IPTracker) AddContact(host string) { +func (it *IPTracker) AddContact(host netip.Addr) { now := it.clock.Now() it.contact[host] = now if time.Duration(now-it.lastContactGC) >= it.contactWindow { diff --git a/p2p/netutil/net.go b/p2p/netutil/net.go index d6f7ac9d894..aecf8694e35 100644 --- a/p2p/netutil/net.go +++ b/p2p/netutil/net.go @@ -24,23 +24,19 @@ import ( "bytes" "errors" "fmt" + "maps" "net" + "net/netip" "slices" "strings" - "sync" ) -var lan4, lan6, special4, special6 Netlist +var special4, special6 Netlist func init() { // Lists from RFC 5735, RFC 5156, // https://www.iana.org/assignments/iana-ipv4-special-registry/ - lan4.Add("0.0.0.0/8") // "This" network - lan4.Add("10.0.0.0/8") // Private Use - lan4.Add("172.16.0.0/12") // Private Use - lan4.Add("192.168.0.0/16") // Private Use - lan6.Add("fe80::/10") // Link-Local - lan6.Add("fc00::/7") // Unique-Local + special4.Add("0.0.0.0/8") // "This" network. special4.Add("192.0.0.0/29") // IPv4 Service Continuity special4.Add("192.0.0.9/32") // PCP Anycast special4.Add("192.0.0.170/32") // NAT64/DNS64 Discovery @@ -70,7 +66,7 @@ func init() { } // Netlist is a list of IP networks. -type Netlist []net.IPNet +type Netlist []netip.Prefix // ParseNetlist parses a comma-separated list of CIDR masks. // Whitespace and extra commas are ignored. @@ -82,11 +78,11 @@ func ParseNetlist(s string) (*Netlist, error) { if mask == "" { continue } - _, n, err := net.ParseCIDR(mask) + prefix, err := netip.ParsePrefix(mask) if err != nil { return nil, err } - l = append(l, *n) + l = append(l, prefix) } return &l, nil } @@ -107,11 +103,11 @@ func (l *Netlist) UnmarshalTOML(fn func(interface{}) error) error { return err } for _, mask := range masks { - _, n, err := net.ParseCIDR(mask) + prefix, err := netip.ParsePrefix(mask) if err != nil { return err } - *l = append(*l, *n) + *l = append(*l, prefix) } return nil } @@ -119,15 +115,20 @@ func (l *Netlist) UnmarshalTOML(fn func(interface{}) error) error { // Add parses a CIDR mask and appends it to the list. It panics for invalid masks and is // intended to be used for setting up static lists. func (l *Netlist) Add(cidr string) { - _, n, err := net.ParseCIDR(cidr) + prefix, err := netip.ParsePrefix(cidr) if err != nil { panic(err) } - *l = append(*l, *n) + *l = append(*l, prefix) } // Contains reports whether the given IP is contained in the list. func (l *Netlist) Contains(ip net.IP) bool { + return l.ContainsAddr(IPToAddr(ip)) +} + +// ContainsAddr reports whether the given IP is contained in the list. +func (l *Netlist) ContainsAddr(ip netip.Addr) bool { if l == nil { return false } @@ -141,25 +142,39 @@ func (l *Netlist) Contains(ip net.IP) bool { // IsLAN reports whether an IP is a local network address. func IsLAN(ip net.IP) bool { + return AddrIsLAN(IPToAddr(ip)) +} + +// AddrIsLAN reports whether an IP is a local network address. +func AddrIsLAN(ip netip.Addr) bool { + if ip.Is4In6() { + ip = netip.AddrFrom4(ip.As4()) + } if ip.IsLoopback() { return true } - if v4 := ip.To4(); v4 != nil { - return lan4.Contains(v4) - } - return lan6.Contains(ip) + return ip.IsPrivate() || ip.IsLinkLocalUnicast() } // IsSpecialNetwork reports whether an IP is located in a special-use network range // This includes broadcast, multicast and documentation addresses. func IsSpecialNetwork(ip net.IP) bool { + return AddrIsSpecialNetwork(IPToAddr(ip)) +} + +// AddrIsSpecialNetwork reports whether an IP is located in a special-use network range +// This includes broadcast, multicast and documentation addresses. +func AddrIsSpecialNetwork(ip netip.Addr) bool { + if ip.Is4In6() { + ip = netip.AddrFrom4(ip.As4()) + } if ip.IsMulticast() { return true } - if v4 := ip.To4(); v4 != nil { - return special4.Contains(v4) + if ip.Is4() { + return special4.ContainsAddr(ip) } - return special6.Contains(ip) + return special6.ContainsAddr(ip) } var ( @@ -179,19 +194,31 @@ var ( // - LAN addresses are OK if relayed by a LAN host. // - All other addresses are always acceptable. func CheckRelayIP(sender, addr net.IP) error { - if len(addr) != net.IPv4len && len(addr) != net.IPv6len { + return CheckRelayAddr(IPToAddr(sender), IPToAddr(addr)) +} + +// CheckRelayAddr reports whether an IP relayed from the given sender IP +// is a valid connection target. +// +// There are four rules: +// - Special network addresses are never valid. +// - Loopback addresses are OK if relayed by a loopback host. +// - LAN addresses are OK if relayed by a LAN host. +// - All other addresses are always acceptable. +func CheckRelayAddr(sender, addr netip.Addr) error { + if !addr.IsValid() { return errInvalid } if addr.IsUnspecified() { return errUnspecified } - if IsSpecialNetwork(addr) { + if AddrIsSpecialNetwork(addr) { return errSpecial } if addr.IsLoopback() && !sender.IsLoopback() { return errLoopback } - if IsLAN(addr) && !IsLAN(sender) { + if AddrIsLAN(addr) && !AddrIsLAN(sender) { return errLAN } return nil @@ -225,20 +252,22 @@ type DistinctNetSet struct { Subnet uint // number of common prefix bits Limit uint // maximum number of IPs in each subnet - members *sync.Map - buf net.IP + members map[netip.Prefix]uint } // Add adds an IP address to the set. It returns false (and doesn't add the IP) if the // number of existing IPs in the defined range exceeds the limit. func (s *DistinctNetSet) Add(ip net.IP) bool { + return s.AddAddr(IPToAddr(ip)) +} + +// AddAddr adds an IP address to the set. It returns false (and doesn't add the IP) if the +// number of existing IPs in the defined range exceeds the limit. +func (s *DistinctNetSet) AddAddr(ip netip.Addr) bool { key := s.key(ip) - n := uint(0) - if value, ok := s.members.Load(string(key)); ok { - n = value.(uint) - } + n := s.members[key] if n < s.Limit { - s.members.Store(string(key), n+1) + s.members[key] = n + 1 return true } return false @@ -246,90 +275,65 @@ func (s *DistinctNetSet) Add(ip net.IP) bool { // Remove removes an IP from the set. func (s *DistinctNetSet) Remove(ip net.IP) { + s.RemoveAddr(IPToAddr(ip)) +} + +// RemoveAddr removes an IP from the set. +func (s *DistinctNetSet) RemoveAddr(ip netip.Addr) { key := s.key(ip) - if n, ok := s.members.Load(string(key)); ok { - if n.(uint) == 1 { - s.members.Delete(string(key)) + if n, ok := s.members[key]; ok { + if n == 1 { + delete(s.members, key) } else { - s.members.Store(string(key), n.(uint)-1) + s.members[key] = n - 1 } } } -// Contains whether the given IP is contained in the set. +// Contains reports whether the given IP is contained in the set. func (s DistinctNetSet) Contains(ip net.IP) bool { + return s.ContainsAddr(IPToAddr(ip)) +} + +// ContainsAddr reports whether the given IP is contained in the set. +func (s DistinctNetSet) ContainsAddr(ip netip.Addr) bool { key := s.key(ip) - _, ok := s.members.Load(string(key)) + _, ok := s.members[key] return ok } // Len returns the number of tracked IPs. func (s DistinctNetSet) Len() int { n := uint(0) - if s.members == nil { - return 0 + for _, i := range s.members { + n += i } - s.members.Range(func(_, v interface{}) bool { - n += v.(uint) - return true - }) return int(n) } -// key encodes the map key for an address into a temporary buffer. -// -// The first byte of key is '4' or '6' to distinguish IPv4/IPv6 address types. -// The remainder of the key is the IP, truncated to the number of bits. -func (s *DistinctNetSet) key(ip net.IP) net.IP { +// key returns the map key for ip. +func (s *DistinctNetSet) key(ip netip.Addr) netip.Prefix { // Lazily initialize storage. if s.members == nil { - s.members = &sync.Map{} - s.buf = make(net.IP, 17) + s.members = make(map[netip.Prefix]uint) } - // Canonicalize ip and bits. - typ := byte('6') - if ip4 := ip.To4(); ip4 != nil { - typ, ip = '4', ip4 - } - bits := min(s.Subnet, uint(len(ip)*8)) - // Encode the prefix into s.buf. - nb := int(bits / 8) - mask := ^byte(0xFF >> (bits % 8)) - s.buf[0] = typ - buf := append(s.buf[:1], ip[:nb]...) - if nb < len(ip) && mask != 0 { - buf = append(buf, ip[nb]&mask) + p, err := ip.Prefix(int(s.Subnet)) + if err != nil { + panic(err) } - return buf + return p } // String implements fmt.Stringer func (s DistinctNetSet) String() string { + keys := slices.SortedFunc(maps.Keys(s.members), func(a, b netip.Prefix) int { + return strings.Compare(a.String(), b.String()) + }) + var buf bytes.Buffer buf.WriteString("{") - if s.members == nil { - return "{}" - } - keys := []string{} - s.members.Range(func(k, v interface{}) bool { - keys = append(keys, k.(string)) - return true - }) - slices.Sort(keys) for i, k := range keys { - var ip net.IP - if k[0] == '4' { - ip = make(net.IP, 4) - } else { - ip = make(net.IP, 16) - } - copy(ip, k[1:]) - v, ok := s.members.Load(k) - vs := uint(0) - if ok { - vs = v.(uint) - } - fmt.Fprintf(&buf, "%v×%d", ip, vs) + fmt.Fprintf(&buf, "%v×%d", k, s.members[k]) if i != len(keys)-1 { buf.WriteString(" ") } From 93483cbfda361cfc08bdb526053889f7a9ae6f2e Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Fri, 30 Jan 2026 17:04:06 +1100 Subject: [PATCH 02/16] cmd/erigon builds --- cl/p2p/p2p_localnode.go | 6 +-- execution/rlp/decode.go | 93 ++++++++++++++++++++++++++++++++++++++++- execution/rlp/encode.go | 7 +++- p2p/enode/node.go | 16 +++++++ p2p/server.go | 31 +++++++------- 5 files changed, 132 insertions(+), 21 deletions(-) diff --git a/cl/p2p/p2p_localnode.go b/cl/p2p/p2p_localnode.go index 4c4e864b890..50089f128ee 100644 --- a/cl/p2p/p2p_localnode.go +++ b/cl/p2p/p2p_localnode.go @@ -20,11 +20,11 @@ func newLocalNode( tmpDir string, logger log.Logger, ) (*enode.LocalNode, error) { - db, err := enode.OpenDB(ctx, "", tmpDir, logger) + db, err := enode.OpenDBEx(ctx, "", tmpDir, logger) if err != nil { return nil, fmt.Errorf("could not open node's peer database: %w", err) } - localNode := enode.NewLocalNode(db, privKey, logger) + localNode := enode.NewLocalNode(db, privKey) ipEntry := enr.IP(ipAddr) udpEntry := enr.UDP(udpPort) @@ -75,7 +75,7 @@ func NewUDPv5Listener(ctx context.Context, cfg *P2PConfig, discCfg discover.Conf } // Start stream handlers - net, err := discover.ListenV5(ctx, "any", conn, localNode, discCfg) + net, err := discover.ListenV5(conn, localNode, discCfg) if err != nil { return nil, err } diff --git a/execution/rlp/decode.go b/execution/rlp/decode.go index df81db180a9..8fabe523d42 100644 --- a/execution/rlp/decode.go +++ b/execution/rlp/decode.go @@ -48,7 +48,9 @@ var ( ErrCanonSize = errors.New("rlp: non-canonical size information") ErrElemTooLarge = errors.New("rlp: element is larger than containing list") ErrValueTooLarge = errors.New("rlp: value size exceeds available input length") - ErrMoreThanOneValue = errors.New("rlp: input contains more than one value") + ErrMoreThanOneValue = errors.New("rlp: input contains more than one value") + ErrWrongTxTypePrefix = errors.New("rlp: only 1-byte tx type prefix is supported") + ErrUnknownTxTypePrefix = errors.New("rlp: unknown tx type prefix") // internal errors errNotInList = errors.New("rlp: call of ListEnd outside of any list") @@ -63,6 +65,23 @@ var ( } ) +// IsInvalidRLPError reports whether err is an RLP decoding error. +func IsInvalidRLPError(err error) bool { + return errors.Is(err, ErrExpectedString) || + errors.Is(err, ErrExpectedList) || + errors.Is(err, ErrCanonInt) || + errors.Is(err, ErrCanonSize) || + errors.Is(err, ErrElemTooLarge) || + errors.Is(err, ErrValueTooLarge) || + errors.Is(err, ErrMoreThanOneValue) || + errors.Is(err, ErrWrongTxTypePrefix) || + errors.Is(err, ErrUnknownTxTypePrefix) || + errors.Is(err, errNotInList) || + errors.Is(err, errNotAtEOL) || + errors.Is(err, errUintOverflow) || + errors.Is(err, errUint256Large) +} + // Decoder is implemented by types that require custom RLP decoding rules or need to decode // into private fields. // @@ -143,6 +162,11 @@ func wrapStreamError(err error, typ reflect.Type) error { return err } +// WrapStreamError wraps a Stream decoding error. +func WrapStreamError(err error, typ reflect.Type) error { + return wrapStreamError(err, typ) +} + func addErrorContext(err error, ctx string) error { if decErr, ok := err.(*decodeError); ok { decErr.ctx = append(decErr.ctx, ctx) @@ -150,6 +174,18 @@ func addErrorContext(err error, ctx string) error { return err } +// DecodeBytesPartial parses RLP data from b into val. +// Unlike DecodeBytes, it does not require that all bytes are consumed. +func DecodeBytesPartial(b []byte, val any) error { + r := (*sliceReader)(&b) + + stream := streamPool.Get().(*Stream) + defer streamPool.Put(stream) + + stream.Reset(r, uint64(len(b))) + return stream.Decode(val) +} + var ( decoderInterface = reflect.TypeFor[Decoder]() bigInt = reflect.TypeFor[big.Int]() @@ -599,6 +635,11 @@ type Stream struct { limited bool // true if input limit is in effect } +// Remaining returns number of bytes remaining to be read. +func (s *Stream) Remaining() uint64 { + return s.remaining +} + // NewStream creates a new decoding stream reading from r. // // If r implements the ByteReader interface, Stream will @@ -632,6 +673,15 @@ func NewListStream(r io.Reader, len uint64) *Stream { return s } +// NewStreamFromPool returns a Stream from the pool. +func NewStreamFromPool(r io.Reader, inputLimit uint64) (stream *Stream, done func()) { + stream = streamPool.Get().(*Stream) + stream.Reset(r, inputLimit) + return stream, func() { + streamPool.Put(stream) + } +} + // Bytes reads an RLP string and returns its contents as a byte slice. // If the input does not contain an RLP string, the returned // error will be ErrExpectedString. @@ -817,6 +867,13 @@ func (s *Stream) List() (size uint64, err error) { return size, nil } +// NewList starts decoding an RLP list, but without reading its prefix. +func (s *Stream) NewList(size uint64) { + s.stack = append(s.stack, size) + s.kind = -1 + s.size = 0 +} + // ListEnd returns to the enclosing list. // The input reader must be positioned at the end of a list. func (s *Stream) ListEnd() error { @@ -930,6 +987,40 @@ func (s *Stream) ReadUint256(dst *uint256.Int) error { return nil } +// Uint256Bytes decodes the next value as a uint256 and returns the raw bytes. +func (s *Stream) Uint256Bytes() ([]byte, error) { + var buffer []byte + kind, size, err := s.Kind() + switch { + case err != nil: + return nil, err + case kind == List: + return nil, ErrExpectedString + case kind == Byte: + buffer = s.uintbuf[:1] + buffer[0] = s.byteval + s.kind = -1 // re-arm Kind + case size == 0: + s.kind = -1 + case size <= uint64(len(s.uintbuf)): + buffer = s.uintbuf[:size] + if err := s.readFull(buffer); err != nil { + return nil, err + } + if size == 1 && buffer[0] < 128 { + return nil, ErrCanonSize + } + default: + return nil, errUint256Large + } + if len(buffer) > 0 && buffer[0] == 0 { + return nil, ErrCanonInt + } + out := make([]byte, len(buffer)) + copy(out, buffer) + return out, nil +} + // Decode decodes a value and stores the result in the value pointed // to by val. Please see the documentation for the Decode function // to learn about the decoding rules. diff --git a/execution/rlp/encode.go b/execution/rlp/encode.go index bf52f792c68..92f8326bbec 100644 --- a/execution/rlp/encode.go +++ b/execution/rlp/encode.go @@ -33,6 +33,11 @@ import ( "github.com/holiman/uint256" ) +const ( + EmptyStringCode = 0x80 + EmptyListCode = 0xc0 +) + var ( // Common encoded values. // These are useful when implementing EncodeRLP. @@ -40,7 +45,7 @@ var ( // EmptyString is the encoding of an empty string. EmptyString = []byte{0x80} // EmptyList is the encoding of an empty list. - EmptyList = []byte{0xC0} + EmptyList = []byte{EmptyListCode} ) var ErrNegativeBigInt = errors.New("rlp: cannot encode negative big.Int") diff --git a/p2p/enode/node.go b/p2p/enode/node.go index 72de555633b..34c49dd1e4d 100644 --- a/p2p/enode/node.go +++ b/p2p/enode/node.go @@ -163,6 +163,22 @@ func Parse(validSchemes enr.IdentityScheme, input string) (*Node, error) { return New(validSchemes, &r) } +// ParseNodesFromURLs parses a list of node URLs. +func ParseNodesFromURLs(urls []string) ([]*Node, error) { + nodes := make([]*Node, 0, len(urls)) + for _, url := range urls { + if url == "" { + continue + } + n, err := Parse(ValidSchemes, url) + if err != nil { + return nil, fmt.Errorf("invalid node URL %s: %w", url, err) + } + nodes = append(nodes, n) + } + return nodes, nil +} + // ID returns the node identifier. func (n *Node) ID() ID { return n.id diff --git a/p2p/server.go b/p2p/server.go index ec7362842b4..2e63f66b61d 100644 --- a/p2p/server.go +++ b/p2p/server.go @@ -28,8 +28,8 @@ import ( "encoding/hex" "errors" "net" + "net/netip" "sort" - "strconv" "strings" "sync" "sync/atomic" @@ -346,11 +346,11 @@ type sharedUDPConn struct { unhandled chan discover.ReadPacket } -// ReadFromUDP implements discover.UDPConn -func (s *sharedUDPConn) ReadFromUDP(b []byte) (n int, addr *net.UDPAddr, err error) { +// ReadFromUDPAddrPort implements discover.UDPConn +func (s *sharedUDPConn) ReadFromUDPAddrPort(b []byte) (n int, addr netip.AddrPort, err error) { packet, ok := <-s.unhandled if !ok { - return 0, nil, errors.New("connection was closed") + return 0, netip.AddrPort{}, errors.New("connection was closed") } l := min(len(packet.Data), len(b)) copy(b[:l], packet.Data[:l]) @@ -436,13 +436,13 @@ func (srv *Server) setupLocalNode() error { } sort.Sort(capsByNameAndVersion(srv.ourHandshake.Caps)) // Create the local node - db, err := enode.OpenDB(srv.quitCtx, srv.Config.NodeDatabase, srv.Config.TmpDir, srv.logger) + db, err := enode.OpenDBEx(srv.quitCtx, srv.Config.NodeDatabase, srv.Config.TmpDir, srv.logger) if err != nil { return err } srv.nodedb = db - srv.localnode = enode.NewLocalNode(db, srv.PrivateKey, srv.logger) + srv.localnode = enode.NewLocalNode(db, srv.PrivateKey) srv.localnode.SetFallbackIP(net.IP{127, 0, 0, 1}) // TODO: check conflicts @@ -533,7 +533,7 @@ func (srv *Server) setupDiscovery(ctx context.Context) error { Unhandled: unhandled, Log: srv.logger, } - ntab, err := discover.ListenV4(ctx, strconv.FormatUint(uint64(srv.Config.Protocols[0].Version), 10), conn, srv.localnode, cfg) + ntab, err := discover.ListenV4(conn, srv.localnode, cfg) if err != nil { return err } @@ -546,12 +546,11 @@ func (srv *Server) setupDiscovery(ctx context.Context) error { Bootnodes: srv.BootstrapNodesV5, Log: srv.logger, } - version := uint64(srv.Config.Protocols[0].Version) var err error if sconn != nil { - srv.discv5, err = discover.ListenV5(ctx, strconv.FormatUint(version, 10), sconn, srv.localnode, cfg) + srv.discv5, err = discover.ListenV5(sconn, srv.localnode, cfg) } else { - srv.discv5, err = discover.ListenV5(ctx, strconv.FormatUint(version, 10), conn, srv.localnode, cfg) + srv.discv5, err = discover.ListenV5(conn, srv.localnode, cfg) } if err != nil { // Clean up v4 if v5 setup fails. @@ -860,14 +859,14 @@ func (srv *Server) listenLoop(ctx context.Context) { break } - remoteIP := netutil.AddrIP(fd.RemoteAddr()) + remoteIP := netutil.AddrAddr(fd.RemoteAddr()) if err := srv.checkInboundConn(fd, remoteIP); err != nil { srv.logger.Trace("Rejected inbound connection", "addr", fd.RemoteAddr(), "err", err) _ = fd.Close() slots.Release(1) continue } - if remoteIP != nil { + if remoteIP.IsValid() { var addr *net.TCPAddr if tcp, ok := fd.RemoteAddr().(*net.TCPAddr); ok { addr = tcp @@ -884,18 +883,18 @@ func (srv *Server) listenLoop(ctx context.Context) { } } -func (srv *Server) checkInboundConn(fd net.Conn, remoteIP net.IP) error { - if remoteIP == nil { +func (srv *Server) checkInboundConn(fd net.Conn, remoteIP netip.Addr) error { + if !remoteIP.IsValid() { return nil } // Reject connections that do not match NetRestrict. - if srv.NetRestrict != nil && !srv.NetRestrict.Contains(remoteIP) { + if srv.NetRestrict != nil && !srv.NetRestrict.ContainsAddr(remoteIP) { return errors.New("not whitelisted in NetRestrict") } // Reject Internet peers that try too often. now := srv.clock.Now() srv.inboundHistory.expire(now, nil) - if !netutil.IsLAN(remoteIP) && srv.inboundHistory.contains(remoteIP.String()) { + if !netutil.AddrIsLAN(remoteIP) && srv.inboundHistory.contains(remoteIP.String()) { return errors.New("too many attempts") } srv.inboundHistory.add(remoteIP.String(), now.Add(inboundThrottleTime)) From 273cf27598d923e3177e61e492ada4d5edd1dd13 Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Mon, 2 Feb 2026 17:13:01 +1100 Subject: [PATCH 03/16] Claude makes a big mess --- cl/sentinel/handlers/heartbeats_test.go | 4 +- cmd/bootnode/main.go | 8 ++-- common/metrics/testdata/opentsb.want | 23 ++++++++++ execution/rlp/decode.go | 29 ++++++++---- execution/rlp/encbuffer_example_test.go | 2 +- execution/rlp/encode.go | 11 +++++ execution/rlp/encode_test.go | 2 +- execution/rlp/internal/rlpstruct/rlpstruct.go | 27 ++++++++--- execution/rlp/raw_test.go | 1 - execution/rlp/typecache.go | 8 ++++ execution/types/receipt.go | 20 ++++---- p2p/enode/localnode_test.go | 46 ++++++++++--------- p2p/enode/nodedb.go | 12 +++-- p2p/enode/nodedb_test.go | 40 ++++++++-------- p2p/enode/urlv4.go | 2 +- p2p/netutil/iptrack_test.go | 32 ++++++++----- p2p/netutil/net_test.go | 16 +++---- 17 files changed, 180 insertions(+), 103 deletions(-) create mode 100644 common/metrics/testdata/opentsb.want diff --git a/cl/sentinel/handlers/heartbeats_test.go b/cl/sentinel/handlers/heartbeats_test.go index 892156eec27..7bb3f6b37de 100644 --- a/cl/sentinel/handlers/heartbeats_test.go +++ b/cl/sentinel/handlers/heartbeats_test.go @@ -59,11 +59,11 @@ func newkey() *ecdsa.PrivateKey { } func testLocalNode() *enode.LocalNode { - db, err := enode.OpenDB(context.TODO(), "", "", log.Root()) + db, err := enode.OpenDBEx(context.TODO(), "", "", log.Root()) if err != nil { panic(err) } - ln := enode.NewLocalNode(db, newkey(), log.Root()) + ln := enode.NewLocalNode(db, newkey()) ln.Set(enr.WithEntry("attnets", attnetsTestVal)) ln.Set(enr.WithEntry("syncnets", syncnetsTestVal)) return ln diff --git a/cmd/bootnode/main.go b/cmd/bootnode/main.go index e7e9a7998e6..f75fdabd133 100644 --- a/cmd/bootnode/main.go +++ b/cmd/bootnode/main.go @@ -122,22 +122,22 @@ func main() { ctx, cancel := common.RootContext() defer cancel() - db, err := enode.OpenDB(ctx, "" /* path */, "" /* tmpDir */, logger) + db, err := enode.OpenDBEx(ctx, "" /* path */, "" /* tmpDir */, logger) if err != nil { panic(err) } - ln := enode.NewLocalNode(db, nodeKey, logger) + ln := enode.NewLocalNode(db, nodeKey) cfg := discover.Config{ PrivateKey: nodeKey, NetRestrict: restrictList, } if *runv5 { - if _, err := discover.ListenV5(ctx, "any", conn, ln, cfg); err != nil { + if _, err := discover.ListenV5(conn, ln, cfg); err != nil { utils.Fatalf("%v", err) } } else { - if _, err := discover.ListenUDP(ctx, "any", conn, ln, cfg); err != nil { + if _, err := discover.ListenUDP(conn, ln, cfg); err != nil { utils.Fatalf("%v", err) } } diff --git a/common/metrics/testdata/opentsb.want b/common/metrics/testdata/opentsb.want new file mode 100644 index 00000000000..43fe1b2ac27 --- /dev/null +++ b/common/metrics/testdata/opentsb.want @@ -0,0 +1,23 @@ +put pre.elite.count 978307200 1337 host=hal9000 +put pre.elite.one-minute 978307200 0.00 host=hal9000 +put pre.elite.five-minute 978307200 0.00 host=hal9000 +put pre.elite.fifteen-minute 978307200 0.00 host=hal9000 +put pre.elite.mean 978307200 0.00 host=hal9000 +put pre.foo.value 978307200 {"chain_id":"5"} host=hal9000 +put pre.months.count 978307200 12 host=hal9000 +put pre.pi.value 978307200 3.140000 host=hal9000 +put pre.second.count 978307200 1 host=hal9000 +put pre.second.min 978307200 1000 host=hal9000 +put pre.second.max 978307200 1000 host=hal9000 +put pre.second.mean 978307200 1000.00 host=hal9000 +put pre.second.std-dev 978307200 0.00 host=hal9000 +put pre.second.50-percentile 978307200 1000.00 host=hal9000 +put pre.second.75-percentile 978307200 1000.00 host=hal9000 +put pre.second.95-percentile 978307200 1000.00 host=hal9000 +put pre.second.99-percentile 978307200 1000.00 host=hal9000 +put pre.second.999-percentile 978307200 1000.00 host=hal9000 +put pre.second.one-minute 978307200 0.00 host=hal9000 +put pre.second.five-minute 978307200 0.00 host=hal9000 +put pre.second.fifteen-minute 978307200 0.00 host=hal9000 +put pre.second.mean-rate 978307200 0.00 host=hal9000 +put pre.tau.count 978307200 1.570000 host=hal9000 diff --git a/execution/rlp/decode.go b/execution/rlp/decode.go index 8fabe523d42..b5b95a0a8e7 100644 --- a/execution/rlp/decode.go +++ b/execution/rlp/decode.go @@ -79,7 +79,13 @@ func IsInvalidRLPError(err error) bool { errors.Is(err, errNotInList) || errors.Is(err, errNotAtEOL) || errors.Is(err, errUintOverflow) || - errors.Is(err, errUint256Large) + errors.Is(err, errUint256Large) || + // wrapStreamError produces *decodeError which doesn't implement Is() + strings.Contains(err.Error(), "rlp: input list has too many elements") || + strings.Contains(err.Error(), "rlp: expected input string or byte") || + strings.Contains(err.Error(), "rlp: expected input list") || + strings.Contains(err.Error(), "rlp: non-canonical size information") || + strings.Contains(err.Error(), "rlp: non-canonical integer (leading zero bytes)") } // Decoder is implemented by types that require custom RLP decoding rules or need to decode @@ -156,6 +162,8 @@ func wrapStreamError(err error, typ reflect.Type) error { return &decodeError{msg: "expected input string or byte", typ: typ} case errUintOverflow: return &decodeError{msg: "input string too long", typ: typ} + case errUint256Large: + return &decodeError{msg: "input string too long", typ: typ} case errNotAtEOL: return &decodeError{msg: "input list has too many elements", typ: typ} } @@ -211,6 +219,8 @@ func makeDecoder(typ reflect.Type, tags rlpstruct.Tags) (dec decoder, err error) return decodeDecoder, nil case isUint(kind): return decodeUint, nil + case isInt(kind): + return decodeInt, nil case kind == reflect.Bool: return decodeBool, nil case kind == reflect.String: @@ -245,6 +255,16 @@ func decodeUint(s *Stream, val reflect.Value) error { return nil } +func decodeInt(s *Stream, val reflect.Value) error { + typ := val.Type() + num, err := s.uint(typ.Bits()) + if err != nil { + return wrapStreamError(err, val.Type()) + } + val.SetInt(int64(num)) + return nil +} + func decodeBool(s *Stream, val reflect.Value) error { b, err := s.Bool() if err != nil { @@ -867,13 +887,6 @@ func (s *Stream) List() (size uint64, err error) { return size, nil } -// NewList starts decoding an RLP list, but without reading its prefix. -func (s *Stream) NewList(size uint64) { - s.stack = append(s.stack, size) - s.kind = -1 - s.size = 0 -} - // ListEnd returns to the enclosing list. // The input reader must be positioned at the end of a list. func (s *Stream) ListEnd() error { diff --git a/execution/rlp/encbuffer_example_test.go b/execution/rlp/encbuffer_example_test.go index ee15d82a77b..6168a2be3a1 100644 --- a/execution/rlp/encbuffer_example_test.go +++ b/execution/rlp/encbuffer_example_test.go @@ -20,7 +20,7 @@ import ( "bytes" "fmt" - "github.com/ethereum/go-ethereum/rlp" + "github.com/erigontech/erigon/execution/rlp" ) func ExampleEncoderBuffer() { diff --git a/execution/rlp/encode.go b/execution/rlp/encode.go index 92f8326bbec..95c767b49ff 100644 --- a/execution/rlp/encode.go +++ b/execution/rlp/encode.go @@ -166,6 +166,8 @@ func makeWriter(typ reflect.Type, ts rlpstruct.Tags) (writer, error) { return makeEncoderWriter(typ), nil case isUint(kind): return writeUint, nil + case isInt(kind): + return writeInt, nil case kind == reflect.Bool: return writeBool, nil case kind == reflect.String: @@ -195,6 +197,15 @@ func writeUint(val reflect.Value, w *encBuffer) error { return nil } +func writeInt(val reflect.Value, w *encBuffer) error { + i := val.Int() + if i < 0 { + return fmt.Errorf("rlp: type %v -ve values are not RLP-serializable", val.Type()) + } + w.writeUint64(uint64(i)) + return nil +} + func writeBool(val reflect.Value, w *encBuffer) error { w.writeBool(val.Bool()) return nil diff --git a/execution/rlp/encode_test.go b/execution/rlp/encode_test.go index d6fd24093ef..761b2d05c16 100644 --- a/execution/rlp/encode_test.go +++ b/execution/rlp/encode_test.go @@ -288,7 +288,7 @@ var encTests = []encTest{ {val: simplestruct{A: 3, B: "foo"}, output: "C50383666F6F"}, {val: &recstruct{5, nil}, output: "C205C0"}, {val: &recstruct{5, &recstruct{4, &recstruct{3, nil}}}, output: "C605C404C203C0"}, - {val: &intField{X: -3}, error: "rlp: type reflect.Value -ve values are not RLP-serializable"}, + {val: &intField{X: -3}, error: "rlp: type int -ve values are not RLP-serializable"}, {val: &intField{X: 3}, output: "C103"}, // struct tag "-" diff --git a/execution/rlp/internal/rlpstruct/rlpstruct.go b/execution/rlp/internal/rlpstruct/rlpstruct.go index 99c37173166..97d5e3cf669 100644 --- a/execution/rlp/internal/rlpstruct/rlpstruct.go +++ b/execution/rlp/internal/rlpstruct/rlpstruct.go @@ -51,7 +51,7 @@ type Type struct { // as an empty string or empty list. func (t Type) DefaultNilValue() NilKind { k := t.Kind - if isUint(k) || k == reflect.String || k == reflect.Bool || isByteArray(t) { + if isUint(k) || isInt(k) || k == reflect.String || k == reflect.Bool || isByteArray(t) { return NilKindString } return NilKindList @@ -102,6 +102,20 @@ func (e TagError) Error() string { return fmt.Sprintf("rlp: invalid struct tag %q for %s (%s)", e.Tag, field, e.Err) } +// OptionalFieldError is raised when a non-optional field follows an optional field. +type OptionalFieldError struct { + StructType string + Field string +} + +func (e OptionalFieldError) Error() string { + name := e.Field + if e.StructType != "" { + name = e.StructType + "." + e.Field + } + return fmt.Sprintf("rlp: struct field %s needs %q tag", name, "optional") +} + // ProcessFields filters the given struct fields, returning only fields // that should be considered for encoding/decoding. func ProcessFields(allFields []Field) ([]Field, []Tags, error) { @@ -129,18 +143,13 @@ func ProcessFields(allFields []Field) ([]Field, []Tags, error) { // all fields after it must also be optional. Note: optional + tail // is supported. var anyOptional bool - var firstOptionalName string for i, ts := range tags { name := fields[i].Name if ts.Optional || ts.Tail { - if !anyOptional { - firstOptionalName = name - } anyOptional = true } else { if anyOptional { - msg := fmt.Sprintf("must be optional because preceding field %q is optional", firstOptionalName) - return nil, nil, TagError{Field: name, Err: msg} + return nil, nil, OptionalFieldError{Field: name} } } } @@ -207,6 +216,10 @@ func isUint(k reflect.Kind) bool { return k >= reflect.Uint && k <= reflect.Uintptr } +func isInt(k reflect.Kind) bool { + return k >= reflect.Int && k <= reflect.Int64 +} + func isByte(typ Type) bool { return typ.Kind == reflect.Uint8 && !typ.IsEncoder } diff --git a/execution/rlp/raw_test.go b/execution/rlp/raw_test.go index 6627307b908..6b6fbe7345e 100644 --- a/execution/rlp/raw_test.go +++ b/execution/rlp/raw_test.go @@ -23,7 +23,6 @@ import ( "bytes" "errors" "io" - "reflect" "testing" "testing/quick" ) diff --git a/execution/rlp/typecache.go b/execution/rlp/typecache.go index c03a69efc61..2bf85092103 100644 --- a/execution/rlp/typecache.go +++ b/execution/rlp/typecache.go @@ -147,6 +147,10 @@ func structFields(typ reflect.Type) (fields []field, err error) { tagErr.StructType = typ.String() return nil, tagErr } + if optErr, ok := err.(rlpstruct.OptionalFieldError); ok { + optErr.StructType = typ.String() + return nil, optErr + } return nil, err } @@ -236,6 +240,10 @@ func isUint(k reflect.Kind) bool { return k >= reflect.Uint && k <= reflect.Uintptr } +func isInt(k reflect.Kind) bool { + return k >= reflect.Int && k <= reflect.Int64 +} + func isByte(typ reflect.Type) bool { return typ.Kind() == reflect.Uint8 && !typ.Implements(encoderInterface) } diff --git a/execution/types/receipt.go b/execution/types/receipt.go index a1b2d388573..7be60f063e5 100644 --- a/execution/types/receipt.go +++ b/execution/types/receipt.go @@ -310,27 +310,25 @@ func (r *Receipt) DecodeRLP(s *rlp.Stream) error { } r.Type = LegacyTxType case rlp.String: - // It's an EIP-2718 typed txn receipt. - s.NewList(size) // Hack - convert String (envelope) into List - var b []byte - if b, err = s.Bytes(); err != nil { - return fmt.Errorf("read TxType: %w", err) + // EIP-2718 typed txn receipt. Read the envelope as raw bytes, + // then decode from them using a fresh stream. + if size == 0 { + return rlp.EOL } - if len(b) != 1 { - return fmt.Errorf("%w, got %d bytes", rlp.ErrWrongTxTypePrefix, len(b)) + b := make([]byte, size) + if err = s.ReadBytes(b); err != nil { + return fmt.Errorf("read typed receipt: %w", err) } r.Type = b[0] switch r.Type { case AccessListTxType, DynamicFeeTxType, BlobTxType, SetCodeTxType: - if err := r.decodePayload(s); err != nil { + inner := rlp.NewStream(bytes.NewReader(b[1:]), uint64(len(b)-1)) + if err := r.decodePayload(inner); err != nil { return err } default: return ErrTxTypeNotSupported } - if err = s.ListEnd(); err != nil { - return err - } default: return rlp.ErrExpectedList } diff --git a/p2p/enode/localnode_test.go b/p2p/enode/localnode_test.go index 046f799f700..11d886f8e17 100644 --- a/p2p/enode/localnode_test.go +++ b/p2p/enode/localnode_test.go @@ -23,6 +23,7 @@ import ( "context" "math/rand" "net" + "net/netip" "testing" "github.com/stretchr/testify/assert" @@ -33,12 +34,12 @@ import ( ) func newLocalNodeForTesting(tmpDir string, logger log.Logger) (*LocalNode, *DB) { - db, err := OpenDB(context.Background(), "", tmpDir, logger) + db, err := OpenDBEx(context.Background(), "", tmpDir, logger) if err != nil { panic(err) } key, _ := crypto.GenerateKey() - return NewLocalNode(db, key, logger), db + return NewLocalNode(db, key), db } func TestLocalNode(t *testing.T) { @@ -66,28 +67,27 @@ func TestLocalNodeSeqPersist(t *testing.T) { ln, db := newLocalNodeForTesting(tmpDir, logger) defer db.Close() - if s := ln.Node().Seq(); s != 1 { - t.Fatalf("wrong initial seq %d, want 1", s) - } + initialSeq := ln.Node().Seq() ln.Set(enr.WithEntry("x", uint(1))) - if s := ln.Node().Seq(); s != 2 { - t.Fatalf("wrong seq %d after set, want 2", s) + if s := ln.Node().Seq(); s != initialSeq+1 { + t.Fatalf("wrong seq %d after set, want %d", s, initialSeq+1) } // Create a new instance, it should reload the sequence number. // The number increases just after that because a new record is // created without the "x" entry. - ln2 := NewLocalNode(db, ln.key, logger) - if s := ln2.Node().Seq(); s != 3 { - t.Fatalf("wrong seq %d on new instance, want 3", s) + ln2 := NewLocalNode(db, ln.key) + if s := ln2.Node().Seq(); s != initialSeq+2 { + t.Fatalf("wrong seq %d on new instance, want %d", s, initialSeq+2) } // Create a new instance with a different node key on the same database. - // This should reset the sequence number. + // This should reset the sequence number, but the new initial seq is + // time-based, so just check it's greater than zero. key, _ := crypto.GenerateKey() - ln3 := NewLocalNode(db, key, logger) - if s := ln3.Node().Seq(); s != 1 { - t.Fatalf("wrong seq %d on instance with changed key, want 1", s) + ln3 := NewLocalNode(db, key) + if s := ln3.Node().Seq(); s == 0 { + t.Fatalf("wrong seq %d on instance with changed key, want > 0", s) } } @@ -104,34 +104,36 @@ func TestLocalNodeEndpoint(t *testing.T) { defer db.Close() // Nothing is set initially. + initialSeq := ln.Node().Seq() assert.Equal(t, net.IP(nil), ln.Node().IP()) assert.Equal(t, 0, ln.Node().UDP()) - assert.Equal(t, uint64(1), ln.Node().Seq()) // Set up fallback address. ln.SetFallbackIP(fallback.IP) ln.SetFallbackUDP(fallback.Port) assert.Equal(t, fallback.IP, ln.Node().IP()) assert.Equal(t, fallback.Port, ln.Node().UDP()) - assert.Equal(t, uint64(2), ln.Node().Seq()) + assert.Equal(t, initialSeq+1, ln.Node().Seq()) // Add endpoint statements from random hosts. for i := 0; i < iptrackMinStatements; i++ { assert.Equal(t, fallback.IP, ln.Node().IP()) assert.Equal(t, fallback.Port, ln.Node().UDP()) - assert.Equal(t, uint64(2), ln.Node().Seq()) + assert.Equal(t, initialSeq+1, ln.Node().Seq()) - from := &net.UDPAddr{IP: make(net.IP, 4), Port: 90} - rand.Read(from.IP) - ln.UDPEndpointStatement(from, predicted) + fromIP := make(net.IP, 4) + rand.Read(fromIP) + fromAddr := netip.AddrPortFrom(netip.AddrFrom4([4]byte(fromIP)), 90) + predictedAddr := netip.AddrPortFrom(netip.AddrFrom4([4]byte(predicted.IP.To4())), uint16(predicted.Port)) + ln.UDPEndpointStatement(fromAddr, predictedAddr) } assert.Equal(t, predicted.IP, ln.Node().IP()) assert.Equal(t, predicted.Port, ln.Node().UDP()) - assert.Equal(t, uint64(3), ln.Node().Seq()) + assert.Equal(t, initialSeq+2, ln.Node().Seq()) // Static IP overrides prediction. ln.SetStaticIP(staticIP) assert.Equal(t, staticIP, ln.Node().IP()) assert.Equal(t, fallback.Port, ln.Node().UDP()) - assert.Equal(t, uint64(4), ln.Node().Seq()) + assert.Equal(t, initialSeq+3, ln.Node().Seq()) } diff --git a/p2p/enode/nodedb.go b/p2p/enode/nodedb.go index a2cab0b44c0..5209e9c93bc 100644 --- a/p2p/enode/nodedb.go +++ b/p2p/enode/nodedb.go @@ -39,6 +39,7 @@ import ( "github.com/erigontech/erigon/db/kv/dbcfg" "github.com/erigontech/erigon/db/kv/mdbx" "github.com/erigontech/erigon/execution/rlp" + "github.com/erigontech/erigon/p2p/enr" ) // Keys in the node database. @@ -223,6 +224,7 @@ func splitNodeItemKey(key []byte) (id ID, ip netip.Addr, field string) { key = key[len(dbDiscoverRoot)+1:] // Split out the IP. ip, _ = netip.AddrFromSlice(key[:16]) + ip = ip.Unmap() key = key[16+1:] // Field is the remainder of key. field = string(key) @@ -331,13 +333,13 @@ func (db *DB) Node(id ID) *Node { } func mustDecodeNode(id, data []byte) *Node { - node := new(Node) - if err := rlp.DecodeBytes(data, &node.r); err != nil { + var r enr.Record + if err := rlp.DecodeBytes(data, &r); err != nil { panic(fmt.Errorf("p2p/enode: can't decode node %x in DB: %w", id, err)) } - // Restore node id cache. - copy(node.id[:], id) - return node + var nodeID ID + copy(nodeID[:], id) + return newNodeWithID(&r, nodeID) } // UpdateNode inserts - potentially overwriting - a node into the peer database. diff --git a/p2p/enode/nodedb_test.go b/p2p/enode/nodedb_test.go index 07a5219097c..dda4e890aed 100644 --- a/p2p/enode/nodedb_test.go +++ b/p2p/enode/nodedb_test.go @@ -24,6 +24,7 @@ import ( "context" "fmt" "net" + "net/netip" "os" "path/filepath" "reflect" @@ -175,7 +176,7 @@ func TestDBNodeKey(t *testing.T) { } func TestDBNodeItemKey(t *testing.T) { - wantIP := net.IP{127, 0, 0, 3} + wantIP := netip.AddrFrom4([4]byte{127, 0, 0, 3}) wantField := "foobar" enc := nodeItemKey(keytestID, wantIP, wantField) want := []byte{ @@ -196,7 +197,7 @@ func TestDBNodeItemKey(t *testing.T) { if id != keytestID { t.Errorf("splitNodeItemKey returned wrong ID: %v", id) } - if !ip.Equal(wantIP) { + if ip != wantIP { t.Errorf("splitNodeItemKey returned wrong IP: %v", ip) } if field != wantField { @@ -214,7 +215,6 @@ var nodeDBInt64Tests = []struct { } func TestDBInt64(t *testing.T) { - tmpDir := t.TempDir() db, err := OpenDB("") if err != nil { panic(err) @@ -247,7 +247,6 @@ func TestDBFetchStore(t *testing.T) { 30303, 30303, ) - tmpDir := t.TempDir() inst := time.Now() num := 314 @@ -257,34 +256,35 @@ func TestDBFetchStore(t *testing.T) { } defer db.Close() + ip := node.IPAddr() // Check fetch/store operations on a node ping object - if stored := db.LastPingReceived(node.ID(), node.IP()); stored.Unix() != 0 { + if stored := db.LastPingReceived(node.ID(), ip); stored.Unix() != 0 { t.Errorf("ping: non-existing object: %v", stored) } - if err := db.UpdateLastPingReceived(node.ID(), node.IP(), inst); err != nil { + if err := db.UpdateLastPingReceived(node.ID(), ip, inst); err != nil { t.Errorf("ping: failed to update: %v", err) } - if stored := db.LastPingReceived(node.ID(), node.IP()); stored.Unix() != inst.Unix() { + if stored := db.LastPingReceived(node.ID(), ip); stored.Unix() != inst.Unix() { t.Errorf("ping: value mismatch: have %v, want %v", stored, inst) } // Check fetch/store operations on a node pong object - if stored := db.LastPongReceived(node.ID(), node.IP()); stored.Unix() != 0 { + if stored := db.LastPongReceived(node.ID(), ip); stored.Unix() != 0 { t.Errorf("pong: non-existing object: %v", stored) } - if err := db.UpdateLastPongReceived(node.ID(), node.IP(), inst); err != nil { + if err := db.UpdateLastPongReceived(node.ID(), ip, inst); err != nil { t.Errorf("pong: failed to update: %v", err) } - if stored := db.LastPongReceived(node.ID(), node.IP()); stored.Unix() != inst.Unix() { + if stored := db.LastPongReceived(node.ID(), ip); stored.Unix() != inst.Unix() { t.Errorf("pong: value mismatch: have %v, want %v", stored, inst) } // Check fetch/store operations on a node findnode-failure object - if stored := db.FindFails(node.ID(), node.IP()); stored != 0 { + if stored := db.FindFails(node.ID(), ip); stored != 0 { t.Errorf("find-node fails: non-existing object: %v", stored) } - if err := db.UpdateFindFails(node.ID(), node.IP(), num); err != nil { + if err := db.UpdateFindFails(node.ID(), ip, num); err != nil { t.Errorf("find-node fails: failed to update: %v", err) } - if stored := db.FindFails(node.ID(), node.IP()); stored != num { + if stored := db.FindFails(node.ID(), ip); stored != num { t.Errorf("find-node fails: value mismatch: have %v, want %v", stored, num) } // Check fetch/store operations on an actual node object @@ -405,7 +405,7 @@ func testSeedQuery(tmpDir string) error { if err := db.UpdateNode(seed.node); err != nil { return fmt.Errorf("node %d: failed to insert: %w", i, err) } - if err := db.UpdateLastPongReceived(seed.node.ID(), seed.node.IP(), seed.pong); err != nil { + if err := db.UpdateLastPongReceived(seed.node.ID(), seed.node.IPAddr(), seed.pong); err != nil { return fmt.Errorf("node %d: failed to insert bondTime: %w", i, err) } } @@ -444,7 +444,7 @@ func TestDBPersistency(t *testing.T) { ) // Create a persistent database and store some values - db, err := OpenDB(context.Background(), filepath.Join(root, "database"), root, log.Root()) + db, err := OpenDBEx(context.Background(), filepath.Join(root, "database"), root, log.Root()) if err != nil { t.Fatalf("failed to create persistent database: %v", err) } @@ -455,7 +455,7 @@ func TestDBPersistency(t *testing.T) { db.Close() // OpenSegments the database and check the value - db, err = OpenDB(context.Background(), filepath.Join(root, "database"), root, log.Root()) + db, err = OpenDBEx(context.Background(), filepath.Join(root, "database"), root, log.Root()) if err != nil { t.Fatalf("failed to open persistent database: %v", err) } @@ -557,7 +557,6 @@ var nodeDBExpirationNodes = []struct { } func TestDBExpiration(t *testing.T) { - tmpDir := t.TempDir() db, err := OpenDB("") if err != nil { panic(err) @@ -571,7 +570,7 @@ func TestDBExpiration(t *testing.T) { t.Fatalf("node %d: failed to insert: %v", i, err) } } - if err := db.UpdateLastPongReceived(seed.node.ID(), seed.node.IP(), seed.pong); err != nil { + if err := db.UpdateLastPongReceived(seed.node.ID(), seed.node.IPAddr(), seed.pong); err != nil { t.Fatalf("node %d: failed to update bondTime: %v", i, err) } } @@ -582,7 +581,7 @@ func TestDBExpiration(t *testing.T) { unixZeroTime := time.Unix(0, 0) for i, seed := range nodeDBExpirationNodes { node := db.Node(seed.node.ID()) - pong := db.LastPongReceived(seed.node.ID(), seed.node.IP()) + pong := db.LastPongReceived(seed.node.ID(), seed.node.IPAddr()) if seed.exp { if seed.storeNode && node != nil { t.Errorf("node %d (%s) shouldn't be present after expiration", i, seed.node.ID().TerminalString()) @@ -604,14 +603,13 @@ func TestDBExpiration(t *testing.T) { // This test checks that expiration works when discovery v5 data is present // in the database. func TestDBExpireV5(t *testing.T) { - tmpDir := t.TempDir() db, err := OpenDB("") if err != nil { panic(err) } defer db.Close() - ip := net.IP{127, 0, 0, 1} + ip := netip.AddrFrom4([4]byte{127, 0, 0, 1}) db.UpdateFindFailsV5(ID{}, ip, 4) db.expireNodes() } diff --git a/p2p/enode/urlv4.go b/p2p/enode/urlv4.go index 031bd9b6392..fc6de21f826 100644 --- a/p2p/enode/urlv4.go +++ b/p2p/enode/urlv4.go @@ -161,7 +161,7 @@ func parsePubkey(in string) (*ecdsa.PublicKey, error) { return nil, fmt.Errorf("wrong length, want %d hex chars", 128) } b = append([]byte{0x4}, b...) - return crypto.UnmarshalPubkey(b) + return crypto.UnmarshalPubkeyStd(b) } func (n *Node) URLv4() string { diff --git a/p2p/netutil/iptrack_test.go b/p2p/netutil/iptrack_test.go index 98c9131ad92..7d1b98de1bc 100644 --- a/p2p/netutil/iptrack_test.go +++ b/p2p/netutil/iptrack_test.go @@ -21,7 +21,7 @@ package netutil import ( "fmt" - mrand "math/rand" + "net/netip" "testing" "time" @@ -96,12 +96,24 @@ func runIPTrackerTest(t *testing.T, evs []iptrackTestEvent) { clock.Run(evtime - time.Duration(clock.Now())) switch ev.op { case opStatement: - it.AddStatement(ev.from, ev.ip) + from := netip.MustParseAddr(ev.from) + endpoint := netip.AddrPortFrom(netip.MustParseAddr(ev.ip), 0) + it.AddStatement(from, endpoint) case opContact: - it.AddContact(ev.from) + from := netip.MustParseAddr(ev.from) + it.AddContact(from) case opPredict: - if pred := it.PredictEndpoint(); pred != ev.ip { - t.Errorf("op %d: wrong prediction %q, want %q", i, pred, ev.ip) + pred := it.PredictEndpoint() + var wantStr string + if ev.ip != "" { + wantStr = netip.AddrPortFrom(netip.MustParseAddr(ev.ip), 0).String() + } + predStr := "" + if pred.IsValid() { + predStr = pred.String() + } + if predStr != wantStr { + t.Errorf("op %d: wrong prediction %q, want %q", i, predStr, wantStr) } case opCheckFullCone: pred := fmt.Sprintf("%t", it.PredictFullConeNAT()) @@ -124,12 +136,10 @@ func TestIPTrackerForceGC(t *testing.T) { it.clock = &clock for i := 0; i < 5*max; i++ { - e1 := make([]byte, 4) - e2 := make([]byte, 4) - mrand.Read(e1) - mrand.Read(e2) - it.AddStatement(string(e1), string(e2)) - it.AddContact(string(e1)) + addr := netip.AddrFrom4([4]byte{byte(i >> 24), byte(i >> 16), byte(i >> 8), byte(i)}) + endpoint := netip.AddrPortFrom(addr, uint16(i)) + it.AddStatement(addr, endpoint) + it.AddContact(addr) clock.Run(rate) } if len(it.contact) > 2*max { diff --git a/p2p/netutil/net_test.go b/p2p/netutil/net_test.go index c61a2e2520e..92e9c265c22 100644 --- a/p2p/netutil/net_test.go +++ b/p2p/netutil/net_test.go @@ -22,6 +22,7 @@ package netutil import ( "fmt" "net" + "net/netip" "reflect" "testing" "testing/quick" @@ -32,7 +33,7 @@ import ( func TestParseNetlist(t *testing.T) { var tests = []struct { input string - wantErr error + wantErr bool wantList *Netlist }{ { @@ -41,26 +42,25 @@ func TestParseNetlist(t *testing.T) { }, { input: "127.0.0.0/8", - wantErr: nil, - wantList: &Netlist{{IP: net.IP{127, 0, 0, 0}, Mask: net.CIDRMask(8, 32)}}, + wantList: &Netlist{netip.MustParsePrefix("127.0.0.0/8")}, }, { input: "127.0.0.0/44", - wantErr: &net.ParseError{Type: "CIDR address", Text: "127.0.0.0/44"}, + wantErr: true, }, { input: "127.0.0.0/16, 23.23.23.23/24,", wantList: &Netlist{ - {IP: net.IP{127, 0, 0, 0}, Mask: net.CIDRMask(16, 32)}, - {IP: net.IP{23, 23, 23, 0}, Mask: net.CIDRMask(24, 32)}, + netip.MustParsePrefix("127.0.0.0/16"), + netip.MustParsePrefix("23.23.23.23/24"), }, }, } for _, test := range tests { l, err := ParseNetlist(test.input) - if !reflect.DeepEqual(err, test.wantErr) { - t.Errorf("%q: got error %q, want %q", test.input, err, test.wantErr) + if (err != nil) != test.wantErr { + t.Errorf("%q: got error %q, wantErr=%v", test.input, err, test.wantErr) continue } if !reflect.DeepEqual(l, test.wantList) { From 3a485de107fb0e637535a5f4fb460303f14c605a Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Tue, 3 Feb 2026 12:59:23 +1100 Subject: [PATCH 04/16] Claude finishes fixing the tests --- p2p/enode/idscheme.go | 2 +- p2p/enode/urlv4.go | 5 +---- p2p/enode/urlv4_test.go | 19 +++++++------------ p2p/netutil/net_test.go | 5 +++-- 4 files changed, 12 insertions(+), 19 deletions(-) diff --git a/p2p/enode/idscheme.go b/p2p/enode/idscheme.go index 21d0e181dae..0b89c9ceb80 100644 --- a/p2p/enode/idscheme.go +++ b/p2p/enode/idscheme.go @@ -161,5 +161,5 @@ func SignNull(r *enr.Record, id ID) *Node { if err := r.SetSig(NullID{}, []byte{}); err != nil { panic(err) } - return &Node{r: *r, id: id} + return newNodeWithID(r, id) } diff --git a/p2p/enode/urlv4.go b/p2p/enode/urlv4.go index fc6de21f826..19008240dcc 100644 --- a/p2p/enode/urlv4.go +++ b/p2p/enode/urlv4.go @@ -34,10 +34,7 @@ import ( "github.com/erigontech/erigon/p2p/enr" ) -var ( - incompleteNodeURL = regexp.MustCompile("(?i)^(?:enode://)?([0-9a-f]+)$") - lookupIPFunc = net.LookupIP -) +var incompleteNodeURL = regexp.MustCompile("(?i)^(?:enode://)?([0-9a-f]+)$") // MustParseV4 parses a node URL. It panics if the URL is not valid. func MustParseV4(rawurl string) *Node { diff --git a/p2p/enode/urlv4_test.go b/p2p/enode/urlv4_test.go index 92d49e12762..3ab1eff386c 100644 --- a/p2p/enode/urlv4_test.go +++ b/p2p/enode/urlv4_test.go @@ -21,7 +21,6 @@ package enode import ( "crypto/ecdsa" - "errors" "net" "reflect" "strings" @@ -31,15 +30,6 @@ import ( "github.com/erigontech/erigon/p2p/enr" ) -func init() { - lookupIPFunc = func(name string) ([]net.IP, error) { - if name == "node.example.org" { - return []net.IP{{33, 44, 55, 66}}, nil - } - return nil, errors.New("no such host") - } -} - var parseNodeTests = []struct { input string wantError string @@ -74,8 +64,13 @@ var parseNodeTests = []struct { }, // Complete node URLs with IP address and ports { - input: "enode://1dd9d65c4552b5eb43d5ad55a2ee3f56c6cbc1c64a5c8d659f51fcd51bace24351232b8d7821617d2b29b54b81cdefb9b3e9c37d7fd5f63270bcc9e1a6f6a439@invalid.:3", - wantError: `no such host`, + input: "enode://1dd9d65c4552b5eb43d5ad55a2ee3f56c6cbc1c64a5c8d659f51fcd51bace24351232b8d7821617d2b29b54b81cdefb9b3e9c37d7fd5f63270bcc9e1a6f6a439@valid.:3", + wantResult: NewV4( + hexPubkey("1dd9d65c4552b5eb43d5ad55a2ee3f56c6cbc1c64a5c8d659f51fcd51bace24351232b8d7821617d2b29b54b81cdefb9b3e9c37d7fd5f63270bcc9e1a6f6a439"), + nil, + 3, + 3, + ).WithHostname("valid."), }, { input: "enode://1dd9d65c4552b5eb43d5ad55a2ee3f56c6cbc1c64a5c8d659f51fcd51bace24351232b8d7821617d2b29b54b81cdefb9b3e9c37d7fd5f63270bcc9e1a6f6a439@127.0.0.1:foo", diff --git a/p2p/netutil/net_test.go b/p2p/netutil/net_test.go index 92e9c265c22..0fa5c8d7304 100644 --- a/p2p/netutil/net_test.go +++ b/p2p/netutil/net_test.go @@ -79,8 +79,6 @@ func TestNilNetListContains(t *testing.T) { func TestIsLAN(t *testing.T) { checkContains(t, IsLAN, []string{ // included - "0.0.0.0", - "0.2.0.8", "127.0.0.1", "10.0.1.1", "10.22.0.3", @@ -89,12 +87,15 @@ func TestIsLAN(t *testing.T) { "fe80::f4a1:8eff:fec5:9d9d", "febf::ab32:2233", "fc00::4", + "::ffff:127.0.0.1", + "::ffff:10.10.0.2", }, []string{ // excluded "192.0.2.1", "1.0.0.0", "172.32.0.1", "fec0::2233", + "::ffff:88.99.100.2", }, ) } From 7871caf04e6de41924ba75f66bf28a3b138caec0 Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Tue, 3 Feb 2026 17:33:55 +1100 Subject: [PATCH 05/16] Just source common/lru from go-ethereum --- common/lru/basiclru.go | 228 ----------------------------- common/lru/basiclru_test.go | 255 --------------------------------- common/lru/blob_lru.go | 84 ----------- common/lru/blob_lru_test.go | 155 -------------------- common/lru/lru.go | 95 ------------ go.mod | 10 +- go.sum | 25 +++- p2p/discover/v5wire/session.go | 2 +- 8 files changed, 24 insertions(+), 830 deletions(-) delete mode 100644 common/lru/basiclru.go delete mode 100644 common/lru/basiclru_test.go delete mode 100644 common/lru/blob_lru.go delete mode 100644 common/lru/blob_lru_test.go delete mode 100644 common/lru/lru.go diff --git a/common/lru/basiclru.go b/common/lru/basiclru.go deleted file mode 100644 index 154831a7960..00000000000 --- a/common/lru/basiclru.go +++ /dev/null @@ -1,228 +0,0 @@ -// Copyright 2022 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -// Package lru implements generically-typed LRU caches. -package lru - -// BasicLRU is a simple LRU cache. -// -// This type is not safe for concurrent use. -// The zero value is not valid, instances must be created using NewCache. -type BasicLRU[K comparable, V any] struct { - list *list[K] - items map[K]cacheItem[K, V] - cap int -} - -type cacheItem[K any, V any] struct { - elem *listElem[K] - value V -} - -// NewBasicLRU creates a new LRU cache. -func NewBasicLRU[K comparable, V any](capacity int) BasicLRU[K, V] { - if capacity <= 0 { - capacity = 1 - } - c := BasicLRU[K, V]{ - items: make(map[K]cacheItem[K, V]), - list: newList[K](), - cap: capacity, - } - return c -} - -// Add adds a value to the cache. Returns true if an item was evicted to store the new item. -func (c *BasicLRU[K, V]) Add(key K, value V) (evicted bool) { - _, _, evicted = c.Add3(key, value) - return evicted -} - -// Add3 adds a value to the cache. If an item was evicted to store the new one, it returns the evicted item. -func (c *BasicLRU[K, V]) Add3(key K, value V) (ek K, ev V, evicted bool) { - item, ok := c.items[key] - if ok { - item.value = value - c.items[key] = item - c.list.moveToFront(item.elem) - return ek, ev, false - } - - var elem *listElem[K] - if c.Len() >= c.cap { - elem = c.list.removeLast() - evicted = true - ek = elem.v - ev = c.items[ek].value - delete(c.items, ek) - } else { - elem = new(listElem[K]) - } - - // Store the new item. - // Note that if another item was evicted, we re-use its list element here. - elem.v = key - c.items[key] = cacheItem[K, V]{elem, value} - c.list.pushElem(elem) - return ek, ev, evicted -} - -// Contains reports whether the given key exists in the cache. -func (c *BasicLRU[K, V]) Contains(key K) bool { - _, ok := c.items[key] - return ok -} - -// Get retrieves a value from the cache. This marks the key as recently used. -func (c *BasicLRU[K, V]) Get(key K) (value V, ok bool) { - item, ok := c.items[key] - if !ok { - return value, false - } - c.list.moveToFront(item.elem) - return item.value, true -} - -// GetOldest retrieves the least-recently-used item. -// Note that this does not update the item's recency. -func (c *BasicLRU[K, V]) GetOldest() (key K, value V, ok bool) { - lastElem := c.list.last() - if lastElem == nil { - return key, value, false - } - key = lastElem.v - item := c.items[key] - return key, item.value, true -} - -// Len returns the current number of items in the cache. -func (c *BasicLRU[K, V]) Len() int { - return len(c.items) -} - -// Peek retrieves a value from the cache, but does not mark the key as recently used. -func (c *BasicLRU[K, V]) Peek(key K) (value V, ok bool) { - item, ok := c.items[key] - return item.value, ok -} - -// Purge empties the cache. -func (c *BasicLRU[K, V]) Purge() { - c.list.init() - clear(c.items) -} - -// Remove drops an item from the cache. Returns true if the key was present in cache. -func (c *BasicLRU[K, V]) Remove(key K) bool { - item, ok := c.items[key] - if ok { - delete(c.items, key) - c.list.remove(item.elem) - } - return ok -} - -// RemoveOldest drops the least recently used item. -func (c *BasicLRU[K, V]) RemoveOldest() (key K, value V, ok bool) { - lastElem := c.list.last() - if lastElem == nil { - return key, value, false - } - - key = lastElem.v - item := c.items[key] - delete(c.items, key) - c.list.remove(lastElem) - return key, item.value, true -} - -// Keys returns all keys in the cache. -func (c *BasicLRU[K, V]) Keys() []K { - keys := make([]K, 0, len(c.items)) - return c.list.appendTo(keys) -} - -// list is a doubly-linked list holding items of type he. -// The zero value is not valid, use newList to create lists. -type list[T any] struct { - root listElem[T] -} - -type listElem[T any] struct { - next *listElem[T] - prev *listElem[T] - v T -} - -func newList[T any]() *list[T] { - l := new(list[T]) - l.init() - return l -} - -// init reinitializes the list, making it empty. -func (l *list[T]) init() { - l.root.next = &l.root - l.root.prev = &l.root -} - -// pushElem adds an element to the front of the list. -func (l *list[T]) pushElem(e *listElem[T]) { - e.prev = &l.root - e.next = l.root.next - l.root.next = e - e.next.prev = e -} - -// moveToFront makes 'node' the head of the list. -func (l *list[T]) moveToFront(e *listElem[T]) { - e.prev.next = e.next - e.next.prev = e.prev - l.pushElem(e) -} - -// remove removes an element from the list. -func (l *list[T]) remove(e *listElem[T]) { - e.prev.next = e.next - e.next.prev = e.prev - e.next, e.prev = nil, nil -} - -// removeLast removes the last element of the list. -func (l *list[T]) removeLast() *listElem[T] { - last := l.last() - if last != nil { - l.remove(last) - } - return last -} - -// last returns the last element of the list, or nil if the list is empty. -func (l *list[T]) last() *listElem[T] { - e := l.root.prev - if e == &l.root { - return nil - } - return e -} - -// appendTo appends all list elements to a slice. -func (l *list[T]) appendTo(slice []T) []T { - for e := l.root.prev; e != &l.root; e = e.prev { - slice = append(slice, e.v) - } - return slice -} diff --git a/common/lru/basiclru_test.go b/common/lru/basiclru_test.go deleted file mode 100644 index 29812bda157..00000000000 --- a/common/lru/basiclru_test.go +++ /dev/null @@ -1,255 +0,0 @@ -// Copyright 2022 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package lru - -import ( - crand "crypto/rand" - "fmt" - "io" - "math/rand" - "testing" -) - -// Some of these test cases were adapted -// from https://github.com/hashicorp/golang-lru/blob/master/simplelru/lru_test.go - -func TestBasicLRU(t *testing.T) { - cache := NewBasicLRU[int, int](128) - - for i := 0; i < 256; i++ { - cache.Add(i, i) - } - if cache.Len() != 128 { - t.Fatalf("bad len: %v", cache.Len()) - } - - // Check that Keys returns least-recent key first. - keys := cache.Keys() - if len(keys) != 128 { - t.Fatal("wrong Keys() length", len(keys)) - } - for i, k := range keys { - v, ok := cache.Peek(k) - if !ok { - t.Fatalf("expected key %d be present", i) - } - if v != k { - t.Fatalf("expected %d == %d", k, v) - } - if v != i+128 { - t.Fatalf("wrong value at key %d: %d, want %d", i, v, i+128) - } - } - - for i := 0; i < 128; i++ { - _, ok := cache.Get(i) - if ok { - t.Fatalf("%d should be evicted", i) - } - } - for i := 128; i < 256; i++ { - _, ok := cache.Get(i) - if !ok { - t.Fatalf("%d should not be evicted", i) - } - } - - for i := 128; i < 192; i++ { - ok := cache.Remove(i) - if !ok { - t.Fatalf("%d should be in cache", i) - } - ok = cache.Remove(i) - if ok { - t.Fatalf("%d should not be in cache", i) - } - _, ok = cache.Get(i) - if ok { - t.Fatalf("%d should be deleted", i) - } - } - - // Request item 192. - cache.Get(192) - // It should be the last item returned by Keys(). - for i, k := range cache.Keys() { - if (i < 63 && k != i+193) || (i == 63 && k != 192) { - t.Fatalf("out of order key: %v", k) - } - } - - cache.Purge() - if cache.Len() != 0 { - t.Fatalf("bad len: %v", cache.Len()) - } - if _, ok := cache.Get(200); ok { - t.Fatalf("should contain nothing") - } -} - -func TestBasicLRUAddExistingKey(t *testing.T) { - cache := NewBasicLRU[int, int](1) - - cache.Add(1, 1) - cache.Add(1, 2) - - v, _ := cache.Get(1) - if v != 2 { - t.Fatal("wrong value:", v) - } -} - -// This test checks GetOldest and RemoveOldest. -func TestBasicLRUGetOldest(t *testing.T) { - cache := NewBasicLRU[int, int](128) - for i := 0; i < 256; i++ { - cache.Add(i, i) - } - - k, _, ok := cache.GetOldest() - if !ok { - t.Fatalf("missing") - } - if k != 128 { - t.Fatalf("bad: %v", k) - } - - k, _, ok = cache.RemoveOldest() - if !ok { - t.Fatalf("missing") - } - if k != 128 { - t.Fatalf("bad: %v", k) - } - - k, _, ok = cache.RemoveOldest() - if !ok { - t.Fatalf("missing oldest item") - } - if k != 129 { - t.Fatalf("wrong oldest item: %v", k) - } -} - -// Test that Add returns true/false if an eviction occurred -func TestBasicLRUAddReturnValue(t *testing.T) { - cache := NewBasicLRU[int, int](1) - if cache.Add(1, 1) { - t.Errorf("first add shouldn't have evicted") - } - if !cache.Add(2, 2) { - t.Errorf("second add should have evicted") - } -} - -// This test verifies that Contains doesn't change item recency. -func TestBasicLRUContains(t *testing.T) { - cache := NewBasicLRU[int, int](2) - cache.Add(1, 1) - cache.Add(2, 2) - if !cache.Contains(1) { - t.Errorf("1 should be in the cache") - } - cache.Add(3, 3) - if cache.Contains(1) { - t.Errorf("Contains should not have updated recency of 1") - } -} - -// Test that Peek doesn't update recent-ness -func TestBasicLRUPeek(t *testing.T) { - cache := NewBasicLRU[int, int](2) - cache.Add(1, 1) - cache.Add(2, 2) - if v, ok := cache.Peek(1); !ok || v != 1 { - t.Errorf("1 should be set to 1") - } - cache.Add(3, 3) - if cache.Contains(1) { - t.Errorf("should not have updated recent-ness of 1") - } -} - -func BenchmarkLRU(b *testing.B) { - var ( - capacity = 1000 - indexes = make([]int, capacity*20) - keys = make([]string, capacity) - values = make([][]byte, capacity) - ) - for i := range indexes { - indexes[i] = rand.Intn(capacity) - } - for i := range keys { - b := make([]byte, 32) - crand.Read(b) - keys[i] = string(b) - crand.Read(b) - values[i] = b - } - - var sink []byte - - b.Run("Add/BasicLRU", func(b *testing.B) { - cache := NewBasicLRU[int, int](capacity) - for i := 0; i < b.N; i++ { - cache.Add(i, i) - } - }) - b.Run("Get/BasicLRU", func(b *testing.B) { - cache := NewBasicLRU[string, []byte](capacity) - for i := 0; i < capacity; i++ { - index := indexes[i] - cache.Add(keys[index], values[index]) - } - - b.ResetTimer() - for i := 0; i < b.N; i++ { - k := keys[indexes[i%len(indexes)]] - v, ok := cache.Get(k) - if ok { - sink = v - } - } - }) - - // // vs. github.com/hashicorp/golang-lru/simplelru - // b.Run("Add/simplelru.LRU", func(b *testing.B) { - // cache, _ := simplelru.NewLRU(capacity, nil) - // for i := 0; i < b.N; i++ { - // cache.Add(i, i) - // } - // }) - // b.Run("Get/simplelru.LRU", func(b *testing.B) { - // cache, _ := simplelru.NewLRU(capacity, nil) - // for i := 0; i < capacity; i++ { - // index := indexes[i] - // cache.Add(keys[index], values[index]) - // } - // - // b.ResetTimer() - // for i := 0; i < b.N; i++ { - // k := keys[indexes[i%len(indexes)]] - // v, ok := cache.Get(k) - // if ok { - // sink = v.([]byte) - // } - // } - // }) - - fmt.Fprintln(io.Discard, sink) -} diff --git a/common/lru/blob_lru.go b/common/lru/blob_lru.go deleted file mode 100644 index c9b33985032..00000000000 --- a/common/lru/blob_lru.go +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright 2022 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package lru - -import ( - "math" - "sync" -) - -// blobType is the type constraint for values stored in SizeConstrainedCache. -type blobType interface { - ~[]byte | ~string -} - -// SizeConstrainedCache is a cache where capacity is in bytes (instead of item count). When the cache -// is at capacity, and a new item is added, older items are evicted until the size -// constraint is met. -// -// OBS: This cache assumes that items are content-addressed: keys are unique per content. -// In other words: two Add(..) with the same key K, will always have the same value V. -type SizeConstrainedCache[K comparable, V blobType] struct { - size uint64 - maxSize uint64 - lru BasicLRU[K, V] - lock sync.Mutex -} - -// NewSizeConstrainedCache creates a new size-constrained LRU cache. -func NewSizeConstrainedCache[K comparable, V blobType](maxSize uint64) *SizeConstrainedCache[K, V] { - return &SizeConstrainedCache[K, V]{ - size: 0, - maxSize: maxSize, - lru: NewBasicLRU[K, V](math.MaxInt), - } -} - -// Add adds a value to the cache. Returns true if an eviction occurred. -// OBS: This cache assumes that items are content-addressed: keys are unique per content. -// In other words: two Add(..) with the same key K, will always have the same value V. -// OBS: The value is _not_ copied on Add, so the caller must not modify it afterwards. -func (c *SizeConstrainedCache[K, V]) Add(key K, value V) (evicted bool) { - c.lock.Lock() - defer c.lock.Unlock() - - // Unless it is already present, might need to evict something. - // OBS: If it is present, we still call Add internally to bump the recentness. - if !c.lru.Contains(key) { - targetSize := c.size + uint64(len(value)) - for targetSize > c.maxSize { - evicted = true - _, v, ok := c.lru.RemoveOldest() - if !ok { - // list is now empty. Break - break - } - targetSize -= uint64(len(v)) - } - c.size = targetSize - } - c.lru.Add(key, value) - return evicted -} - -// Get looks up a key's value from the cache. -func (c *SizeConstrainedCache[K, V]) Get(key K) (V, bool) { - c.lock.Lock() - defer c.lock.Unlock() - - return c.lru.Get(key) -} diff --git a/common/lru/blob_lru_test.go b/common/lru/blob_lru_test.go deleted file mode 100644 index ca1b0ddd742..00000000000 --- a/common/lru/blob_lru_test.go +++ /dev/null @@ -1,155 +0,0 @@ -// Copyright 2022 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package lru - -import ( - "encoding/binary" - "fmt" - "testing" -) - -type testKey [8]byte - -func mkKey(i int) (key testKey) { - binary.LittleEndian.PutUint64(key[:], uint64(i)) - return key -} - -func TestSizeConstrainedCache(t *testing.T) { - lru := NewSizeConstrainedCache[testKey, []byte](100) - var want uint64 - // Add 11 items of 10 byte each. First item should be swapped out - for i := 0; i < 11; i++ { - k := mkKey(i) - v := fmt.Sprintf("value-%04d", i) - lru.Add(k, []byte(v)) - want += uint64(len(v)) - if want > 100 { - want = 100 - } - if have := lru.size; have != want { - t.Fatalf("size wrong, have %d want %d", have, want) - } - } - // Zero:th should be evicted - { - k := mkKey(0) - if _, ok := lru.Get(k); ok { - t.Fatalf("should be evicted: %v", k) - } - } - // Elems 1-11 should be present - for i := 1; i < 11; i++ { - k := mkKey(i) - want := fmt.Sprintf("value-%04d", i) - have, ok := lru.Get(k) - if !ok { - t.Fatalf("missing key %v", k) - } - if string(have) != want { - t.Fatalf("wrong value, have %v want %v", have, want) - } - } -} - -// This test adds inserting an element exceeding the max size. -func TestSizeConstrainedCacheOverflow(t *testing.T) { - lru := NewSizeConstrainedCache[testKey, []byte](100) - - // Add 10 items of 10 byte each, filling the cache - for i := 0; i < 10; i++ { - k := mkKey(i) - v := fmt.Sprintf("value-%04d", i) - lru.Add(k, []byte(v)) - } - // Add one single large elem. We expect it to swap out all entries. - { - k := mkKey(1337) - v := make([]byte, 200) - lru.Add(k, v) - } - // Elems 0-9 should be missing - for i := 1; i < 10; i++ { - k := mkKey(i) - if _, ok := lru.Get(k); ok { - t.Fatalf("should be evicted: %v", k) - } - } - // The size should be accurate - if have, want := lru.size, uint64(200); have != want { - t.Fatalf("size wrong, have %d want %d", have, want) - } - // Adding one small item should swap out the large one - { - i := 0 - k := mkKey(i) - v := fmt.Sprintf("value-%04d", i) - lru.Add(k, []byte(v)) - if have, want := lru.size, uint64(10); have != want { - t.Fatalf("size wrong, have %d want %d", have, want) - } - } -} - -// This checks what happens when inserting the same k/v multiple times. -func TestSizeConstrainedCacheSameItem(t *testing.T) { - lru := NewSizeConstrainedCache[testKey, []byte](100) - - // Add one 10 byte-item 10 times. - k := mkKey(0) - v := fmt.Sprintf("value-%04d", 0) - for i := 0; i < 10; i++ { - lru.Add(k, []byte(v)) - } - - // The size should be accurate. - if have, want := lru.size, uint64(10); have != want { - t.Fatalf("size wrong, have %d want %d", have, want) - } -} - -// This tests that empty/nil values are handled correctly. -func TestSizeConstrainedCacheEmpties(t *testing.T) { - lru := NewSizeConstrainedCache[testKey, []byte](100) - - // This test abuses the lru a bit, using different keys for identical value(s). - for i := 0; i < 10; i++ { - lru.Add(testKey{byte(i)}, []byte{}) - lru.Add(testKey{byte(255 - i)}, nil) - } - - // The size should not count, only the values count. So this could be a DoS - // since it basically has no cap, and it is intentionally overloaded with - // different-keyed 0-length values. - if have, want := lru.size, uint64(0); have != want { - t.Fatalf("size wrong, have %d want %d", have, want) - } - - for i := 0; i < 10; i++ { - if v, ok := lru.Get(testKey{byte(i)}); !ok { - t.Fatalf("test %d: expected presence", i) - } else if v == nil { - t.Fatalf("test %d, v is nil", i) - } - - if v, ok := lru.Get(testKey{byte(255 - i)}); !ok { - t.Fatalf("test %d: expected presence", i) - } else if v != nil { - t.Fatalf("test %d, v is not nil", i) - } - } -} diff --git a/common/lru/lru.go b/common/lru/lru.go deleted file mode 100644 index 45965adb0df..00000000000 --- a/common/lru/lru.go +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright 2022 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package lru - -import "sync" - -// Cache is a LRU cache. -// This type is safe for concurrent use. -type Cache[K comparable, V any] struct { - cache BasicLRU[K, V] - mu sync.Mutex -} - -// NewCache creates an LRU cache. -func NewCache[K comparable, V any](capacity int) *Cache[K, V] { - return &Cache[K, V]{cache: NewBasicLRU[K, V](capacity)} -} - -// Add adds a value to the cache. Returns true if an item was evicted to store the new item. -func (c *Cache[K, V]) Add(key K, value V) (evicted bool) { - c.mu.Lock() - defer c.mu.Unlock() - - return c.cache.Add(key, value) -} - -// Contains reports whether the given key exists in the cache. -func (c *Cache[K, V]) Contains(key K) bool { - c.mu.Lock() - defer c.mu.Unlock() - - return c.cache.Contains(key) -} - -// Get retrieves a value from the cache. This marks the key as recently used. -func (c *Cache[K, V]) Get(key K) (value V, ok bool) { - c.mu.Lock() - defer c.mu.Unlock() - - return c.cache.Get(key) -} - -// Len returns the current number of items in the cache. -func (c *Cache[K, V]) Len() int { - c.mu.Lock() - defer c.mu.Unlock() - - return c.cache.Len() -} - -// Peek retrieves a value from the cache, but does not mark the key as recently used. -func (c *Cache[K, V]) Peek(key K) (value V, ok bool) { - c.mu.Lock() - defer c.mu.Unlock() - - return c.cache.Peek(key) -} - -// Purge empties the cache. -func (c *Cache[K, V]) Purge() { - c.mu.Lock() - defer c.mu.Unlock() - - c.cache.Purge() -} - -// Remove drops an item from the cache. Returns true if the key was present in cache. -func (c *Cache[K, V]) Remove(key K) bool { - c.mu.Lock() - defer c.mu.Unlock() - - return c.cache.Remove(key) -} - -// Keys returns all keys of items currently in the LRU. -func (c *Cache[K, V]) Keys() []K { - c.mu.Lock() - defer c.mu.Unlock() - - return c.cache.Keys() -} diff --git a/go.mod b/go.mod index 56305a1076b..96753ab834a 100644 --- a/go.mod +++ b/go.mod @@ -40,10 +40,11 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc github.com/deckarep/golang-set/v2 v2.8.0 github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 - github.com/dop251/goja v0.0.0-20220405120441-9037c2b61cbf + github.com/dop251/goja v0.0.0-20230605162241-28ee0ee714f3 github.com/edsrzf/mmap-go v1.2.0 github.com/elastic/go-freelru v0.16.0 github.com/ethereum/go-bigmodexpfix v0.0.0-20250911101455-f9e208c548ab + github.com/ethereum/go-ethereum v1.16.8 github.com/felixge/fgprof v0.9.5 github.com/go-chi/chi/v5 v5.2.3 github.com/go-chi/cors v1.2.2 @@ -99,13 +100,14 @@ require ( github.com/quic-go/quic-go v0.49.1 github.com/rs/cors v1.11.1 github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417 + github.com/shirou/gopsutil v3.21.11+incompatible github.com/shirou/gopsutil/v4 v4.25.10 github.com/spaolacci/murmur3 v1.1.0 github.com/spf13/afero v1.10.0 github.com/spf13/cobra v1.10.1 github.com/spf13/pflag v1.0.10 github.com/stretchr/testify v1.11.1 - github.com/supranational/blst v0.3.14 + github.com/supranational/blst v0.3.16-0.20250831170142-f48500c1fdbe github.com/thomaso-mirodin/intmath v0.0.0-20160323211736-5dc6d854e46e github.com/tidwall/btree v1.8.1 github.com/ugorji/go/codec v1.2.13 @@ -179,10 +181,9 @@ require ( github.com/elastic/gosigar v0.14.3 // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect - github.com/fjl/gencodec v0.0.0-20220412091415-8bb9e558978c // indirect + github.com/fjl/gencodec v0.1.0 // indirect github.com/flynn/noise v1.1.0 // indirect github.com/francoispqt/gojay v1.2.13 // indirect - github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/garslo/gogen v0.0.0-20170307003452-d6ebae628c7c // indirect github.com/go-llsqlite/adapter v0.0.0-20230927005056-7f5ce7f0c916 // indirect github.com/go-llsqlite/crawshaw v0.6.0 // indirect @@ -286,7 +287,6 @@ require ( github.com/rivo/uniseg v0.4.7 // indirect github.com/rogpeppe/go-internal v1.14.1 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/shirou/gopsutil v3.21.11+incompatible // indirect github.com/shopspring/decimal v1.2.0 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/sosodev/duration v1.3.1 // indirect diff --git a/go.sum b/go.sum index 36a76b4dcad..cc051f5012a 100644 --- a/go.sum +++ b/go.sum @@ -220,10 +220,13 @@ github.com/chromedp/cdproto v0.0.0-20230802225258-3cf4e6d46a89/go.mod h1:GKljq0V github.com/chromedp/chromedp v0.9.2/go.mod h1:LkSXJKONWTCHAfQasKFUZI+mxqS4tZqhmtGzzhLsnLs= github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moAV0xufSww= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/logex v1.2.0/go.mod h1:9+9sk7u7pGNWYMkh0hdiL++6OeibzJccyQU4p4MedaY= github.com/chzyer/logex v1.2.1/go.mod h1:JLbx6lG2kDbNRFnfkgvh4eRJRPX1QCoOIWomwysCBrQ= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/readline v1.5.0/go.mod h1:x22KAscuvRqlLoK9CsoYsmxoXZMMFVyOl86cAH8qUic= github.com/chzyer/readline v1.5.1/go.mod h1:Eh+b79XXUwfKfcPLepksvw2tcLE/Ct21YObkaSkeBlk= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/chzyer/test v0.0.0-20210722231415-061457976a23/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/chzyer/test v1.0.0/go.mod h1:2JlltgoNkt4TW/z9V/IzDdFaMTM2JPIi26O1pF38GC8= github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs= github.com/cilium/ebpf v0.16.0 h1:+BiEnHL6Z7lXnlGUsXQPPAE7+kenAd4ES8MQ5min0Ok= @@ -272,9 +275,11 @@ github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDD github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/dop251/goja v0.0.0-20220405120441-9037c2b61cbf h1:Yt+4K30SdjOkRoRRm3vYNQgR+/ZIy0RmeUDZo7Y8zeQ= -github.com/dop251/goja v0.0.0-20220405120441-9037c2b61cbf/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk= +github.com/dop251/goja v0.0.0-20211022113120-dc8c55024d06/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk= +github.com/dop251/goja v0.0.0-20230605162241-28ee0ee714f3 h1:+3HCtB74++ClLy8GgjUQYeC8R4ILzVcIe8+5edAJJnE= +github.com/dop251/goja v0.0.0-20230605162241-28ee0ee714f3/go.mod h1:QMWlm50DNe14hD7t24KEqZuUdC9sOTy8W6XbCU1mlw4= github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y= +github.com/dop251/goja_nodejs v0.0.0-20211022123610-8dd9abb0616d/go.mod h1:DngW8aVqWbuLRMHItjPUyqdj+HWPvnQe8V8y1nDpIbM= github.com/dustin/go-humanize v0.0.0-20180421182945-02af3965c54e/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= @@ -313,12 +318,14 @@ github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6 github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM= github.com/ethereum/go-bigmodexpfix v0.0.0-20250911101455-f9e208c548ab h1:rvv6MJhy07IMfEKuARQ9TKojGqLVNxQajaXEp/BoqSk= github.com/ethereum/go-bigmodexpfix v0.0.0-20250911101455-f9e208c548ab/go.mod h1:IuLm4IsPipXKF7CW5Lzf68PIbZ5yl7FFd74l/E0o9A8= +github.com/ethereum/go-ethereum v1.16.8 h1:LLLfkZWijhR5m6yrAXbdlTeXoqontH+Ga2f9igY7law= +github.com/ethereum/go-ethereum v1.16.8/go.mod h1:Fs6QebQbavneQTYcA39PEKv2+zIjX7rPUZ14DER46wk= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/felixge/fgprof v0.9.5 h1:8+vR6yu2vvSKn08urWyEuxx75NWPEvybbkBirEpsbVY= github.com/felixge/fgprof v0.9.5/go.mod h1:yKl+ERSa++RYOs32d8K6WEXCB4uXdLls4ZaZPpayhMM= -github.com/fjl/gencodec v0.0.0-20220412091415-8bb9e558978c h1:CndMRAH4JIwxbW8KYq6Q+cGWcGHz0FjGR3QqcInWcW0= -github.com/fjl/gencodec v0.0.0-20220412091415-8bb9e558978c/go.mod h1:AzA8Lj6YtixmJWL+wkKoBGsLWy9gFrAzi4g+5bCKwpY= +github.com/fjl/gencodec v0.1.0 h1:B3K0xPfc52cw52BBgUbSPxYo+HlLfAgWMVKRWXUXBcs= +github.com/fjl/gencodec v0.1.0/go.mod h1:Um1dFHPONZGTHog1qD1NaWjXJW/SPB38wPv0O8uZ2fI= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/flynn/noise v1.1.0 h1:KjPQoQCEFdZDiP03phOvGi11+SVVhBG2wOWAorLsstg= github.com/flynn/noise v1.1.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag= @@ -467,6 +474,7 @@ github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20230207041349-798e818bf904/go.mod h1:uglQLonpP8qtYCYyzA+8c/9qtqgA3qsXGYqCPKARAFg= github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs= github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= @@ -530,6 +538,7 @@ github.com/ianlancetaylor/cgosymbolizer v0.0.0-20241129212102-9c50ad6b591e h1:8A github.com/ianlancetaylor/cgosymbolizer v0.0.0-20241129212102-9c50ad6b591e/go.mod h1:DvXTE/K/RtHehxU8/GtDs4vFtfw64jJ3PaCnFri8CRg= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20220319035150-800ac71e25c2/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= github.com/ianlancetaylor/demangle v0.0.0-20230524184225-eabc099b10ab/go.mod h1:gx7rwoVhcfuVKG5uya9Hs3Sxj7EIvldVofAWIUtGouw= github.com/imdario/mergo v0.3.11 h1:3tnifQM4i+fbajXKBHXWEH+KvNHqojZ778UH75j3bGA= github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= @@ -580,6 +589,7 @@ github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -861,6 +871,7 @@ github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJ github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= @@ -954,8 +965,8 @@ github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -github.com/supranational/blst v0.3.14 h1:xNMoHRJOTwMn63ip6qoWJ2Ymgvj7E2b9jY2FAwY+qRo= -github.com/supranational/blst v0.3.14/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= +github.com/supranational/blst v0.3.16-0.20250831170142-f48500c1fdbe h1:nbdqkIGOGfUAD54q1s2YBcBz/WcsxCO9HUQ4aGV5hUw= +github.com/supranational/blst v0.3.16-0.20250831170142-f48500c1fdbe/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= github.com/thomaso-mirodin/intmath v0.0.0-20160323211736-5dc6d854e46e h1:cR8/SYRgyQCt5cNCMniB/ZScMkhI9nk8U5C7SbISXjo= github.com/thomaso-mirodin/intmath v0.0.0-20160323211736-5dc6d854e46e/go.mod h1:Tu4lItkATkonrYuvtVjG0/rhy15qrNGNTjPdaphtZ/8= @@ -1252,7 +1263,6 @@ golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1282,6 +1292,7 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= diff --git a/p2p/discover/v5wire/session.go b/p2p/discover/v5wire/session.go index ed3bc490f9b..2857128d973 100644 --- a/p2p/discover/v5wire/session.go +++ b/p2p/discover/v5wire/session.go @@ -22,7 +22,7 @@ import ( "encoding/binary" "time" - "github.com/erigontech/erigon/common/lru" + "github.com/ethereum/go-ethereum/common/lru" "github.com/erigontech/erigon/common/mclock" "github.com/erigontech/erigon/common/crypto" "github.com/erigontech/erigon/p2p/enode" From c8b43fdf31c4e3bd0577a5cef5cf7440f3eaaa99 Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Tue, 3 Feb 2026 18:17:25 +1100 Subject: [PATCH 06/16] Merge used common/metrics stuff into diagnostics/metrics --- cmd/utils/flags.go | 2 +- common/metrics/FORK.md | 1 - common/metrics/LICENSE | 29 -- common/metrics/README.md | 147 ------ common/metrics/config.go | 33 -- common/metrics/counter.go | 55 --- common/metrics/counter_float64.go | 66 --- common/metrics/counter_float_64_test.go | 73 --- common/metrics/counter_test.go | 61 --- common/metrics/cpu.go | 25 -- common/metrics/cpu_disabled.go | 24 - common/metrics/cpu_enabled.go | 44 -- common/metrics/cputime_nop.go | 26 -- common/metrics/cputime_unix.go | 36 -- common/metrics/debug.go | 76 ---- common/metrics/debug_test.go | 48 -- common/metrics/disk.go | 25 -- common/metrics/disk_linux.go | 72 --- common/metrics/disk_nop.go | 27 -- common/metrics/ewma.go | 91 ---- common/metrics/ewma_test.go | 89 ---- common/metrics/gauge.go | 67 --- common/metrics/gauge_float64.go | 47 -- common/metrics/gauge_float64_test.go | 51 --- common/metrics/gauge_info.go | 61 --- common/metrics/gauge_info_test.go | 36 -- common/metrics/gauge_test.go | 31 -- common/metrics/healthcheck.go | 35 -- common/metrics/histogram.go | 66 --- common/metrics/histogram_test.go | 95 ---- common/metrics/init_test.go | 5 - common/metrics/json.go | 31 -- common/metrics/json_test.go | 28 -- common/metrics/log.go | 82 ---- common/metrics/memory.md | 285 ------------ common/metrics/meter.go | 167 ------- common/metrics/meter_test.go | 83 ---- common/metrics/metrics.go | 204 --------- common/metrics/metrics_test.go | 62 --- common/metrics/opentsdb.go | 128 ------ common/metrics/opentsdb_test.go | 66 --- common/metrics/registry.go | 363 --------------- common/metrics/registry_test.go | 335 -------------- common/metrics/resetting_sample.go | 24 - common/metrics/resetting_timer.go | 134 ------ common/metrics/resetting_timer_test.go | 197 -------- common/metrics/runtimehistogram.go | 298 ------------ common/metrics/runtimehistogram_test.go | 162 ------- common/metrics/sample.go | 425 ------------------ common/metrics/sample_test.go | 324 ------------- common/metrics/syslog.go | 83 ---- common/metrics/testdata/opentsb.want | 23 - common/metrics/timer.go | 149 ------ common/metrics/timer_test.go | 114 ----- common/metrics/validate.sh | 10 - common/metrics/writer.go | 99 ---- common/metrics/writer_test.go | 22 - .../metrics/block_metrics.go | 37 +- diagnostics/metrics/config.go | 17 + execution/execmodule/forkchoice.go | 2 +- execution/execmodule/inserters.go | 2 +- .../stagedsync/headerdownload/header_algos.go | 2 +- execution/stagedsync/stage_bodies.go | 2 +- execution/stagedsync/stage_mining_exec.go | 2 +- execution/stagedsync/stageloop/stageloop.go | 2 +- node/logging/logging.go | 2 +- p2p/discover/metrics.go | 22 +- p2p/discover/table.go | 9 +- 68 files changed, 46 insertions(+), 5495 deletions(-) delete mode 100644 common/metrics/FORK.md delete mode 100644 common/metrics/LICENSE delete mode 100644 common/metrics/README.md delete mode 100644 common/metrics/config.go delete mode 100644 common/metrics/counter.go delete mode 100644 common/metrics/counter_float64.go delete mode 100644 common/metrics/counter_float_64_test.go delete mode 100644 common/metrics/counter_test.go delete mode 100644 common/metrics/cpu.go delete mode 100644 common/metrics/cpu_disabled.go delete mode 100644 common/metrics/cpu_enabled.go delete mode 100644 common/metrics/cputime_nop.go delete mode 100644 common/metrics/cputime_unix.go delete mode 100644 common/metrics/debug.go delete mode 100644 common/metrics/debug_test.go delete mode 100644 common/metrics/disk.go delete mode 100644 common/metrics/disk_linux.go delete mode 100644 common/metrics/disk_nop.go delete mode 100644 common/metrics/ewma.go delete mode 100644 common/metrics/ewma_test.go delete mode 100644 common/metrics/gauge.go delete mode 100644 common/metrics/gauge_float64.go delete mode 100644 common/metrics/gauge_float64_test.go delete mode 100644 common/metrics/gauge_info.go delete mode 100644 common/metrics/gauge_info_test.go delete mode 100644 common/metrics/gauge_test.go delete mode 100644 common/metrics/healthcheck.go delete mode 100644 common/metrics/histogram.go delete mode 100644 common/metrics/histogram_test.go delete mode 100644 common/metrics/init_test.go delete mode 100644 common/metrics/json.go delete mode 100644 common/metrics/json_test.go delete mode 100644 common/metrics/log.go delete mode 100644 common/metrics/memory.md delete mode 100644 common/metrics/meter.go delete mode 100644 common/metrics/meter_test.go delete mode 100644 common/metrics/metrics.go delete mode 100644 common/metrics/metrics_test.go delete mode 100644 common/metrics/opentsdb.go delete mode 100644 common/metrics/opentsdb_test.go delete mode 100644 common/metrics/registry.go delete mode 100644 common/metrics/registry_test.go delete mode 100644 common/metrics/resetting_sample.go delete mode 100644 common/metrics/resetting_timer.go delete mode 100644 common/metrics/resetting_timer_test.go delete mode 100644 common/metrics/runtimehistogram.go delete mode 100644 common/metrics/runtimehistogram_test.go delete mode 100644 common/metrics/sample.go delete mode 100644 common/metrics/sample_test.go delete mode 100644 common/metrics/syslog.go delete mode 100644 common/metrics/testdata/opentsb.want delete mode 100644 common/metrics/timer.go delete mode 100644 common/metrics/timer_test.go delete mode 100755 common/metrics/validate.sh delete mode 100644 common/metrics/writer.go delete mode 100644 common/metrics/writer_test.go rename {common => diagnostics}/metrics/block_metrics.go (51%) create mode 100644 diagnostics/metrics/config.go diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 15d7972ccec..02b550e4d56 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -44,7 +44,7 @@ import ( "github.com/erigontech/erigon/common" libkzg "github.com/erigontech/erigon/common/crypto/kzg" "github.com/erigontech/erigon/common/log/v3" - "github.com/erigontech/erigon/common/metrics" + "github.com/erigontech/erigon/diagnostics/metrics" "github.com/erigontech/erigon/db/datadir" "github.com/erigontech/erigon/db/downloader/downloadercfg" "github.com/erigontech/erigon/db/snapcfg" diff --git a/common/metrics/FORK.md b/common/metrics/FORK.md deleted file mode 100644 index b19985bf56e..00000000000 --- a/common/metrics/FORK.md +++ /dev/null @@ -1 +0,0 @@ -This repo has been forked from https://github.com/rcrowley/go-metrics at commit e181e09 diff --git a/common/metrics/LICENSE b/common/metrics/LICENSE deleted file mode 100644 index 363fa9ee77b..00000000000 --- a/common/metrics/LICENSE +++ /dev/null @@ -1,29 +0,0 @@ -Copyright 2012 Richard Crowley. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following - disclaimer in the documentation and/or other materials provided - with the distribution. - -THIS SOFTWARE IS PROVIDED BY RICHARD CROWLEY ``AS IS'' AND ANY EXPRESS -OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL RICHARD CROWLEY OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF -THE POSSIBILITY OF SUCH DAMAGE. - -The views and conclusions contained in the software and documentation -are those of the authors and should not be interpreted as representing -official policies, either expressed or implied, of Richard Crowley. diff --git a/common/metrics/README.md b/common/metrics/README.md deleted file mode 100644 index 85b119470a9..00000000000 --- a/common/metrics/README.md +++ /dev/null @@ -1,147 +0,0 @@ -go-metrics -========== - -![travis build status](https://travis-ci.org/rcrowley/go-metrics.svg?branch=master) - -Go port of Coda Hale's Metrics library: . - -Documentation: . - -Usage ------ - -Create and update metrics: - -```go -c := metrics.NewCounter() -metrics.Register("foo", c) -c.Inc(47) - -g := metrics.NewGauge() -metrics.Register("bar", g) -g.Update(47) - -r := NewRegistry() -g := metrics.NewRegisteredFunctionalGauge("cache-evictions", r, func() int64 { return cache.getEvictionsCount() }) - -s := metrics.NewExpDecaySample(1028, 0.015) // or metrics.NewUniformSample(1028) -h := metrics.NewHistogram(s) -metrics.Register("baz", h) -h.Update(47) - -m := metrics.NewMeter() -metrics.Register("quux", m) -m.Mark(47) - -t := metrics.NewTimer() -metrics.Register("bang", t) -t.Time(func() {}) -t.Update(47) -``` - -Register() is not threadsafe. For threadsafe metric registration use -GetOrRegister: - -```go -t := metrics.GetOrRegisterTimer("account.create.latency", nil) -t.Time(func() {}) -t.Update(47) -``` - -**NOTE:** Be sure to unregister short-lived meters and timers otherwise they will -leak memory: - -```go -// Will call Stop() on the Meter to allow for garbage collection -metrics.Unregister("quux") -// Or similarly for a Timer that embeds a Meter -metrics.Unregister("bang") -``` - -Periodically log every metric in human-readable form to standard error: - -```go -go metrics.Log(metrics.DefaultRegistry, 5 * time.Second, log.New(os.Stderr, "metrics: ", log.Lmicroseconds)) -``` - -Periodically log every metric in slightly-more-parseable form to syslog: - -```go -w, _ := syslog.Dial("unixgram", "/dev/log", syslog.LOG_INFO, "metrics") -go metrics.Syslog(metrics.DefaultRegistry, 60e9, w) -``` - -Periodically emit every metric to Graphite using the [Graphite client](https://github.com/cyberdelia/go-metrics-graphite): - -```go - -import "github.com/cyberdelia/go-metrics-graphite" - -addr, _ := net.ResolveTCPAddr("tcp", "127.0.0.1:2003") -go graphite.Graphite(metrics.DefaultRegistry, 10e9, "metrics", addr) -``` - -Periodically emit every metric into InfluxDB: - -**NOTE:** this has been pulled out of the library due to constant fluctuations -in the InfluxDB API. In fact, all client libraries are on their way out. see -issues [#121](https://github.com/rcrowley/go-metrics/issues/121) and -[#124](https://github.com/rcrowley/go-metrics/issues/124) for progress and details. - -```go -import "github.com/vrischmann/go-metrics-influxdb" - -go influxdb.InfluxDB(metrics.DefaultRegistry, - 10e9, - "127.0.0.1:8086", - "database-name", - "username", - "password" -) -``` - -Periodically emit every metric to StatHat: - -```go -import "github.com/rcrowley/go-metrics/stathat" - -go stathat.Stathat(metrics.DefaultRegistry, 10e9, "example@example.com") -``` - -Maintain all metrics along with expvars at `/debug/metrics`: - -This uses the same mechanism as [the official expvar](https://golang.org/pkg/expvar/) -but exposed under `/debug/metrics`, which shows a json representation of all your usual expvars -as well as all your go-metrics. - - -```go -import "github.com/rcrowley/go-metrics/exp" - -exp.Exp(metrics.DefaultRegistry) -``` - -Installation ------------- - -```sh -go get github.com/rcrowley/go-metrics -``` - -StatHat support additionally requires their Go client: - -```sh -go get github.com/stathat/go -``` - -Publishing Metrics ------------------- - -Clients are available for the following destinations: - -* Graphite - https://github.com/cyberdelia/go-metrics-graphite -* InfluxDB - https://github.com/vrischmann/go-metrics-influxdb -* Ganglia - https://github.com/appscode/metlia -* Prometheus - https://github.com/deathowl/go-metrics-prometheus -* DataDog - https://github.com/syntaqx/go-metrics-datadog -* SignalFX - https://github.com/pascallouisperez/go-metrics-signalfx diff --git a/common/metrics/config.go b/common/metrics/config.go deleted file mode 100644 index bccc97ec5d6..00000000000 --- a/common/metrics/config.go +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2021 The Erigon Authors -// This file is part of Erigon. -// -// Erigon is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// Erigon is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with Erigon. If not, see . - -package metrics - -// Config contains the configuration for the metric collection. -type Config struct { //nolint:maligned - Enabled bool `toml:",omitempty"` - EnabledExpensive bool `toml:",omitempty"` - HTTP string `toml:",omitempty"` - Port int `toml:",omitempty"` -} - -// DefaultConfig is the default config for metrics used in go-ethereum. -var DefaultConfig = Config{ - Enabled: false, - EnabledExpensive: false, - HTTP: "127.0.0.1", - Port: 6061, -} diff --git a/common/metrics/counter.go b/common/metrics/counter.go deleted file mode 100644 index c884e9a1784..00000000000 --- a/common/metrics/counter.go +++ /dev/null @@ -1,55 +0,0 @@ -package metrics - -import ( - "sync/atomic" -) - -// GetOrRegisterCounter returns an existing Counter or constructs and registers -// a new Counter. -func GetOrRegisterCounter(name string, r Registry) *Counter { - return getOrRegister(name, NewCounter, r) -} - -// NewCounter constructs a new Counter. -func NewCounter() *Counter { - return new(Counter) -} - -// NewRegisteredCounter constructs and registers a new Counter. -func NewRegisteredCounter(name string, r Registry) *Counter { - c := NewCounter() - if r == nil { - r = DefaultRegistry - } - r.Register(name, c) - return c -} - -// CounterSnapshot is a read-only copy of a Counter. -type CounterSnapshot int64 - -// Count returns the count at the time the snapshot was taken. -func (c CounterSnapshot) Count() int64 { return int64(c) } - -// Counter hold an int64 value that can be incremented and decremented. -type Counter atomic.Int64 - -// Clear sets the counter to zero. -func (c *Counter) Clear() { - (*atomic.Int64)(c).Store(0) -} - -// Dec decrements the counter by the given amount. -func (c *Counter) Dec(i int64) { - (*atomic.Int64)(c).Add(-i) -} - -// Inc increments the counter by the given amount. -func (c *Counter) Inc(i int64) { - (*atomic.Int64)(c).Add(i) -} - -// Snapshot returns a read-only copy of the counter. -func (c *Counter) Snapshot() CounterSnapshot { - return CounterSnapshot((*atomic.Int64)(c).Load()) -} diff --git a/common/metrics/counter_float64.go b/common/metrics/counter_float64.go deleted file mode 100644 index 6cc73d89a29..00000000000 --- a/common/metrics/counter_float64.go +++ /dev/null @@ -1,66 +0,0 @@ -package metrics - -import ( - "math" - "sync/atomic" -) - -// GetOrRegisterCounterFloat64 returns an existing *CounterFloat64 or constructs and registers -// a new CounterFloat64. -func GetOrRegisterCounterFloat64(name string, r Registry) *CounterFloat64 { - return getOrRegister(name, NewCounterFloat64, r) -} - -// NewCounterFloat64 constructs a new CounterFloat64. -func NewCounterFloat64() *CounterFloat64 { - return new(CounterFloat64) -} - -// NewRegisteredCounterFloat64 constructs and registers a new CounterFloat64. -func NewRegisteredCounterFloat64(name string, r Registry) *CounterFloat64 { - c := NewCounterFloat64() - if r == nil { - r = DefaultRegistry - } - r.Register(name, c) - return c -} - -// CounterFloat64Snapshot is a read-only copy of a float64 counter. -type CounterFloat64Snapshot float64 - -// Count returns the value at the time the snapshot was taken. -func (c CounterFloat64Snapshot) Count() float64 { return float64(c) } - -// CounterFloat64 holds a float64 value that can be incremented and decremented. -type CounterFloat64 atomic.Uint64 - -// Clear sets the counter to zero. -func (c *CounterFloat64) Clear() { - (*atomic.Uint64)(c).Store(0) -} - -// Dec decrements the counter by the given amount. -func (c *CounterFloat64) Dec(v float64) { - atomicAddFloat((*atomic.Uint64)(c), -v) -} - -// Inc increments the counter by the given amount. -func (c *CounterFloat64) Inc(v float64) { - atomicAddFloat((*atomic.Uint64)(c), v) -} - -// Snapshot returns a read-only copy of the counter. -func (c *CounterFloat64) Snapshot() CounterFloat64Snapshot { - return CounterFloat64Snapshot(math.Float64frombits((*atomic.Uint64)(c).Load())) -} - -func atomicAddFloat(fbits *atomic.Uint64, v float64) { - for { - loadedBits := fbits.Load() - newBits := math.Float64bits(math.Float64frombits(loadedBits) + v) - if fbits.CompareAndSwap(loadedBits, newBits) { - break - } - } -} diff --git a/common/metrics/counter_float_64_test.go b/common/metrics/counter_float_64_test.go deleted file mode 100644 index 618cbbbc2b0..00000000000 --- a/common/metrics/counter_float_64_test.go +++ /dev/null @@ -1,73 +0,0 @@ -package metrics - -import ( - "sync" - "testing" -) - -func BenchmarkCounterFloat64(b *testing.B) { - c := NewCounterFloat64() - b.ResetTimer() - for i := 0; i < b.N; i++ { - c.Inc(1.0) - } -} - -func BenchmarkCounterFloat64Parallel(b *testing.B) { - c := NewCounterFloat64() - b.ResetTimer() - var wg sync.WaitGroup - for i := 0; i < 10; i++ { - wg.Add(1) - go func() { - for i := 0; i < b.N; i++ { - c.Inc(1.0) - } - wg.Done() - }() - } - wg.Wait() - if have, want := c.Snapshot().Count(), 10.0*float64(b.N); have != want { - b.Fatalf("have %f want %f", have, want) - } -} - -func TestCounterFloat64(t *testing.T) { - c := NewCounterFloat64() - if count := c.Snapshot().Count(); count != 0 { - t.Errorf("wrong count: %v", count) - } - c.Dec(1.0) - if count := c.Snapshot().Count(); count != -1.0 { - t.Errorf("wrong count: %v", count) - } - snapshot := c.Snapshot() - c.Dec(2.0) - if count := c.Snapshot().Count(); count != -3.0 { - t.Errorf("wrong count: %v", count) - } - c.Inc(1.0) - if count := c.Snapshot().Count(); count != -2.0 { - t.Errorf("wrong count: %v", count) - } - c.Inc(2.0) - if count := c.Snapshot().Count(); count != 0.0 { - t.Errorf("wrong count: %v", count) - } - if count := snapshot.Count(); count != -1.0 { - t.Errorf("snapshot count wrong: %v", count) - } - c.Inc(1.0) - c.Clear() - if count := c.Snapshot().Count(); count != 0.0 { - t.Errorf("wrong count: %v", count) - } -} - -func TestGetOrRegisterCounterFloat64(t *testing.T) { - r := NewRegistry() - NewRegisteredCounterFloat64("foo", r).Inc(47.0) - if c := GetOrRegisterCounterFloat64("foo", r).Snapshot(); c.Count() != 47.0 { - t.Fatal(c) - } -} diff --git a/common/metrics/counter_test.go b/common/metrics/counter_test.go deleted file mode 100644 index bf0ca6bae44..00000000000 --- a/common/metrics/counter_test.go +++ /dev/null @@ -1,61 +0,0 @@ -package metrics - -import "testing" - -func BenchmarkCounter(b *testing.B) { - c := NewCounter() - b.ResetTimer() - for i := 0; i < b.N; i++ { - c.Inc(1) - } -} - -func TestCounterClear(t *testing.T) { - c := NewCounter() - c.Inc(1) - c.Clear() - if count := c.Snapshot().Count(); count != 0 { - t.Errorf("c.Count(): 0 != %v\n", count) - } -} - -func TestCounter(t *testing.T) { - c := NewCounter() - if count := c.Snapshot().Count(); count != 0 { - t.Errorf("wrong count: %v", count) - } - c.Dec(1) - if count := c.Snapshot().Count(); count != -1 { - t.Errorf("wrong count: %v", count) - } - c.Dec(2) - if count := c.Snapshot().Count(); count != -3 { - t.Errorf("wrong count: %v", count) - } - c.Inc(1) - if count := c.Snapshot().Count(); count != -2 { - t.Errorf("wrong count: %v", count) - } - c.Inc(2) - if count := c.Snapshot().Count(); count != 0 { - t.Errorf("wrong count: %v", count) - } -} - -func TestCounterSnapshot(t *testing.T) { - c := NewCounter() - c.Inc(1) - snapshot := c.Snapshot() - c.Inc(1) - if count := snapshot.Count(); count != 1 { - t.Errorf("c.Count(): 1 != %v\n", count) - } -} - -func TestGetOrRegisterCounter(t *testing.T) { - r := NewRegistry() - NewRegisteredCounter("foo", r).Inc(47) - if c := GetOrRegisterCounter("foo", r).Snapshot(); c.Count() != 47 { - t.Fatal(c) - } -} diff --git a/common/metrics/cpu.go b/common/metrics/cpu.go deleted file mode 100644 index 3a49cd42493..00000000000 --- a/common/metrics/cpu.go +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2018 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package metrics - -// CPUStats is the system and process CPU stats. -// All values are in seconds. -type CPUStats struct { - GlobalTime float64 // Time spent by the CPU working on all processes - GlobalWait float64 // Time spent by waiting on disk for all processes - LocalTime float64 // Time spent by the CPU working on this process -} diff --git a/common/metrics/cpu_disabled.go b/common/metrics/cpu_disabled.go deleted file mode 100644 index 37c3e1b8f7f..00000000000 --- a/common/metrics/cpu_disabled.go +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright 2020 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -//go:build ios || js || wasip1 || tinygo -// +build ios js wasip1 tinygo - -package metrics - -// ReadCPUStats retrieves the current CPU stats. Internally this uses `gosigar`, -// which is not supported on the platforms in this file. -func ReadCPUStats(stats *CPUStats) {} diff --git a/common/metrics/cpu_enabled.go b/common/metrics/cpu_enabled.go deleted file mode 100644 index 7daadf1dc21..00000000000 --- a/common/metrics/cpu_enabled.go +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright 2020 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -//go:build !ios && !js && !wasip1 && !tinygo -// +build !ios,!js,!wasip1,!tinygo - -package metrics - -import ( - "github.com/erigontech/erigon/common/log/v3" - "github.com/shirou/gopsutil/cpu" -) - -// ReadCPUStats retrieves the current CPU stats. -func ReadCPUStats(stats *CPUStats) { - // passing false to request all cpu times - timeStats, err := cpu.Times(false) - if err != nil { - log.Error("Could not read cpu stats", "err", err) - return - } - if len(timeStats) == 0 { - log.Error("Empty cpu stats") - return - } - // requesting all cpu times will always return an array with only one time stats entry - timeStat := timeStats[0] - stats.GlobalTime = timeStat.User + timeStat.Nice + timeStat.System - stats.GlobalWait = timeStat.Iowait - stats.LocalTime = getProcessCPUTime() -} diff --git a/common/metrics/cputime_nop.go b/common/metrics/cputime_nop.go deleted file mode 100644 index a6285ec10ad..00000000000 --- a/common/metrics/cputime_nop.go +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright 2018 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -//go:build windows || js || tinygo -// +build windows js tinygo - -package metrics - -// getProcessCPUTime returns 0 on Windows as there is no system call to resolve -// the actual process' CPU time. -func getProcessCPUTime() float64 { - return 0 -} diff --git a/common/metrics/cputime_unix.go b/common/metrics/cputime_unix.go deleted file mode 100644 index 970d3076747..00000000000 --- a/common/metrics/cputime_unix.go +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2018 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -//go:build !windows && !js && !wasip1 && !tinygo -// +build !windows,!js,!wasip1,!tinygo - -package metrics - -import ( - syscall "golang.org/x/sys/unix" - - "github.com/erigontech/erigon/common/log/v3" -) - -// getProcessCPUTime retrieves the process' CPU time since program startup. -func getProcessCPUTime() float64 { - var usage syscall.Rusage - if err := syscall.Getrusage(syscall.RUSAGE_SELF, &usage); err != nil { - log.Warn("Failed to retrieve CPU time", "err", err) - return 0 - } - return float64(usage.Utime.Sec+usage.Stime.Sec) + float64(usage.Utime.Usec+usage.Stime.Usec)/1000000 //nolint:unconvert -} diff --git a/common/metrics/debug.go b/common/metrics/debug.go deleted file mode 100644 index 5d0d3992f10..00000000000 --- a/common/metrics/debug.go +++ /dev/null @@ -1,76 +0,0 @@ -package metrics - -import ( - "runtime/debug" - "time" -) - -var ( - debugMetrics struct { - GCStats struct { - LastGC *Gauge - NumGC *Gauge - Pause Histogram - //PauseQuantiles Histogram - PauseTotal *Gauge - } - ReadGCStats *Timer - } - gcStats debug.GCStats -) - -// CaptureDebugGCStats captures new values for the Go garbage collector statistics -// exported in debug.GCStats. This is designed to be called as a goroutine. -func CaptureDebugGCStats(r Registry, d time.Duration) { - for range time.Tick(d) { - CaptureDebugGCStatsOnce(r) - } -} - -// CaptureDebugGCStatsOnce captures new values for the Go garbage collector -// statistics exported in debug.GCStats. This is designed to be called in -// a background goroutine. Giving a registry which has not been given to -// RegisterDebugGCStats will panic. -// -// Be careful (but much less so) with this because debug.ReadGCStats calls -// the C function runtime·lock(runtime·mheap) which, while not a stop-the-world -// operation, isn't something you want to be doing all the time. -func CaptureDebugGCStatsOnce(r Registry) { - lastGC := gcStats.LastGC - t := time.Now() - debug.ReadGCStats(&gcStats) - debugMetrics.ReadGCStats.UpdateSince(t) - - debugMetrics.GCStats.LastGC.Update(gcStats.LastGC.UnixNano()) - debugMetrics.GCStats.NumGC.Update(gcStats.NumGC) - if lastGC != gcStats.LastGC && 0 < len(gcStats.Pause) { - debugMetrics.GCStats.Pause.Update(int64(gcStats.Pause[0])) - } - //debugMetrics.GCStats.PauseQuantiles.Update(gcStats.PauseQuantiles) - debugMetrics.GCStats.PauseTotal.Update(int64(gcStats.PauseTotal)) -} - -// RegisterDebugGCStats registers metrics for the Go garbage collector statistics -// exported in debug.GCStats. The metrics are named by their fully-qualified Go -// symbols, i.e. debug.GCStats.PauseTotal. -func RegisterDebugGCStats(r Registry) { - debugMetrics.GCStats.LastGC = NewGauge() - debugMetrics.GCStats.NumGC = NewGauge() - debugMetrics.GCStats.Pause = NewHistogram(NewExpDecaySample(1028, 0.015)) - //debugMetrics.GCStats.PauseQuantiles = NewHistogram(NewExpDecaySample(1028, 0.015)) - debugMetrics.GCStats.PauseTotal = NewGauge() - debugMetrics.ReadGCStats = NewTimer() - - r.Register("debug.GCStats.LastGC", debugMetrics.GCStats.LastGC) - r.Register("debug.GCStats.NumGC", debugMetrics.GCStats.NumGC) - r.Register("debug.GCStats.Pause", debugMetrics.GCStats.Pause) - //r.Register("debug.GCStats.PauseQuantiles", debugMetrics.GCStats.PauseQuantiles) - r.Register("debug.GCStats.PauseTotal", debugMetrics.GCStats.PauseTotal) - r.Register("debug.ReadGCStats", debugMetrics.ReadGCStats) -} - -// Allocate an initial slice for gcStats.Pause to avoid allocations during -// normal operation. -func init() { - gcStats.Pause = make([]time.Duration, 11) -} diff --git a/common/metrics/debug_test.go b/common/metrics/debug_test.go deleted file mode 100644 index 07eb8678416..00000000000 --- a/common/metrics/debug_test.go +++ /dev/null @@ -1,48 +0,0 @@ -package metrics - -import ( - "runtime" - "runtime/debug" - "testing" - "time" -) - -func BenchmarkDebugGCStats(b *testing.B) { - r := NewRegistry() - RegisterDebugGCStats(r) - b.ResetTimer() - for i := 0; i < b.N; i++ { - CaptureDebugGCStatsOnce(r) - } -} - -func TestDebugGCStatsBlocking(t *testing.T) { - if g := runtime.GOMAXPROCS(0); g < 2 { - t.Skipf("skipping TestDebugGCMemStatsBlocking with GOMAXPROCS=%d\n", g) - return - } - ch := make(chan int) - go testDebugGCStatsBlocking(ch) - var gcStats debug.GCStats - t0 := time.Now() - debug.ReadGCStats(&gcStats) - t1 := time.Now() - t.Log("i++ during debug.ReadGCStats:", <-ch) - go testDebugGCStatsBlocking(ch) - d := t1.Sub(t0) - t.Log(d) - time.Sleep(d) - t.Log("i++ during time.Sleep:", <-ch) -} - -func testDebugGCStatsBlocking(ch chan int) { - i := 0 - for { - select { - case ch <- i: - return - default: - i++ - } - } -} diff --git a/common/metrics/disk.go b/common/metrics/disk.go deleted file mode 100644 index 25142d2ad1e..00000000000 --- a/common/metrics/disk.go +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package metrics - -// DiskStats is the per process disk io stats. -type DiskStats struct { - ReadCount int64 // Number of read operations executed - ReadBytes int64 // Total number of bytes read - WriteCount int64 // Number of write operations executed - WriteBytes int64 // Total number of byte written -} diff --git a/common/metrics/disk_linux.go b/common/metrics/disk_linux.go deleted file mode 100644 index 8d610cd6749..00000000000 --- a/common/metrics/disk_linux.go +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -// Contains the Linux implementation of process disk IO counter retrieval. - -package metrics - -import ( - "bufio" - "fmt" - "io" - "os" - "strconv" - "strings" -) - -// ReadDiskStats retrieves the disk IO stats belonging to the current process. -func ReadDiskStats(stats *DiskStats) error { - // Open the process disk IO counter file - inf, err := os.Open(fmt.Sprintf("/proc/%d/io", os.Getpid())) - if err != nil { - return err - } - defer inf.Close() - in := bufio.NewReader(inf) - - // Iterate over the IO counter, and extract what we need - for { - // Read the next line and split to key and value - line, err := in.ReadString('\n') - if err != nil { - if err == io.EOF { - return nil - } - return err - } - parts := strings.Split(line, ":") - if len(parts) != 2 { - continue - } - key := strings.TrimSpace(parts[0]) - value, err := strconv.ParseInt(strings.TrimSpace(parts[1]), 10, 64) - if err != nil { - return err - } - - // Update the counter based on the key - switch key { - case "syscr": - stats.ReadCount = value - case "syscw": - stats.WriteCount = value - case "rchar": - stats.ReadBytes = value - case "wchar": - stats.WriteBytes = value - } - } -} diff --git a/common/metrics/disk_nop.go b/common/metrics/disk_nop.go deleted file mode 100644 index 41bbe9adb2d..00000000000 --- a/common/metrics/disk_nop.go +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -//go:build !linux -// +build !linux - -package metrics - -import "errors" - -// ReadDiskStats retrieves the disk IO stats belonging to the current process. -func ReadDiskStats(stats *DiskStats) error { - return errors.New("not implemented") -} diff --git a/common/metrics/ewma.go b/common/metrics/ewma.go deleted file mode 100644 index 194527a7989..00000000000 --- a/common/metrics/ewma.go +++ /dev/null @@ -1,91 +0,0 @@ -package metrics - -import ( - "math" - "sync" - "sync/atomic" - "time" -) - -// EWMASnapshot is a read-only copy of an EWMA. -type EWMASnapshot float64 - -// Rate returns the rate of events per second at the time the snapshot was -// taken. -func (a EWMASnapshot) Rate() float64 { return float64(a) } - -// NewEWMA constructs a new EWMA with the given alpha. -func NewEWMA(alpha float64) *EWMA { - return &EWMA{alpha: alpha} -} - -// NewEWMA1 constructs a new EWMA for a one-minute moving average. -func NewEWMA1() *EWMA { - return NewEWMA(1 - math.Exp(-5.0/60.0/1)) -} - -// NewEWMA5 constructs a new EWMA for a five-minute moving average. -func NewEWMA5() *EWMA { - return NewEWMA(1 - math.Exp(-5.0/60.0/5)) -} - -// NewEWMA15 constructs a new EWMA for a fifteen-minute moving average. -func NewEWMA15() *EWMA { - return NewEWMA(1 - math.Exp(-5.0/60.0/15)) -} - -// EWMA continuously calculate an exponentially-weighted moving average -// based on an outside source of clock ticks. -type EWMA struct { - uncounted atomic.Int64 - alpha float64 - rate atomic.Uint64 - init atomic.Bool - mutex sync.Mutex -} - -// Snapshot returns a read-only copy of the EWMA. -func (a *EWMA) Snapshot() EWMASnapshot { - r := math.Float64frombits(a.rate.Load()) * float64(time.Second) - return EWMASnapshot(r) -} - -// tick ticks the clock to update the moving average. It assumes it is called -// every five seconds. -func (a *EWMA) tick() { - // Optimization to avoid mutex locking in the hot-path. - if a.init.Load() { - a.updateRate(a.fetchInstantRate()) - return - } - // Slow-path: this is only needed on the first tick() and preserves transactional updating - // of init and rate in the else block. The first conditional is needed below because - // a different thread could have set a.init = 1 between the time of the first atomic load and when - // the lock was acquired. - a.mutex.Lock() - if a.init.Load() { - // The fetchInstantRate() uses atomic loading, which is unnecessary in this critical section - // but again, this section is only invoked on the first successful tick() operation. - a.updateRate(a.fetchInstantRate()) - } else { - a.init.Store(true) - a.rate.Store(math.Float64bits(a.fetchInstantRate())) - } - a.mutex.Unlock() -} - -func (a *EWMA) fetchInstantRate() float64 { - count := a.uncounted.Swap(0) - return float64(count) / float64(5*time.Second) -} - -func (a *EWMA) updateRate(instantRate float64) { - currentRate := math.Float64frombits(a.rate.Load()) - currentRate += a.alpha * (instantRate - currentRate) - a.rate.Store(math.Float64bits(currentRate)) -} - -// Update adds n uncounted events. -func (a *EWMA) Update(n int64) { - a.uncounted.Add(n) -} diff --git a/common/metrics/ewma_test.go b/common/metrics/ewma_test.go deleted file mode 100644 index 4b9bde3a4b3..00000000000 --- a/common/metrics/ewma_test.go +++ /dev/null @@ -1,89 +0,0 @@ -package metrics - -import ( - "math" - "testing" -) - -const epsilon = 0.0000000000000001 - -func BenchmarkEWMA(b *testing.B) { - a := NewEWMA1() - b.ResetTimer() - for i := 0; i < b.N; i++ { - a.Update(1) - a.tick() - } -} - -func BenchmarkEWMAParallel(b *testing.B) { - a := NewEWMA1() - b.ResetTimer() - - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - a.Update(1) - a.tick() - } - }) -} - -func TestEWMA1(t *testing.T) { - a := NewEWMA1() - a.Update(3) - a.tick() - for i, want := range []float64{0.6, - 0.22072766470286553, 0.08120116994196772, 0.029872241020718428, - 0.01098938333324054, 0.004042768199451294, 0.0014872513059998212, - 0.0005471291793327122, 0.00020127757674150815, 7.404588245200814e-05, - 2.7239957857491083e-05, 1.0021020474147462e-05, 3.6865274119969525e-06, - 1.3561976441886433e-06, 4.989172314621449e-07, 1.8354139230109722e-07, - } { - if rate := a.Snapshot().Rate(); math.Abs(want-rate) > epsilon { - t.Errorf("%d minute a.Snapshot().Rate(): %f != %v\n", i, want, rate) - } - elapseMinute(a) - } -} - -func TestEWMA5(t *testing.T) { - a := NewEWMA5() - a.Update(3) - a.tick() - for i, want := range []float64{ - 0.6, 0.49123845184678905, 0.4021920276213837, 0.32928698165641596, - 0.269597378470333, 0.2207276647028654, 0.18071652714732128, - 0.14795817836496392, 0.12113791079679326, 0.09917933293295193, - 0.08120116994196763, 0.06648189501740036, 0.05443077197364752, - 0.04456414692860035, 0.03648603757513079, 0.0298722410207183831020718428, - } { - if rate := a.Snapshot().Rate(); math.Abs(want-rate) > epsilon { - t.Errorf("%d minute a.Snapshot().Rate(): %f != %v\n", i, want, rate) - } - elapseMinute(a) - } -} - -func TestEWMA15(t *testing.T) { - a := NewEWMA15() - a.Update(3) - a.tick() - for i, want := range []float64{ - 0.6, 0.5613041910189706, 0.5251039914257684, 0.4912384518467888184678905, - 0.459557003018789, 0.4299187863442732, 0.4021920276213831, - 0.37625345116383313, 0.3519877317060185, 0.3292869816564153165641596, - 0.3080502714195546, 0.2881831806538789, 0.26959737847033216, - 0.2522102307052083, 0.23594443252115815, 0.2207276647028646247028654470286553, - } { - if rate := a.Snapshot().Rate(); math.Abs(want-rate) > epsilon { - t.Errorf("%d minute a.Snapshot().Rate(): %f != %v\n", i, want, rate) - } - elapseMinute(a) - } -} - -func elapseMinute(a *EWMA) { - for i := 0; i < 12; i++ { - a.tick() - } -} diff --git a/common/metrics/gauge.go b/common/metrics/gauge.go deleted file mode 100644 index 20de95255bd..00000000000 --- a/common/metrics/gauge.go +++ /dev/null @@ -1,67 +0,0 @@ -package metrics - -import "sync/atomic" - -// GaugeSnapshot is a read-only copy of a Gauge. -type GaugeSnapshot int64 - -// Value returns the value at the time the snapshot was taken. -func (g GaugeSnapshot) Value() int64 { return int64(g) } - -// GetOrRegisterGauge returns an existing Gauge or constructs and registers a -// new Gauge. -func GetOrRegisterGauge(name string, r Registry) *Gauge { - return getOrRegister(name, NewGauge, r) -} - -// NewGauge constructs a new Gauge. -func NewGauge() *Gauge { - return &Gauge{} -} - -// NewRegisteredGauge constructs and registers a new Gauge. -func NewRegisteredGauge(name string, r Registry) *Gauge { - c := NewGauge() - if r == nil { - r = DefaultRegistry - } - r.Register(name, c) - return c -} - -// Gauge holds an int64 value that can be set arbitrarily. -type Gauge atomic.Int64 - -// Snapshot returns a read-only copy of the gauge. -func (g *Gauge) Snapshot() GaugeSnapshot { - return GaugeSnapshot((*atomic.Int64)(g).Load()) -} - -// Update updates the gauge's value. -func (g *Gauge) Update(v int64) { - (*atomic.Int64)(g).Store(v) -} - -// UpdateIfGt updates the gauge's value if v is larger then the current value. -func (g *Gauge) UpdateIfGt(v int64) { - value := (*atomic.Int64)(g) - for { - exist := value.Load() - if exist >= v { - break - } - if value.CompareAndSwap(exist, v) { - break - } - } -} - -// Dec decrements the gauge's current value by the given amount. -func (g *Gauge) Dec(i int64) { - (*atomic.Int64)(g).Add(-i) -} - -// Inc increments the gauge's current value by the given amount. -func (g *Gauge) Inc(i int64) { - (*atomic.Int64)(g).Add(i) -} diff --git a/common/metrics/gauge_float64.go b/common/metrics/gauge_float64.go deleted file mode 100644 index 48524e4c3f7..00000000000 --- a/common/metrics/gauge_float64.go +++ /dev/null @@ -1,47 +0,0 @@ -package metrics - -import ( - "math" - "sync/atomic" -) - -// GetOrRegisterGaugeFloat64 returns an existing GaugeFloat64 or constructs and registers a -// new GaugeFloat64. -func GetOrRegisterGaugeFloat64(name string, r Registry) *GaugeFloat64 { - return getOrRegister(name, NewGaugeFloat64, r) -} - -// GaugeFloat64Snapshot is a read-only copy of a GaugeFloat64. -type GaugeFloat64Snapshot float64 - -// Value returns the value at the time the snapshot was taken. -func (g GaugeFloat64Snapshot) Value() float64 { return float64(g) } - -// NewGaugeFloat64 constructs a new GaugeFloat64. -func NewGaugeFloat64() *GaugeFloat64 { - return new(GaugeFloat64) -} - -// NewRegisteredGaugeFloat64 constructs and registers a new GaugeFloat64. -func NewRegisteredGaugeFloat64(name string, r Registry) *GaugeFloat64 { - c := NewGaugeFloat64() - if nil == r { - r = DefaultRegistry - } - r.Register(name, c) - return c -} - -// GaugeFloat64 hold a float64 value that can be set arbitrarily. -type GaugeFloat64 atomic.Uint64 - -// Snapshot returns a read-only copy of the gauge. -func (g *GaugeFloat64) Snapshot() GaugeFloat64Snapshot { - v := math.Float64frombits((*atomic.Uint64)(g).Load()) - return GaugeFloat64Snapshot(v) -} - -// Update updates the gauge's value. -func (g *GaugeFloat64) Update(v float64) { - (*atomic.Uint64)(g).Store(math.Float64bits(v)) -} diff --git a/common/metrics/gauge_float64_test.go b/common/metrics/gauge_float64_test.go deleted file mode 100644 index 194a18821f8..00000000000 --- a/common/metrics/gauge_float64_test.go +++ /dev/null @@ -1,51 +0,0 @@ -package metrics - -import ( - "sync" - "testing" -) - -func BenchmarkGaugeFloat64(b *testing.B) { - g := NewGaugeFloat64() - b.ResetTimer() - for i := 0; i < b.N; i++ { - g.Update(float64(i)) - } -} - -func BenchmarkGaugeFloat64Parallel(b *testing.B) { - c := NewGaugeFloat64() - var wg sync.WaitGroup - for i := 0; i < 10; i++ { - wg.Add(1) - go func() { - for i := 0; i < b.N; i++ { - c.Update(float64(i)) - } - wg.Done() - }() - } - wg.Wait() - if have, want := c.Snapshot().Value(), float64(b.N-1); have != want { - b.Fatalf("have %f want %f", have, want) - } -} - -func TestGaugeFloat64Snapshot(t *testing.T) { - g := NewGaugeFloat64() - g.Update(47.0) - snapshot := g.Snapshot() - g.Update(float64(0)) - if v := snapshot.Value(); v != 47.0 { - t.Errorf("g.Value(): 47.0 != %v\n", v) - } -} - -func TestGetOrRegisterGaugeFloat64(t *testing.T) { - r := NewRegistry() - NewRegisteredGaugeFloat64("foo", r).Update(47.0) - t.Logf("registry: %v", r) - if g := GetOrRegisterGaugeFloat64("foo", r).Snapshot(); g.Value() != 47.0 { - t.Fatal(g) - } -} diff --git a/common/metrics/gauge_info.go b/common/metrics/gauge_info.go deleted file mode 100644 index 34ac9179194..00000000000 --- a/common/metrics/gauge_info.go +++ /dev/null @@ -1,61 +0,0 @@ -package metrics - -import ( - "encoding/json" - "sync" -) - -// GaugeInfoValue is a mapping of keys to values -type GaugeInfoValue map[string]string - -func (val GaugeInfoValue) String() string { - data, _ := json.Marshal(val) - return string(data) -} - -// GetOrRegisterGaugeInfo returns an existing GaugeInfo or constructs and registers a -// new GaugeInfo. -func GetOrRegisterGaugeInfo(name string, r Registry) *GaugeInfo { - return getOrRegister(name, NewGaugeInfo, r) -} - -// NewGaugeInfo constructs a new GaugeInfo. -func NewGaugeInfo() *GaugeInfo { - return &GaugeInfo{ - value: GaugeInfoValue{}, - } -} - -// NewRegisteredGaugeInfo constructs and registers a new GaugeInfo. -func NewRegisteredGaugeInfo(name string, r Registry) *GaugeInfo { - c := NewGaugeInfo() - if nil == r { - r = DefaultRegistry - } - r.Register(name, c) - return c -} - -// GaugeInfoSnapshot is a read-only copy of another GaugeInfo. -type GaugeInfoSnapshot GaugeInfoValue - -// Value returns the value at the time the snapshot was taken. -func (g GaugeInfoSnapshot) Value() GaugeInfoValue { return GaugeInfoValue(g) } - -// GaugeInfo maintains a set of key/value mappings. -type GaugeInfo struct { - mutex sync.Mutex - value GaugeInfoValue -} - -// Snapshot returns a read-only copy of the gauge. -func (g *GaugeInfo) Snapshot() GaugeInfoSnapshot { - return GaugeInfoSnapshot(g.value) -} - -// Update updates the gauge's value. -func (g *GaugeInfo) Update(v GaugeInfoValue) { - g.mutex.Lock() - defer g.mutex.Unlock() - g.value = v -} diff --git a/common/metrics/gauge_info_test.go b/common/metrics/gauge_info_test.go deleted file mode 100644 index 319afbf92e8..00000000000 --- a/common/metrics/gauge_info_test.go +++ /dev/null @@ -1,36 +0,0 @@ -package metrics - -import ( - "testing" -) - -func TestGaugeInfoJsonString(t *testing.T) { - g := NewGaugeInfo() - g.Update(GaugeInfoValue{ - "chain_id": "5", - "anotherKey": "any_string_value", - "third_key": "anything", - }, - ) - want := `{"anotherKey":"any_string_value","chain_id":"5","third_key":"anything"}` - - original := g.Snapshot() - g.Update(GaugeInfoValue{"value": "updated"}) - - if have := original.Value().String(); have != want { - t.Errorf("\nhave: %v\nwant: %v\n", have, want) - } - if have, want := g.Snapshot().Value().String(), `{"value":"updated"}`; have != want { - t.Errorf("\nhave: %v\nwant: %v\n", have, want) - } -} - -func TestGetOrRegisterGaugeInfo(t *testing.T) { - r := NewRegistry() - NewRegisteredGaugeInfo("foo", r).Update( - GaugeInfoValue{"chain_id": "5"}) - g := GetOrRegisterGaugeInfo("foo", r).Snapshot() - if have, want := g.Value().String(), `{"chain_id":"5"}`; have != want { - t.Errorf("have\n%v\nwant\n%v\n", have, want) - } -} diff --git a/common/metrics/gauge_test.go b/common/metrics/gauge_test.go deleted file mode 100644 index f2ba930bc46..00000000000 --- a/common/metrics/gauge_test.go +++ /dev/null @@ -1,31 +0,0 @@ -package metrics - -import ( - "testing" -) - -func BenchmarkGauge(b *testing.B) { - g := NewGauge() - b.ResetTimer() - for i := 0; i < b.N; i++ { - g.Update(int64(i)) - } -} - -func TestGaugeSnapshot(t *testing.T) { - g := NewGauge() - g.Update(int64(47)) - snapshot := g.Snapshot() - g.Update(int64(0)) - if v := snapshot.Value(); v != 47 { - t.Errorf("g.Value(): 47 != %v\n", v) - } -} - -func TestGetOrRegisterGauge(t *testing.T) { - r := NewRegistry() - NewRegisteredGauge("foo", r).Update(47) - if g := GetOrRegisterGauge("foo", r); g.Snapshot().Value() != 47 { - t.Fatal(g) - } -} diff --git a/common/metrics/healthcheck.go b/common/metrics/healthcheck.go deleted file mode 100644 index 435e5e0bf93..00000000000 --- a/common/metrics/healthcheck.go +++ /dev/null @@ -1,35 +0,0 @@ -package metrics - -// NewHealthcheck constructs a new Healthcheck which will use the given -// function to update its status. -func NewHealthcheck(f func(*Healthcheck)) *Healthcheck { - return &Healthcheck{nil, f} -} - -// Healthcheck is the standard implementation of a Healthcheck and -// stores the status and a function to call to update the status. -type Healthcheck struct { - err error - f func(*Healthcheck) -} - -// Check runs the healthcheck function to update the healthcheck's status. -func (h *Healthcheck) Check() { - h.f(h) -} - -// Error returns the healthcheck's status, which will be nil if it is healthy. -func (h *Healthcheck) Error() error { - return h.err -} - -// Healthy marks the healthcheck as healthy. -func (h *Healthcheck) Healthy() { - h.err = nil -} - -// Unhealthy marks the healthcheck as unhealthy. The error is stored and -// may be retrieved by the Error method. -func (h *Healthcheck) Unhealthy(err error) { - h.err = err -} diff --git a/common/metrics/histogram.go b/common/metrics/histogram.go deleted file mode 100644 index 18bf6e3d2b7..00000000000 --- a/common/metrics/histogram.go +++ /dev/null @@ -1,66 +0,0 @@ -package metrics - -type HistogramSnapshot interface { - Count() int64 - Max() int64 - Mean() float64 - Min() int64 - Percentile(float64) float64 - Percentiles([]float64) []float64 - Size() int - StdDev() float64 - Sum() int64 - Variance() float64 -} - -// Histogram calculates distribution statistics from a series of int64 values. -type Histogram interface { - Clear() - Update(int64) - Snapshot() HistogramSnapshot -} - -// GetOrRegisterHistogram returns an existing Histogram or constructs and -// registers a new StandardHistogram. -func GetOrRegisterHistogram(name string, r Registry, s Sample) Histogram { - return getOrRegister(name, func() Histogram { return NewHistogram(s) }, r) -} - -// GetOrRegisterHistogramLazy returns an existing Histogram or constructs and -// registers a new StandardHistogram. -func GetOrRegisterHistogramLazy(name string, r Registry, s func() Sample) Histogram { - return getOrRegister(name, func() Histogram { return NewHistogram(s()) }, r) -} - -// NewHistogram constructs a new StandardHistogram from a Sample. -func NewHistogram(s Sample) Histogram { - return &StandardHistogram{s} -} - -// NewRegisteredHistogram constructs and registers a new StandardHistogram from -// a Sample. -func NewRegisteredHistogram(name string, r Registry, s Sample) Histogram { - c := NewHistogram(s) - if nil == r { - r = DefaultRegistry - } - r.Register(name, c) - return c -} - -// StandardHistogram is the standard implementation of a Histogram and uses a -// Sample to bound its memory use. -type StandardHistogram struct { - sample Sample -} - -// Clear clears the histogram and its sample. -func (h *StandardHistogram) Clear() { h.sample.Clear() } - -// Snapshot returns a read-only copy of the histogram. -func (h *StandardHistogram) Snapshot() HistogramSnapshot { - return h.sample.Snapshot() -} - -// Update samples a new value. -func (h *StandardHistogram) Update(v int64) { h.sample.Update(v) } diff --git a/common/metrics/histogram_test.go b/common/metrics/histogram_test.go deleted file mode 100644 index 22fc5468b0b..00000000000 --- a/common/metrics/histogram_test.go +++ /dev/null @@ -1,95 +0,0 @@ -package metrics - -import "testing" - -func BenchmarkHistogram(b *testing.B) { - h := NewHistogram(NewUniformSample(100)) - b.ResetTimer() - for i := 0; i < b.N; i++ { - h.Update(int64(i)) - } -} - -func TestGetOrRegisterHistogram(t *testing.T) { - r := NewRegistry() - s := NewUniformSample(100) - NewRegisteredHistogram("foo", r, s).Update(47) - if h := GetOrRegisterHistogram("foo", r, s).Snapshot(); h.Count() != 1 { - t.Fatal(h) - } -} - -func TestHistogram10000(t *testing.T) { - h := NewHistogram(NewUniformSample(100000)) - for i := 1; i <= 10000; i++ { - h.Update(int64(i)) - } - testHistogram10000(t, h.Snapshot()) -} - -func TestHistogramEmpty(t *testing.T) { - h := NewHistogram(NewUniformSample(100)).Snapshot() - if count := h.Count(); count != 0 { - t.Errorf("h.Count(): 0 != %v\n", count) - } - if min := h.Min(); min != 0 { - t.Errorf("h.Min(): 0 != %v\n", min) - } - if max := h.Max(); max != 0 { - t.Errorf("h.Max(): 0 != %v\n", max) - } - if mean := h.Mean(); mean != 0.0 { - t.Errorf("h.Mean(): 0.0 != %v\n", mean) - } - if stdDev := h.StdDev(); stdDev != 0.0 { - t.Errorf("h.StdDev(): 0.0 != %v\n", stdDev) - } - ps := h.Percentiles([]float64{0.5, 0.75, 0.99}) - if ps[0] != 0.0 { - t.Errorf("median: 0.0 != %v\n", ps[0]) - } - if ps[1] != 0.0 { - t.Errorf("75th percentile: 0.0 != %v\n", ps[1]) - } - if ps[2] != 0.0 { - t.Errorf("99th percentile: 0.0 != %v\n", ps[2]) - } -} - -func TestHistogramSnapshot(t *testing.T) { - h := NewHistogram(NewUniformSample(100000)) - for i := 1; i <= 10000; i++ { - h.Update(int64(i)) - } - snapshot := h.Snapshot() - h.Update(0) - testHistogram10000(t, snapshot) -} - -func testHistogram10000(t *testing.T, h HistogramSnapshot) { - if count := h.Count(); count != 10000 { - t.Errorf("h.Count(): 10000 != %v\n", count) - } - if min := h.Min(); min != 1 { - t.Errorf("h.Min(): 1 != %v\n", min) - } - if max := h.Max(); max != 10000 { - t.Errorf("h.Max(): 10000 != %v\n", max) - } - if mean := h.Mean(); mean != 5000.5 { - t.Errorf("h.Mean(): 5000.5 != %v\n", mean) - } - if stdDev := h.StdDev(); stdDev != 2886.751331514372 { - t.Errorf("h.StdDev(): 2886.751331514372 != %v\n", stdDev) - } - ps := h.Percentiles([]float64{0.5, 0.75, 0.99}) - if ps[0] != 5000.5 { - t.Errorf("median: 5000.5 != %v\n", ps[0]) - } - if ps[1] != 7500.75 { - t.Errorf("75th percentile: 7500.75 != %v\n", ps[1]) - } - if ps[2] != 9900.99 { - t.Errorf("99th percentile: 9900.99 != %v\n", ps[2]) - } -} diff --git a/common/metrics/init_test.go b/common/metrics/init_test.go deleted file mode 100644 index af75bee425b..00000000000 --- a/common/metrics/init_test.go +++ /dev/null @@ -1,5 +0,0 @@ -package metrics - -func init() { - metricsEnabled = true -} diff --git a/common/metrics/json.go b/common/metrics/json.go deleted file mode 100644 index 6b134d477b6..00000000000 --- a/common/metrics/json.go +++ /dev/null @@ -1,31 +0,0 @@ -package metrics - -import ( - "encoding/json" - "io" - "time" -) - -// MarshalJSON returns a byte slice containing a JSON representation of all -// the metrics in the Registry. -func (r *StandardRegistry) MarshalJSON() ([]byte, error) { - return json.Marshal(r.GetAll()) -} - -// WriteJSON writes metrics from the given registry periodically to the -// specified io.Writer as JSON. -func WriteJSON(r Registry, d time.Duration, w io.Writer) { - for range time.Tick(d) { - WriteJSONOnce(r, w) - } -} - -// WriteJSONOnce writes metrics from the given registry to the specified -// io.Writer as JSON. -func WriteJSONOnce(r Registry, w io.Writer) { - json.NewEncoder(w).Encode(r) -} - -func (r *PrefixedRegistry) MarshalJSON() ([]byte, error) { - return json.Marshal(r.GetAll()) -} diff --git a/common/metrics/json_test.go b/common/metrics/json_test.go deleted file mode 100644 index 811bc29f11e..00000000000 --- a/common/metrics/json_test.go +++ /dev/null @@ -1,28 +0,0 @@ -package metrics - -import ( - "bytes" - "encoding/json" - "testing" -) - -func TestRegistryMarshallJSON(t *testing.T) { - b := &bytes.Buffer{} - enc := json.NewEncoder(b) - r := NewRegistry() - r.Register("counter", NewCounter()) - enc.Encode(r) - if s := b.String(); s != "{\"counter\":{\"count\":0}}\n" { - t.Fatal(s) - } -} - -func TestRegistryWriteJSONOnce(t *testing.T) { - r := NewRegistry() - r.Register("counter", NewCounter()) - b := &bytes.Buffer{} - WriteJSONOnce(r, b) - if s := b.String(); s != "{\"counter\":{\"count\":0}}\n" { - t.Fail() - } -} diff --git a/common/metrics/log.go b/common/metrics/log.go deleted file mode 100644 index 08f3effb81c..00000000000 --- a/common/metrics/log.go +++ /dev/null @@ -1,82 +0,0 @@ -package metrics - -import ( - "time" -) - -type Logger interface { - Printf(format string, v ...interface{}) -} - -func Log(r Registry, freq time.Duration, l Logger) { - LogScaled(r, freq, time.Nanosecond, l) -} - -// LogScaled outputs each metric in the given registry periodically using the given -// logger. Print timings in `scale` units (eg time.Millisecond) rather than nanos. -func LogScaled(r Registry, freq time.Duration, scale time.Duration, l Logger) { - du := float64(scale) - duSuffix := scale.String()[1:] - - for range time.Tick(freq) { - r.Each(func(name string, i interface{}) { - switch metric := i.(type) { - case *Counter: - l.Printf("counter %s\n", name) - l.Printf(" count: %9d\n", metric.Snapshot().Count()) - case *CounterFloat64: - l.Printf("counter %s\n", name) - l.Printf(" count: %f\n", metric.Snapshot().Count()) - case *Gauge: - l.Printf("gauge %s\n", name) - l.Printf(" value: %9d\n", metric.Snapshot().Value()) - case *GaugeFloat64: - l.Printf("gauge %s\n", name) - l.Printf(" value: %f\n", metric.Snapshot().Value()) - case *GaugeInfo: - l.Printf("gauge %s\n", name) - l.Printf(" value: %s\n", metric.Snapshot().Value()) - case Histogram: - h := metric.Snapshot() - ps := h.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999}) - l.Printf("histogram %s\n", name) - l.Printf(" count: %9d\n", h.Count()) - l.Printf(" min: %9d\n", h.Min()) - l.Printf(" max: %9d\n", h.Max()) - l.Printf(" mean: %12.2f\n", h.Mean()) - l.Printf(" stddev: %12.2f\n", h.StdDev()) - l.Printf(" median: %12.2f\n", ps[0]) - l.Printf(" 75%%: %12.2f\n", ps[1]) - l.Printf(" 95%%: %12.2f\n", ps[2]) - l.Printf(" 99%%: %12.2f\n", ps[3]) - l.Printf(" 99.9%%: %12.2f\n", ps[4]) - case *Meter: - m := metric.Snapshot() - l.Printf("meter %s\n", name) - l.Printf(" count: %9d\n", m.Count()) - l.Printf(" 1-min rate: %12.2f\n", m.Rate1()) - l.Printf(" 5-min rate: %12.2f\n", m.Rate5()) - l.Printf(" 15-min rate: %12.2f\n", m.Rate15()) - l.Printf(" mean rate: %12.2f\n", m.RateMean()) - case *Timer: - t := metric.Snapshot() - ps := t.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999}) - l.Printf("timer %s\n", name) - l.Printf(" count: %9d\n", t.Count()) - l.Printf(" min: %12.2f%s\n", float64(t.Min())/du, duSuffix) - l.Printf(" max: %12.2f%s\n", float64(t.Max())/du, duSuffix) - l.Printf(" mean: %12.2f%s\n", t.Mean()/du, duSuffix) - l.Printf(" stddev: %12.2f%s\n", t.StdDev()/du, duSuffix) - l.Printf(" median: %12.2f%s\n", ps[0]/du, duSuffix) - l.Printf(" 75%%: %12.2f%s\n", ps[1]/du, duSuffix) - l.Printf(" 95%%: %12.2f%s\n", ps[2]/du, duSuffix) - l.Printf(" 99%%: %12.2f%s\n", ps[3]/du, duSuffix) - l.Printf(" 99.9%%: %12.2f%s\n", ps[4]/du, duSuffix) - l.Printf(" 1-min rate: %12.2f\n", t.Rate1()) - l.Printf(" 5-min rate: %12.2f\n", t.Rate5()) - l.Printf(" 15-min rate: %12.2f\n", t.Rate15()) - l.Printf(" mean rate: %12.2f\n", t.RateMean()) - } - }) - } -} diff --git a/common/metrics/memory.md b/common/metrics/memory.md deleted file mode 100644 index 47454f54b64..00000000000 --- a/common/metrics/memory.md +++ /dev/null @@ -1,285 +0,0 @@ -Memory usage -============ - -(Highly unscientific.) - -Command used to gather static memory usage: - -```sh -grep ^Vm "/proc/$(ps fax | grep [m]etrics-bench | awk '{print $1}')/status" -``` - -Program used to gather baseline memory usage: - -```go -package main - -import "time" - -func main() { - time.Sleep(600e9) -} -``` - -Baseline --------- - -``` -VmPeak: 42604 kB -VmSize: 42604 kB -VmLck: 0 kB -VmHWM: 1120 kB -VmRSS: 1120 kB -VmData: 35460 kB -VmStk: 136 kB -VmExe: 1020 kB -VmLib: 1848 kB -VmPTE: 36 kB -VmSwap: 0 kB -``` - -Program used to gather metric memory usage (with other metrics being similar): - -```go -package main - -import ( - "fmt" - "metrics" - "time" -) - -func main() { - fmt.Sprintf("foo") - metrics.NewRegistry() - time.Sleep(600e9) -} -``` - -1000 counters registered ------------------------- - -``` -VmPeak: 44016 kB -VmSize: 44016 kB -VmLck: 0 kB -VmHWM: 1928 kB -VmRSS: 1928 kB -VmData: 36868 kB -VmStk: 136 kB -VmExe: 1024 kB -VmLib: 1848 kB -VmPTE: 40 kB -VmSwap: 0 kB -``` - -**1.412 kB virtual, TODO 0.808 kB resident per counter.** - -100000 counters registered --------------------------- - -``` -VmPeak: 55024 kB -VmSize: 55024 kB -VmLck: 0 kB -VmHWM: 12440 kB -VmRSS: 12440 kB -VmData: 47876 kB -VmStk: 136 kB -VmExe: 1024 kB -VmLib: 1848 kB -VmPTE: 64 kB -VmSwap: 0 kB -``` - -**0.1242 kB virtual, 0.1132 kB resident per counter.** - -1000 gauges registered ----------------------- - -``` -VmPeak: 44012 kB -VmSize: 44012 kB -VmLck: 0 kB -VmHWM: 1928 kB -VmRSS: 1928 kB -VmData: 36868 kB -VmStk: 136 kB -VmExe: 1020 kB -VmLib: 1848 kB -VmPTE: 40 kB -VmSwap: 0 kB -``` - -**1.408 kB virtual, 0.808 kB resident per counter.** - -100000 gauges registered ------------------------- - -``` -VmPeak: 55020 kB -VmSize: 55020 kB -VmLck: 0 kB -VmHWM: 12432 kB -VmRSS: 12432 kB -VmData: 47876 kB -VmStk: 136 kB -VmExe: 1020 kB -VmLib: 1848 kB -VmPTE: 60 kB -VmSwap: 0 kB -``` - -**0.12416 kB virtual, 0.11312 resident per gauge.** - -1000 histograms with a uniform sample size of 1028 --------------------------------------------------- - -``` -VmPeak: 72272 kB -VmSize: 72272 kB -VmLck: 0 kB -VmHWM: 16204 kB -VmRSS: 16204 kB -VmData: 65100 kB -VmStk: 136 kB -VmExe: 1048 kB -VmLib: 1848 kB -VmPTE: 80 kB -VmSwap: 0 kB -``` - -**29.668 kB virtual, TODO 15.084 resident per histogram.** - -10000 histograms with a uniform sample size of 1028 ---------------------------------------------------- - -``` -VmPeak: 256912 kB -VmSize: 256912 kB -VmLck: 0 kB -VmHWM: 146204 kB -VmRSS: 146204 kB -VmData: 249740 kB -VmStk: 136 kB -VmExe: 1048 kB -VmLib: 1848 kB -VmPTE: 448 kB -VmSwap: 0 kB -``` - -**21.4308 kB virtual, 14.5084 kB resident per histogram.** - -50000 histograms with a uniform sample size of 1028 ---------------------------------------------------- - -``` -VmPeak: 908112 kB -VmSize: 908112 kB -VmLck: 0 kB -VmHWM: 645832 kB -VmRSS: 645588 kB -VmData: 900940 kB -VmStk: 136 kB -VmExe: 1048 kB -VmLib: 1848 kB -VmPTE: 1716 kB -VmSwap: 1544 kB -``` - -**17.31016 kB virtual, 12.88936 kB resident per histogram.** - -1000 histograms with an exponentially-decaying sample size of 1028 and alpha of 0.015 -------------------------------------------------------------------------------------- - -``` -VmPeak: 62480 kB -VmSize: 62480 kB -VmLck: 0 kB -VmHWM: 11572 kB -VmRSS: 11572 kB -VmData: 55308 kB -VmStk: 136 kB -VmExe: 1048 kB -VmLib: 1848 kB -VmPTE: 64 kB -VmSwap: 0 kB -``` - -**19.876 kB virtual, 10.452 kB resident per histogram.** - -10000 histograms with an exponentially-decaying sample size of 1028 and alpha of 0.015 --------------------------------------------------------------------------------------- - -``` -VmPeak: 153296 kB -VmSize: 153296 kB -VmLck: 0 kB -VmHWM: 101176 kB -VmRSS: 101176 kB -VmData: 146124 kB -VmStk: 136 kB -VmExe: 1048 kB -VmLib: 1848 kB -VmPTE: 240 kB -VmSwap: 0 kB -``` - -**11.0692 kB virtual, 10.0056 kB resident per histogram.** - -50000 histograms with an exponentially-decaying sample size of 1028 and alpha of 0.015 --------------------------------------------------------------------------------------- - -``` -VmPeak: 557264 kB -VmSize: 557264 kB -VmLck: 0 kB -VmHWM: 501056 kB -VmRSS: 501056 kB -VmData: 550092 kB -VmStk: 136 kB -VmExe: 1048 kB -VmLib: 1848 kB -VmPTE: 1032 kB -VmSwap: 0 kB -``` - -**10.2932 kB virtual, 9.99872 kB resident per histogram.** - -1000 meters ------------ - -``` -VmPeak: 74504 kB -VmSize: 74504 kB -VmLck: 0 kB -VmHWM: 24124 kB -VmRSS: 24124 kB -VmData: 67340 kB -VmStk: 136 kB -VmExe: 1040 kB -VmLib: 1848 kB -VmPTE: 92 kB -VmSwap: 0 kB -``` - -**31.9 kB virtual, 23.004 kB resident per meter.** - -10000 meters ------------- - -``` -VmPeak: 278920 kB -VmSize: 278920 kB -VmLck: 0 kB -VmHWM: 227300 kB -VmRSS: 227300 kB -VmData: 271756 kB -VmStk: 136 kB -VmExe: 1040 kB -VmLib: 1848 kB -VmPTE: 488 kB -VmSwap: 0 kB -``` - -**23.6316 kB virtual, 22.618 kB resident per meter.** diff --git a/common/metrics/meter.go b/common/metrics/meter.go deleted file mode 100644 index ee23af10ebe..00000000000 --- a/common/metrics/meter.go +++ /dev/null @@ -1,167 +0,0 @@ -package metrics - -import ( - "math" - "sync" - "sync/atomic" - "time" -) - -// GetOrRegisterMeter returns an existing Meter or constructs and registers a -// new Meter. -// Be sure to unregister the meter from the registry once it is of no use to -// allow for garbage collection. -func GetOrRegisterMeter(name string, r Registry) *Meter { - return getOrRegister(name, NewMeter, r) -} - -// NewMeter constructs a new Meter and launches a goroutine. -// Be sure to call Stop() once the meter is of no use to allow for garbage collection. -func NewMeter() *Meter { - m := newMeter() - arbiter.add(m) - return m -} - -// NewInactiveMeter returns a meter but does not start any goroutines. This -// method is mainly intended for testing. -func NewInactiveMeter() *Meter { - return newMeter() -} - -// NewRegisteredMeter constructs and registers a new Meter -// and launches a goroutine. -// Be sure to unregister the meter from the registry once it is of no use to -// allow for garbage collection. -func NewRegisteredMeter(name string, r Registry) *Meter { - return GetOrRegisterMeter(name, r) -} - -// MeterSnapshot is a read-only copy of the meter's internal values. -type MeterSnapshot struct { - count int64 - rate1, rate5, rate15, rateMean float64 -} - -// Count returns the count of events at the time the snapshot was taken. -func (m *MeterSnapshot) Count() int64 { return m.count } - -// Rate1 returns the one-minute moving average rate of events per second at the -// time the snapshot was taken. -func (m *MeterSnapshot) Rate1() float64 { return m.rate1 } - -// Rate5 returns the five-minute moving average rate of events per second at -// the time the snapshot was taken. -func (m *MeterSnapshot) Rate5() float64 { return m.rate5 } - -// Rate15 returns the fifteen-minute moving average rate of events per second -// at the time the snapshot was taken. -func (m *MeterSnapshot) Rate15() float64 { return m.rate15 } - -// RateMean returns the meter's mean rate of events per second at the time the -// snapshot was taken. -func (m *MeterSnapshot) RateMean() float64 { return m.rateMean } - -// Meter count events to produce exponentially-weighted moving average rates -// at one-, five-, and fifteen-minutes and a mean rate. -type Meter struct { - count atomic.Int64 - uncounted atomic.Int64 // not yet added to the EWMAs - rateMean atomic.Uint64 - - a1, a5, a15 *EWMA - startTime time.Time - stopped atomic.Bool -} - -func newMeter() *Meter { - return &Meter{ - a1: NewEWMA1(), - a5: NewEWMA5(), - a15: NewEWMA15(), - startTime: time.Now(), - } -} - -// Stop stops the meter, Mark() will be a no-op if you use it after being stopped. -func (m *Meter) Stop() { - if stopped := m.stopped.Swap(true); !stopped { - arbiter.remove(m) - } -} - -// Mark records the occurrence of n events. -func (m *Meter) Mark(n int64) { - m.uncounted.Add(n) -} - -// Snapshot returns a read-only copy of the meter. -func (m *Meter) Snapshot() *MeterSnapshot { - return &MeterSnapshot{ - count: m.count.Load() + m.uncounted.Load(), - rate1: m.a1.Snapshot().Rate(), - rate5: m.a5.Snapshot().Rate(), - rate15: m.a15.Snapshot().Rate(), - rateMean: math.Float64frombits(m.rateMean.Load()), - } -} - -func (m *Meter) tick() { - // Take the uncounted values, add to count - n := m.uncounted.Swap(0) - count := m.count.Add(n) - m.rateMean.Store(math.Float64bits(float64(count) / time.Since(m.startTime).Seconds())) - // Update the EWMA's internal state - m.a1.Update(n) - m.a5.Update(n) - m.a15.Update(n) - // And trigger them to calculate the rates - m.a1.tick() - m.a5.tick() - m.a15.tick() -} - -var arbiter = meterTicker{meters: make(map[*Meter]struct{})} - -// meterTicker ticks meters every 5s from a single goroutine. -// meters are references in a set for future stopping. -type meterTicker struct { - mu sync.RWMutex - - once sync.Once - meters map[*Meter]struct{} -} - -// add a *Meter to the arbiter -func (ma *meterTicker) add(m *Meter) { - ma.mu.Lock() - defer ma.mu.Unlock() - ma.meters[m] = struct{}{} -} - -// remove removes a meter from the set of ticked meters. -func (ma *meterTicker) remove(m *Meter) { - ma.mu.Lock() - delete(ma.meters, m) - ma.mu.Unlock() -} - -// loop ticks meters on a 5-second interval. -func (ma *meterTicker) loop() { - ticker := time.NewTicker(5 * time.Second) - for range ticker.C { - if !metricsEnabled { - continue - } - ma.mu.RLock() - for meter := range ma.meters { - meter.tick() - } - ma.mu.RUnlock() - } -} - -// startMeterTickerLoop will start the arbiter ticker. -func startMeterTickerLoop() { - arbiter.once.Do(func() { go arbiter.loop() }) -} diff --git a/common/metrics/meter_test.go b/common/metrics/meter_test.go deleted file mode 100644 index e3f39684bd2..00000000000 --- a/common/metrics/meter_test.go +++ /dev/null @@ -1,83 +0,0 @@ -package metrics - -import ( - "testing" - "time" -) - -func BenchmarkMeter(b *testing.B) { - m := NewMeter() - b.ResetTimer() - for i := 0; i < b.N; i++ { - m.Mark(1) - } -} -func TestMeter(t *testing.T) { - m := NewMeter() - m.Mark(47) - if v := m.Snapshot().Count(); v != 47 { - t.Fatalf("have %d want %d", v, 47) - } -} -func TestGetOrRegisterMeter(t *testing.T) { - r := NewRegistry() - NewRegisteredMeter("foo", r).Mark(47) - if m := GetOrRegisterMeter("foo", r).Snapshot(); m.Count() != 47 { - t.Fatal(m.Count()) - } -} - -func TestMeterDecay(t *testing.T) { - m := newMeter() - m.Mark(1) - m.tick() - rateMean := m.Snapshot().RateMean() - time.Sleep(100 * time.Millisecond) - m.tick() - if m.Snapshot().RateMean() >= rateMean { - t.Error("m.RateMean() didn't decrease") - } -} - -func TestMeterNonzero(t *testing.T) { - m := NewMeter() - m.Mark(3) - if count := m.Snapshot().Count(); count != 3 { - t.Errorf("m.Count(): 3 != %v\n", count) - } -} - -func TestMeterStop(t *testing.T) { - l := len(arbiter.meters) - m := NewMeter() - if l+1 != len(arbiter.meters) { - t.Errorf("arbiter.meters: %d != %d\n", l+1, len(arbiter.meters)) - } - m.Stop() - if l != len(arbiter.meters) { - t.Errorf("arbiter.meters: %d != %d\n", l, len(arbiter.meters)) - } -} - -func TestMeterZero(t *testing.T) { - m := NewMeter().Snapshot() - if count := m.Count(); count != 0 { - t.Errorf("m.Count(): 0 != %v\n", count) - } -} - -func TestMeterRepeat(t *testing.T) { - m := NewMeter() - for i := 0; i < 101; i++ { - m.Mark(int64(i)) - } - if count := m.Snapshot().Count(); count != 5050 { - t.Errorf("m.Count(): 5050 != %v\n", count) - } - for i := 0; i < 101; i++ { - m.Mark(int64(i)) - } - if count := m.Snapshot().Count(); count != 10100 { - t.Errorf("m.Count(): 10100 != %v\n", count) - } -} diff --git a/common/metrics/metrics.go b/common/metrics/metrics.go deleted file mode 100644 index 088948d4034..00000000000 --- a/common/metrics/metrics.go +++ /dev/null @@ -1,204 +0,0 @@ -// Go port of Coda Hale's Metrics library -// -// -// -// Coda Hale's original work: - -package metrics - -import ( - "runtime/metrics" - "runtime/pprof" - "time" -) - -var ( - metricsEnabled = false -) - -// Enabled is checked by functions that are deemed 'expensive', e.g. if a -// meter-type does locking and/or non-trivial math operations during update. -func Enabled() bool { - return metricsEnabled -} - -// Enable enables the metrics system. -// The Enabled-flag is expected to be set, once, during startup, but toggling off and on -// is not supported. -// -// Enable is not safe to call concurrently. You need to call this as early as possible in -// the program, before any metrics collection will happen. -func Enable() { - metricsEnabled = true - startMeterTickerLoop() -} - -var threadCreateProfile = pprof.Lookup("threadcreate") - -type runtimeStats struct { - GCPauses *metrics.Float64Histogram - GCAllocBytes uint64 - GCFreedBytes uint64 - - MemTotal uint64 - HeapObjects uint64 - HeapFree uint64 - HeapReleased uint64 - HeapUnused uint64 - - Goroutines uint64 - SchedLatency *metrics.Float64Histogram -} - -var runtimeSamples = []metrics.Sample{ - {Name: "/gc/pauses:seconds"}, // histogram - {Name: "/gc/heap/allocs:bytes"}, - {Name: "/gc/heap/frees:bytes"}, - {Name: "/memory/classes/total:bytes"}, - {Name: "/memory/classes/heap/objects:bytes"}, - {Name: "/memory/classes/heap/free:bytes"}, - {Name: "/memory/classes/heap/released:bytes"}, - {Name: "/memory/classes/heap/unused:bytes"}, - {Name: "/sched/goroutines:goroutines"}, - {Name: "/sched/latencies:seconds"}, // histogram -} - -func ReadRuntimeStats() *runtimeStats { - r := new(runtimeStats) - readRuntimeStats(r) - return r -} - -func readRuntimeStats(v *runtimeStats) { - metrics.Read(runtimeSamples) - for _, s := range runtimeSamples { - // Skip invalid/unknown metrics. This is needed because some metrics - // are unavailable in older Go versions, and attempting to read a 'bad' - // metric panics. - if s.Value.Kind() == metrics.KindBad { - continue - } - - switch s.Name { - case "/gc/pauses:seconds": - v.GCPauses = s.Value.Float64Histogram() - case "/gc/heap/allocs:bytes": - v.GCAllocBytes = s.Value.Uint64() - case "/gc/heap/frees:bytes": - v.GCFreedBytes = s.Value.Uint64() - case "/memory/classes/total:bytes": - v.MemTotal = s.Value.Uint64() - case "/memory/classes/heap/objects:bytes": - v.HeapObjects = s.Value.Uint64() - case "/memory/classes/heap/free:bytes": - v.HeapFree = s.Value.Uint64() - case "/memory/classes/heap/released:bytes": - v.HeapReleased = s.Value.Uint64() - case "/memory/classes/heap/unused:bytes": - v.HeapUnused = s.Value.Uint64() - case "/sched/goroutines:goroutines": - v.Goroutines = s.Value.Uint64() - case "/sched/latencies:seconds": - v.SchedLatency = s.Value.Float64Histogram() - } - } -} - -// CollectProcessMetrics periodically collects various metrics about the running process. -func CollectProcessMetrics(refresh time.Duration) { - // Short circuit if the metrics system is disabled - if !metricsEnabled { - return - } - - // Create the various data collectors - var ( - cpustats = make([]CPUStats, 2) - diskstats = make([]DiskStats, 2) - rstats = make([]runtimeStats, 2) - ) - - // This scale factor is used for the runtime's time metrics. It's useful to convert to - // ns here because the runtime gives times in float seconds, but runtimeHistogram can - // only provide integers for the minimum and maximum values. - const secondsToNs = float64(time.Second) - - // Define the various metrics to collect - var ( - cpuSysLoad = GetOrRegisterGauge("system/cpu/sysload", DefaultRegistry) - cpuSysWait = GetOrRegisterGauge("system/cpu/syswait", DefaultRegistry) - cpuProcLoad = GetOrRegisterGauge("system/cpu/procload", DefaultRegistry) - cpuSysLoadTotal = GetOrRegisterCounterFloat64("system/cpu/sysload/total", DefaultRegistry) - cpuSysWaitTotal = GetOrRegisterCounterFloat64("system/cpu/syswait/total", DefaultRegistry) - cpuProcLoadTotal = GetOrRegisterCounterFloat64("system/cpu/procload/total", DefaultRegistry) - cpuThreads = GetOrRegisterGauge("system/cpu/threads", DefaultRegistry) - cpuGoroutines = GetOrRegisterGauge("system/cpu/goroutines", DefaultRegistry) - cpuSchedLatency = getOrRegisterRuntimeHistogram("system/cpu/schedlatency", secondsToNs, nil) - memPauses = getOrRegisterRuntimeHistogram("system/memory/pauses", secondsToNs, nil) - memAllocs = GetOrRegisterMeter("system/memory/allocs", DefaultRegistry) - memFrees = GetOrRegisterMeter("system/memory/frees", DefaultRegistry) - memTotal = GetOrRegisterGauge("system/memory/held", DefaultRegistry) - heapUsed = GetOrRegisterGauge("system/memory/used", DefaultRegistry) - heapObjects = GetOrRegisterGauge("system/memory/objects", DefaultRegistry) - diskReads = GetOrRegisterMeter("system/disk/readcount", DefaultRegistry) - diskReadBytes = GetOrRegisterMeter("system/disk/readdata", DefaultRegistry) - diskReadBytesCounter = GetOrRegisterCounter("system/disk/readbytes", DefaultRegistry) - diskWrites = GetOrRegisterMeter("system/disk/writecount", DefaultRegistry) - diskWriteBytes = GetOrRegisterMeter("system/disk/writedata", DefaultRegistry) - diskWriteBytesCounter = GetOrRegisterCounter("system/disk/writebytes", DefaultRegistry) - ) - - var lastCollectTime time.Time - - // Iterate loading the different stats and updating the meters. - now, prev := 0, 1 - for ; ; now, prev = prev, now { - // Gather CPU times. - ReadCPUStats(&cpustats[now]) - collectTime := time.Now() - secondsSinceLastCollect := collectTime.Sub(lastCollectTime).Seconds() - lastCollectTime = collectTime - if secondsSinceLastCollect > 0 { - sysLoad := cpustats[now].GlobalTime - cpustats[prev].GlobalTime - sysWait := cpustats[now].GlobalWait - cpustats[prev].GlobalWait - procLoad := cpustats[now].LocalTime - cpustats[prev].LocalTime - // Convert to integer percentage. - cpuSysLoad.Update(int64(sysLoad / secondsSinceLastCollect * 100)) - cpuSysWait.Update(int64(sysWait / secondsSinceLastCollect * 100)) - cpuProcLoad.Update(int64(procLoad / secondsSinceLastCollect * 100)) - // increment counters (ms) - cpuSysLoadTotal.Inc(sysLoad) - cpuSysWaitTotal.Inc(sysWait) - cpuProcLoadTotal.Inc(procLoad) - } - - // Threads - cpuThreads.Update(int64(threadCreateProfile.Count())) - - // Go runtime metrics - readRuntimeStats(&rstats[now]) - - cpuGoroutines.Update(int64(rstats[now].Goroutines)) - cpuSchedLatency.update(rstats[now].SchedLatency) - memPauses.update(rstats[now].GCPauses) - - memAllocs.Mark(int64(rstats[now].GCAllocBytes - rstats[prev].GCAllocBytes)) - memFrees.Mark(int64(rstats[now].GCFreedBytes - rstats[prev].GCFreedBytes)) - - memTotal.Update(int64(rstats[now].MemTotal)) - heapUsed.Update(int64(rstats[now].MemTotal - rstats[now].HeapUnused - rstats[now].HeapFree - rstats[now].HeapReleased)) - heapObjects.Update(int64(rstats[now].HeapObjects)) - - // Disk - if ReadDiskStats(&diskstats[now]) == nil { - diskReads.Mark(diskstats[now].ReadCount - diskstats[prev].ReadCount) - diskReadBytes.Mark(diskstats[now].ReadBytes - diskstats[prev].ReadBytes) - diskWrites.Mark(diskstats[now].WriteCount - diskstats[prev].WriteCount) - diskWriteBytes.Mark(diskstats[now].WriteBytes - diskstats[prev].WriteBytes) - diskReadBytesCounter.Inc(diskstats[now].ReadBytes - diskstats[prev].ReadBytes) - diskWriteBytesCounter.Inc(diskstats[now].WriteBytes - diskstats[prev].WriteBytes) - } - - time.Sleep(refresh) - } -} diff --git a/common/metrics/metrics_test.go b/common/metrics/metrics_test.go deleted file mode 100644 index dc144f2425a..00000000000 --- a/common/metrics/metrics_test.go +++ /dev/null @@ -1,62 +0,0 @@ -package metrics - -import ( - "fmt" - "sync" - "testing" - "time" -) - -func TestReadRuntimeValues(t *testing.T) { - var v runtimeStats - readRuntimeStats(&v) - t.Logf("%+v", v) -} - -func BenchmarkMetrics(b *testing.B) { - var ( - r = NewRegistry() - c = NewRegisteredCounter("counter", r) - cf = NewRegisteredCounterFloat64("counterfloat64", r) - g = NewRegisteredGauge("gauge", r) - gf = NewRegisteredGaugeFloat64("gaugefloat64", r) - h = NewRegisteredHistogram("histogram", r, NewUniformSample(100)) - m = NewRegisteredMeter("meter", r) - t = NewRegisteredTimer("timer", r) - ) - RegisterDebugGCStats(r) - b.ResetTimer() - var wg sync.WaitGroup - wg.Add(128) - for i := 0; i < 128; i++ { - go func() { - defer wg.Done() - for i := 0; i < b.N; i++ { - c.Inc(1) - cf.Inc(1.0) - g.Update(int64(i)) - gf.Update(float64(i)) - h.Update(int64(i)) - m.Mark(1) - t.Update(1) - } - }() - } - wg.Wait() -} - -func Example() { - c := NewCounter() - Register("money", c) - c.Inc(17) - - // Threadsafe registration - t := GetOrRegisterTimer("db.get.latency", nil) - t.Time(func() { time.Sleep(10 * time.Millisecond) }) - t.Update(1) - - fmt.Println(c.Snapshot().Count()) - fmt.Println(t.Snapshot().Min()) - // Output: 17 - // 1 -} diff --git a/common/metrics/opentsdb.go b/common/metrics/opentsdb.go deleted file mode 100644 index 57af3d025e6..00000000000 --- a/common/metrics/opentsdb.go +++ /dev/null @@ -1,128 +0,0 @@ -package metrics - -import ( - "bufio" - "fmt" - "io" - "log" - "net" - "os" - "strings" - "time" -) - -var shortHostName = "" - -// OpenTSDBConfig provides a container with configuration parameters for -// the OpenTSDB exporter -type OpenTSDBConfig struct { - Addr *net.TCPAddr // Network address to connect to - Registry Registry // Registry to be exported - FlushInterval time.Duration // Flush interval - DurationUnit time.Duration // Time conversion unit for durations - Prefix string // Prefix to be prepended to metric names -} - -// OpenTSDB is a blocking exporter function which reports metrics in r -// to a TSDB server located at addr, flushing them every d duration -// and prepending metric names with prefix. -func OpenTSDB(r Registry, d time.Duration, prefix string, addr *net.TCPAddr) { - OpenTSDBWithConfig(OpenTSDBConfig{ - Addr: addr, - Registry: r, - FlushInterval: d, - DurationUnit: time.Nanosecond, - Prefix: prefix, - }) -} - -// OpenTSDBWithConfig is a blocking exporter function just like OpenTSDB, -// but it takes a OpenTSDBConfig instead. -func OpenTSDBWithConfig(c OpenTSDBConfig) { - for range time.Tick(c.FlushInterval) { - if err := openTSDB(&c); nil != err { - log.Println(err) - } - } -} - -func getShortHostname() string { - if shortHostName == "" { - host, _ := os.Hostname() - if index := strings.Index(host, "."); index > 0 { - shortHostName = host[:index] - } else { - shortHostName = host - } - } - return shortHostName -} - -// writeRegistry writes the registry-metrics on the opentsb format. -func (c *OpenTSDBConfig) writeRegistry(w io.Writer, now int64, shortHostname string) { - du := float64(c.DurationUnit) - - c.Registry.Each(func(name string, i interface{}) { - switch metric := i.(type) { - case *Counter: - fmt.Fprintf(w, "put %s.%s.count %d %d host=%s\n", c.Prefix, name, now, metric.Snapshot().Count(), shortHostname) - case *CounterFloat64: - fmt.Fprintf(w, "put %s.%s.count %d %f host=%s\n", c.Prefix, name, now, metric.Snapshot().Count(), shortHostname) - case *Gauge: - fmt.Fprintf(w, "put %s.%s.value %d %d host=%s\n", c.Prefix, name, now, metric.Snapshot().Value(), shortHostname) - case *GaugeFloat64: - fmt.Fprintf(w, "put %s.%s.value %d %f host=%s\n", c.Prefix, name, now, metric.Snapshot().Value(), shortHostname) - case *GaugeInfo: - fmt.Fprintf(w, "put %s.%s.value %d %s host=%s\n", c.Prefix, name, now, metric.Snapshot().Value().String(), shortHostname) - case Histogram: - h := metric.Snapshot() - ps := h.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999}) - fmt.Fprintf(w, "put %s.%s.count %d %d host=%s\n", c.Prefix, name, now, h.Count(), shortHostname) - fmt.Fprintf(w, "put %s.%s.min %d %d host=%s\n", c.Prefix, name, now, h.Min(), shortHostname) - fmt.Fprintf(w, "put %s.%s.max %d %d host=%s\n", c.Prefix, name, now, h.Max(), shortHostname) - fmt.Fprintf(w, "put %s.%s.mean %d %.2f host=%s\n", c.Prefix, name, now, h.Mean(), shortHostname) - fmt.Fprintf(w, "put %s.%s.std-dev %d %.2f host=%s\n", c.Prefix, name, now, h.StdDev(), shortHostname) - fmt.Fprintf(w, "put %s.%s.50-percentile %d %.2f host=%s\n", c.Prefix, name, now, ps[0], shortHostname) - fmt.Fprintf(w, "put %s.%s.75-percentile %d %.2f host=%s\n", c.Prefix, name, now, ps[1], shortHostname) - fmt.Fprintf(w, "put %s.%s.95-percentile %d %.2f host=%s\n", c.Prefix, name, now, ps[2], shortHostname) - fmt.Fprintf(w, "put %s.%s.99-percentile %d %.2f host=%s\n", c.Prefix, name, now, ps[3], shortHostname) - fmt.Fprintf(w, "put %s.%s.999-percentile %d %.2f host=%s\n", c.Prefix, name, now, ps[4], shortHostname) - case *Meter: - m := metric.Snapshot() - fmt.Fprintf(w, "put %s.%s.count %d %d host=%s\n", c.Prefix, name, now, m.Count(), shortHostname) - fmt.Fprintf(w, "put %s.%s.one-minute %d %.2f host=%s\n", c.Prefix, name, now, m.Rate1(), shortHostname) - fmt.Fprintf(w, "put %s.%s.five-minute %d %.2f host=%s\n", c.Prefix, name, now, m.Rate5(), shortHostname) - fmt.Fprintf(w, "put %s.%s.fifteen-minute %d %.2f host=%s\n", c.Prefix, name, now, m.Rate15(), shortHostname) - fmt.Fprintf(w, "put %s.%s.mean %d %.2f host=%s\n", c.Prefix, name, now, m.RateMean(), shortHostname) - case *Timer: - t := metric.Snapshot() - ps := t.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999}) - fmt.Fprintf(w, "put %s.%s.count %d %d host=%s\n", c.Prefix, name, now, t.Count(), shortHostname) - fmt.Fprintf(w, "put %s.%s.min %d %d host=%s\n", c.Prefix, name, now, t.Min()/int64(du), shortHostname) - fmt.Fprintf(w, "put %s.%s.max %d %d host=%s\n", c.Prefix, name, now, t.Max()/int64(du), shortHostname) - fmt.Fprintf(w, "put %s.%s.mean %d %.2f host=%s\n", c.Prefix, name, now, t.Mean()/du, shortHostname) - fmt.Fprintf(w, "put %s.%s.std-dev %d %.2f host=%s\n", c.Prefix, name, now, t.StdDev()/du, shortHostname) - fmt.Fprintf(w, "put %s.%s.50-percentile %d %.2f host=%s\n", c.Prefix, name, now, ps[0]/du, shortHostname) - fmt.Fprintf(w, "put %s.%s.75-percentile %d %.2f host=%s\n", c.Prefix, name, now, ps[1]/du, shortHostname) - fmt.Fprintf(w, "put %s.%s.95-percentile %d %.2f host=%s\n", c.Prefix, name, now, ps[2]/du, shortHostname) - fmt.Fprintf(w, "put %s.%s.99-percentile %d %.2f host=%s\n", c.Prefix, name, now, ps[3]/du, shortHostname) - fmt.Fprintf(w, "put %s.%s.999-percentile %d %.2f host=%s\n", c.Prefix, name, now, ps[4]/du, shortHostname) - fmt.Fprintf(w, "put %s.%s.one-minute %d %.2f host=%s\n", c.Prefix, name, now, t.Rate1(), shortHostname) - fmt.Fprintf(w, "put %s.%s.five-minute %d %.2f host=%s\n", c.Prefix, name, now, t.Rate5(), shortHostname) - fmt.Fprintf(w, "put %s.%s.fifteen-minute %d %.2f host=%s\n", c.Prefix, name, now, t.Rate15(), shortHostname) - fmt.Fprintf(w, "put %s.%s.mean-rate %d %.2f host=%s\n", c.Prefix, name, now, t.RateMean(), shortHostname) - } - }) -} - -func openTSDB(c *OpenTSDBConfig) error { - conn, err := net.DialTCP("tcp", nil, c.Addr) - if nil != err { - return err - } - defer conn.Close() - w := bufio.NewWriter(conn) - c.writeRegistry(w, time.Now().Unix(), getShortHostname()) - w.Flush() - return nil -} diff --git a/common/metrics/opentsdb_test.go b/common/metrics/opentsdb_test.go deleted file mode 100644 index 4548309f9c2..00000000000 --- a/common/metrics/opentsdb_test.go +++ /dev/null @@ -1,66 +0,0 @@ -package metrics - -import ( - "fmt" - "net" - "os" - "strings" - "testing" - "time" -) - -func ExampleOpenTSDB() { - addr, _ := net.ResolveTCPAddr("net", ":2003") - go OpenTSDB(DefaultRegistry, 1*time.Second, "some.prefix", addr) -} - -func ExampleOpenTSDBWithConfig() { - addr, _ := net.ResolveTCPAddr("net", ":2003") - go OpenTSDBWithConfig(OpenTSDBConfig{ - Addr: addr, - Registry: DefaultRegistry, - FlushInterval: 1 * time.Second, - DurationUnit: time.Millisecond, - }) -} - -func TestExampleOpenTSB(t *testing.T) { - r := NewOrderedRegistry() - NewRegisteredGaugeInfo("foo", r).Update(GaugeInfoValue{"chain_id": "5"}) - NewRegisteredGaugeFloat64("pi", r).Update(3.14) - NewRegisteredCounter("months", r).Inc(12) - NewRegisteredCounterFloat64("tau", r).Inc(1.57) - NewRegisteredMeter("elite", r).Mark(1337) - NewRegisteredTimer("second", r).Update(time.Second) - NewRegisteredCounterFloat64("tau", r).Inc(1.57) - NewRegisteredCounterFloat64("tau", r).Inc(1.57) - - w := new(strings.Builder) - (&OpenTSDBConfig{ - Registry: r, - DurationUnit: time.Millisecond, - Prefix: "pre", - }).writeRegistry(w, 978307200, "hal9000") - - wantB, err := os.ReadFile("./testdata/opentsb.want") - if err != nil { - t.Fatal(err) - } - if have, want := w.String(), string(wantB); have != want { - t.Errorf("\nhave:\n%v\nwant:\n%v\n", have, want) - t.Logf("have vs want:\n%v", findFirstDiffPos(have, want)) - } -} - -func findFirstDiffPos(a, b string) string { - yy := strings.Split(b, "\n") - for i, x := range strings.Split(a, "\n") { - if i >= len(yy) { - return fmt.Sprintf("have:%d: %s\nwant:%d: ", i, x, i) - } - if y := yy[i]; x != y { - return fmt.Sprintf("have:%d: %s\nwant:%d: %s", i, x, i, y) - } - } - return "" -} diff --git a/common/metrics/registry.go b/common/metrics/registry.go deleted file mode 100644 index 6070f3d0e9e..00000000000 --- a/common/metrics/registry.go +++ /dev/null @@ -1,363 +0,0 @@ -package metrics - -import ( - "errors" - "fmt" - "sort" - "strings" - "sync" -) - -// ErrDuplicateMetric is the error returned by Registry.Register when a metric -// already exists. If you mean to Register that metric you must first -// Unregister the existing metric. -var ErrDuplicateMetric = errors.New("duplicate metric") - -// A Registry holds references to a set of metrics by name and can iterate -// over them, calling callback functions provided by the user. -// -// This is an interface to encourage other structs to implement -// the Registry API as appropriate. -type Registry interface { - - // Each call the given function for each registered metric. - Each(func(string, interface{})) - - // Get the metric by the given name or nil if none is registered. - Get(string) interface{} - - // GetAll metrics in the Registry. - GetAll() map[string]map[string]interface{} - - // GetOrRegister returns an existing metric or registers the one returned - // by the given constructor. - GetOrRegister(string, func() interface{}) interface{} - - // Register the given metric under the given name. - Register(string, interface{}) error - - // RunHealthchecks run all registered healthchecks. - RunHealthchecks() - - // Unregister the metric with the given name. - Unregister(string) -} - -type orderedRegistry struct { - StandardRegistry -} - -// Each call the given function for each registered metric. -func (r *orderedRegistry) Each(f func(string, interface{})) { - var names []string - reg := r.registered() - for name := range reg { - names = append(names, name) - } - sort.Strings(names) - for _, name := range names { - f(name, reg[name]) - } -} - -// NewRegistry creates a new registry. -func NewRegistry() Registry { - return new(StandardRegistry) -} - -// NewOrderedRegistry creates a new ordered registry (for testing). -func NewOrderedRegistry() Registry { - return new(orderedRegistry) -} - -// StandardRegistry the standard implementation of a Registry uses sync.map -// of names to metrics. -type StandardRegistry struct { - metrics sync.Map -} - -// Each call the given function for each registered metric. -func (r *StandardRegistry) Each(f func(string, interface{})) { - for name, i := range r.registered() { - f(name, i) - } -} - -// Get the metric by the given name or nil if none is registered. -func (r *StandardRegistry) Get(name string) interface{} { - item, _ := r.metrics.Load(name) - return item -} - -// GetOrRegister gets an existing metric or creates and registers a new one. Threadsafe -// alternative to calling Get and Register on failure. -// The interface can be the metric to register if not found in registry, -// or a function returning the metric for lazy instantiation. -func (r *StandardRegistry) GetOrRegister(name string, ctor func() interface{}) interface{} { - // fast path - cached, ok := r.metrics.Load(name) - if ok { - return cached - } - item, _, _ := r.loadOrRegister(name, ctor()) - return item -} - -// Register the given metric under the given name. Returns a ErrDuplicateMetric -// if a metric by the given name is already registered. -func (r *StandardRegistry) Register(name string, i interface{}) error { - // fast path - _, ok := r.metrics.Load(name) - if ok { - return fmt.Errorf("%w: %v", ErrDuplicateMetric, name) - } - - _, loaded, _ := r.loadOrRegister(name, i) - if loaded { - return fmt.Errorf("%w: %v", ErrDuplicateMetric, name) - } - return nil -} - -// RunHealthchecks run all registered healthchecks. -func (r *StandardRegistry) RunHealthchecks() { - r.metrics.Range(func(key, value any) bool { - if h, ok := value.(*Healthcheck); ok { - h.Check() - } - return true - }) -} - -// GetAll metrics in the Registry -func (r *StandardRegistry) GetAll() map[string]map[string]interface{} { - data := make(map[string]map[string]interface{}) - r.Each(func(name string, i interface{}) { - values := make(map[string]interface{}) - switch metric := i.(type) { - case *Counter: - values["count"] = metric.Snapshot().Count() - case *CounterFloat64: - values["count"] = metric.Snapshot().Count() - case *Gauge: - values["value"] = metric.Snapshot().Value() - case *GaugeFloat64: - values["value"] = metric.Snapshot().Value() - case *Healthcheck: - values["error"] = nil - metric.Check() - if err := metric.Error(); nil != err { - values["error"] = metric.Error().Error() - } - case Histogram: - h := metric.Snapshot() - ps := h.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999}) - values["count"] = h.Count() - values["min"] = h.Min() - values["max"] = h.Max() - values["mean"] = h.Mean() - values["stddev"] = h.StdDev() - values["median"] = ps[0] - values["75%"] = ps[1] - values["95%"] = ps[2] - values["99%"] = ps[3] - values["99.9%"] = ps[4] - case *Meter: - m := metric.Snapshot() - values["count"] = m.Count() - values["1m.rate"] = m.Rate1() - values["5m.rate"] = m.Rate5() - values["15m.rate"] = m.Rate15() - values["mean.rate"] = m.RateMean() - case *Timer: - t := metric.Snapshot() - ps := t.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999}) - values["count"] = t.Count() - values["min"] = t.Min() - values["max"] = t.Max() - values["mean"] = t.Mean() - values["stddev"] = t.StdDev() - values["median"] = ps[0] - values["75%"] = ps[1] - values["95%"] = ps[2] - values["99%"] = ps[3] - values["99.9%"] = ps[4] - values["1m.rate"] = t.Rate1() - values["5m.rate"] = t.Rate5() - values["15m.rate"] = t.Rate15() - values["mean.rate"] = t.RateMean() - } - data[name] = values - }) - return data -} - -// Unregister the metric with the given name. -func (r *StandardRegistry) Unregister(name string) { - r.stop(name) - r.metrics.LoadAndDelete(name) -} - -func (r *StandardRegistry) loadOrRegister(name string, i interface{}) (interface{}, bool, bool) { - switch i.(type) { - case *Counter, *CounterFloat64, *Gauge, *GaugeFloat64, *GaugeInfo, *Healthcheck, Histogram, *Meter, *Timer, *ResettingTimer: - default: - return nil, false, false - } - item, loaded := r.metrics.LoadOrStore(name, i) - return item, loaded, true -} - -func (r *StandardRegistry) registered() map[string]interface{} { - metrics := make(map[string]interface{}) - r.metrics.Range(func(key, value any) bool { - metrics[key.(string)] = value - return true - }) - return metrics -} - -func (r *StandardRegistry) stop(name string) { - if i, ok := r.metrics.Load(name); ok { - if s, ok := i.(Stoppable); ok { - s.Stop() - } - } -} - -// Stoppable defines the metrics which has to be stopped. -type Stoppable interface { - Stop() -} - -type PrefixedRegistry struct { - underlying Registry - prefix string -} - -func NewPrefixedRegistry(prefix string) Registry { - return &PrefixedRegistry{ - underlying: NewRegistry(), - prefix: prefix, - } -} - -func NewPrefixedChildRegistry(parent Registry, prefix string) Registry { - return &PrefixedRegistry{ - underlying: parent, - prefix: prefix, - } -} - -// Each call the given function for each registered metric. -func (r *PrefixedRegistry) Each(fn func(string, interface{})) { - wrappedFn := func(prefix string) func(string, interface{}) { - return func(name string, iface interface{}) { - if strings.HasPrefix(name, prefix) { - fn(name, iface) - } else { - return - } - } - } - - baseRegistry, prefix := findPrefix(r, "") - baseRegistry.Each(wrappedFn(prefix)) -} - -func findPrefix(registry Registry, prefix string) (Registry, string) { - switch r := registry.(type) { - case *PrefixedRegistry: - return findPrefix(r.underlying, r.prefix+prefix) - case *StandardRegistry: - return r, prefix - } - return nil, "" -} - -// Get the metric by the given name or nil if none is registered. -func (r *PrefixedRegistry) Get(name string) interface{} { - realName := r.prefix + name - return r.underlying.Get(realName) -} - -// GetOrRegister gets an existing metric or registers the given one. -// The interface can be the metric to register if not found in registry, -// or a function returning the metric for lazy instantiation. -func (r *PrefixedRegistry) GetOrRegister(name string, ctor func() interface{}) interface{} { - realName := r.prefix + name - return r.underlying.GetOrRegister(realName, ctor) -} - -// Register the given metric under the given name. The name will be prefixed. -func (r *PrefixedRegistry) Register(name string, metric interface{}) error { - realName := r.prefix + name - return r.underlying.Register(realName, metric) -} - -// RunHealthchecks run all registered healthchecks. -func (r *PrefixedRegistry) RunHealthchecks() { - r.underlying.RunHealthchecks() -} - -// GetAll metrics in the Registry -func (r *PrefixedRegistry) GetAll() map[string]map[string]interface{} { - return r.underlying.GetAll() -} - -// Unregister the metric with the given name. The name will be prefixed. -func (r *PrefixedRegistry) Unregister(name string) { - realName := r.prefix + name - r.underlying.Unregister(realName) -} - -var ( - DefaultRegistry = NewRegistry() -) - -// Each call the given function for each registered metric. -func Each(f func(string, interface{})) { - DefaultRegistry.Each(f) -} - -// Get the metric by the given name or nil if none is registered. -func Get(name string) interface{} { - return DefaultRegistry.Get(name) -} - -// GetOrRegister gets an existing metric or creates and registers a new one. Threadsafe -// alternative to calling Get and Register on failure. -func GetOrRegister(name string, i func() interface{}) interface{} { - return DefaultRegistry.GetOrRegister(name, i) -} - -func getOrRegister[T any](name string, ctor func() T, r Registry) T { - if r == nil { - r = DefaultRegistry - } - return r.GetOrRegister(name, func() any { return ctor() }).(T) -} - -// Register the given metric under the given name. Returns a ErrDuplicateMetric -// if a metric by the given name is already registered. -func Register(name string, i interface{}) error { - return DefaultRegistry.Register(name, i) -} - -// MustRegister register the given metric under the given name. Panics if a metric by the -// given name is already registered. -func MustRegister(name string, i interface{}) { - if err := Register(name, i); err != nil { - panic(err) - } -} - -// RunHealthchecks run all registered healthchecks. -func RunHealthchecks() { - DefaultRegistry.RunHealthchecks() -} - -// Unregister the metric with the given name. -func Unregister(name string) { - DefaultRegistry.Unregister(name) -} diff --git a/common/metrics/registry_test.go b/common/metrics/registry_test.go deleted file mode 100644 index 6af0796da95..00000000000 --- a/common/metrics/registry_test.go +++ /dev/null @@ -1,335 +0,0 @@ -package metrics - -import ( - "sync" - "testing" -) - -func BenchmarkRegistry(b *testing.B) { - r := NewRegistry() - r.Register("foo", NewCounter()) - b.ResetTimer() - for i := 0; i < b.N; i++ { - r.Each(func(string, interface{}) {}) - } -} - -func BenchmarkRegistryGetOrRegisterParallel_8(b *testing.B) { - benchmarkRegistryGetOrRegisterParallel(b, 8) -} - -func BenchmarkRegistryGetOrRegisterParallel_32(b *testing.B) { - benchmarkRegistryGetOrRegisterParallel(b, 32) -} - -func benchmarkRegistryGetOrRegisterParallel(b *testing.B, amount int) { - r := NewRegistry() - b.ResetTimer() - var wg sync.WaitGroup - for i := 0; i < amount; i++ { - wg.Add(1) - go func() { - for i := 0; i < b.N; i++ { - GetOrRegisterMeter("foo", r) - } - wg.Done() - }() - } - wg.Wait() -} - -func TestRegistry(t *testing.T) { - r := NewRegistry() - r.Register("foo", NewCounter()) - i := 0 - r.Each(func(name string, iface interface{}) { - i++ - if name != "foo" { - t.Fatal(name) - } - if _, ok := iface.(*Counter); !ok { - t.Fatal(iface) - } - }) - if i != 1 { - t.Fatal(i) - } - r.Unregister("foo") - i = 0 - r.Each(func(string, interface{}) { i++ }) - if i != 0 { - t.Fatal(i) - } -} - -func TestRegistryDuplicate(t *testing.T) { - r := NewRegistry() - if err := r.Register("foo", NewCounter()); nil != err { - t.Fatal(err) - } - if err := r.Register("foo", NewGauge()); nil == err { - t.Fatal(err) - } - i := 0 - r.Each(func(name string, iface interface{}) { - i++ - if _, ok := iface.(*Counter); !ok { - t.Fatal(iface) - } - }) - if i != 1 { - t.Fatal(i) - } -} - -func TestRegistryGet(t *testing.T) { - r := NewRegistry() - r.Register("foo", NewCounter()) - if count := r.Get("foo").(*Counter).Snapshot().Count(); count != 0 { - t.Fatal(count) - } - r.Get("foo").(*Counter).Inc(1) - if count := r.Get("foo").(*Counter).Snapshot().Count(); count != 1 { - t.Fatal(count) - } -} - -func TestRegistryGetOrRegister(t *testing.T) { - r := NewRegistry() - - // First metric wins with GetOrRegister - c1 := GetOrRegisterCounter("foo", r) - c2 := GetOrRegisterCounter("foo", r) - if c1 != c2 { - t.Fatal("counters should've matched") - } - - i := 0 - r.Each(func(name string, iface interface{}) { - i++ - if name != "foo" { - t.Fatal(name) - } - if _, ok := iface.(*Counter); !ok { - t.Fatal(iface) - } - }) - if i != 1 { - t.Fatal(i) - } -} - -func TestRegistryGetOrRegisterWithLazyInstantiation(t *testing.T) { - r := NewRegistry() - - // First metric wins with GetOrRegister - c1 := GetOrRegisterCounter("foo", r) - c2 := GetOrRegisterCounter("foo", r) - if c1 != c2 { - t.Fatal("counters should've matched") - } - - i := 0 - r.Each(func(name string, iface interface{}) { - i++ - if name != "foo" { - t.Fatal(name) - } - if _, ok := iface.(*Counter); !ok { - t.Fatal(iface) - } - }) - if i != 1 { - t.Fatal(i) - } -} - -func TestRegistryUnregister(t *testing.T) { - l := len(arbiter.meters) - r := NewRegistry() - r.Register("foo", NewCounter()) - r.Register("bar", NewMeter()) - r.Register("baz", NewTimer()) - if len(arbiter.meters) != l+2 { - t.Errorf("arbiter.meters: %d != %d\n", l+2, len(arbiter.meters)) - } - r.Unregister("foo") - r.Unregister("bar") - r.Unregister("baz") - if len(arbiter.meters) != l { - t.Errorf("arbiter.meters: %d != %d\n", l+2, len(arbiter.meters)) - } -} - -func TestPrefixedChildRegistryGetOrRegister(t *testing.T) { - r := NewRegistry() - pr := NewPrefixedChildRegistry(r, "prefix.") - - _ = GetOrRegisterCounter("foo", pr) - - i := 0 - r.Each(func(name string, m interface{}) { - i++ - if name != "prefix.foo" { - t.Fatal(name) - } - }) - if i != 1 { - t.Fatal(i) - } -} - -func TestPrefixedRegistryGetOrRegister(t *testing.T) { - r := NewPrefixedRegistry("prefix.") - - _ = GetOrRegisterCounter("foo", r) - - i := 0 - r.Each(func(name string, m interface{}) { - i++ - if name != "prefix.foo" { - t.Fatal(name) - } - }) - if i != 1 { - t.Fatal(i) - } -} - -func TestPrefixedRegistryRegister(t *testing.T) { - r := NewPrefixedRegistry("prefix.") - err := r.Register("foo", NewCounter()) - c := NewCounter() - Register("bar", c) - if err != nil { - t.Fatal(err.Error()) - } - - i := 0 - r.Each(func(name string, m interface{}) { - i++ - if name != "prefix.foo" { - t.Fatal(name) - } - }) - if i != 1 { - t.Fatal(i) - } -} - -func TestPrefixedRegistryUnregister(t *testing.T) { - r := NewPrefixedRegistry("prefix.") - - _ = r.Register("foo", NewCounter()) - - i := 0 - r.Each(func(name string, m interface{}) { - i++ - if name != "prefix.foo" { - t.Fatal(name) - } - }) - if i != 1 { - t.Fatal(i) - } - - r.Unregister("foo") - - i = 0 - r.Each(func(name string, m interface{}) { - i++ - }) - - if i != 0 { - t.Fatal(i) - } -} - -func TestPrefixedRegistryGet(t *testing.T) { - pr := NewPrefixedRegistry("prefix.") - name := "foo" - pr.Register(name, NewCounter()) - - fooCounter := pr.Get(name) - if fooCounter == nil { - t.Fatal(name) - } -} - -func TestPrefixedChildRegistryGet(t *testing.T) { - r := NewRegistry() - pr := NewPrefixedChildRegistry(r, "prefix.") - name := "foo" - pr.Register(name, NewCounter()) - fooCounter := pr.Get(name) - if fooCounter == nil { - t.Fatal(name) - } -} - -func TestChildPrefixedRegistryRegister(t *testing.T) { - r := NewPrefixedChildRegistry(DefaultRegistry, "prefix.") - err := r.Register("foo", NewCounter()) - c := NewCounter() - Register("bar", c) - if err != nil { - t.Fatal(err.Error()) - } - - i := 0 - r.Each(func(name string, m interface{}) { - i++ - if name != "prefix.foo" { - t.Fatal(name) - } - }) - if i != 1 { - t.Fatal(i) - } -} - -func TestChildPrefixedRegistryOfChildRegister(t *testing.T) { - r := NewPrefixedChildRegistry(NewRegistry(), "prefix.") - r2 := NewPrefixedChildRegistry(r, "prefix2.") - err := r.Register("foo2", NewCounter()) - if err != nil { - t.Fatal(err.Error()) - } - err = r2.Register("baz", NewCounter()) - if err != nil { - t.Fatal(err.Error()) - } - c := NewCounter() - Register("bars", c) - - i := 0 - r2.Each(func(name string, m interface{}) { - i++ - if name != "prefix.prefix2.baz" { - t.Fatal(name) - } - }) - if i != 1 { - t.Fatal(i) - } -} - -func TestWalkRegistries(t *testing.T) { - r := NewPrefixedChildRegistry(NewRegistry(), "prefix.") - r2 := NewPrefixedChildRegistry(r, "prefix2.") - err := r.Register("foo2", NewCounter()) - if err != nil { - t.Fatal(err.Error()) - } - err = r2.Register("baz", NewCounter()) - if err != nil { - t.Fatal(err.Error()) - } - c := NewCounter() - Register("bars", c) - - _, prefix := findPrefix(r2, "") - if prefix != "prefix.prefix2." { - t.Fatal(prefix) - } -} diff --git a/common/metrics/resetting_sample.go b/common/metrics/resetting_sample.go deleted file mode 100644 index 730ef93416d..00000000000 --- a/common/metrics/resetting_sample.go +++ /dev/null @@ -1,24 +0,0 @@ -package metrics - -// ResettingSample converts an ordinary sample into one that resets whenever its -// snapshot is retrieved. This will break for multi-monitor systems, but when only -// a single metric is being pushed out, this ensure that low-frequency events don't -// skew th charts indefinitely. -func ResettingSample(sample Sample) Sample { - return &resettingSample{ - Sample: sample, - } -} - -// resettingSample is a simple wrapper around a sample that resets it upon the -// snapshot retrieval. -type resettingSample struct { - Sample -} - -// Snapshot returns a read-only copy of the sample with the original reset. -func (rs *resettingSample) Snapshot() *sampleSnapshot { - s := rs.Sample.Snapshot() - rs.Sample.Clear() - return s -} diff --git a/common/metrics/resetting_timer.go b/common/metrics/resetting_timer.go deleted file mode 100644 index 8aa7dc1488f..00000000000 --- a/common/metrics/resetting_timer.go +++ /dev/null @@ -1,134 +0,0 @@ -package metrics - -import ( - "sync" - "time" -) - -// GetOrRegisterResettingTimer returns an existing ResettingTimer or constructs and registers a -// new ResettingTimer. -func GetOrRegisterResettingTimer(name string, r Registry) *ResettingTimer { - return getOrRegister(name, NewResettingTimer, r) -} - -// NewRegisteredResettingTimer constructs and registers a new ResettingTimer. -func NewRegisteredResettingTimer(name string, r Registry) *ResettingTimer { - c := NewResettingTimer() - if nil == r { - r = DefaultRegistry - } - r.Register(name, c) - return c -} - -// NewResettingTimer constructs a new ResettingTimer -func NewResettingTimer() *ResettingTimer { - return &ResettingTimer{ - values: make([]int64, 0, 10), - } -} - -// ResettingTimer is used for storing aggregated values for timers, which are reset on every flush interval. -type ResettingTimer struct { - values []int64 - sum int64 // sum is a running count of the total sum, used later to calculate mean - - mutex sync.Mutex -} - -// Snapshot resets the timer and returns a read-only copy of its contents. -func (t *ResettingTimer) Snapshot() *ResettingTimerSnapshot { - t.mutex.Lock() - defer t.mutex.Unlock() - snapshot := &ResettingTimerSnapshot{} - if len(t.values) > 0 { - snapshot.mean = float64(t.sum) / float64(len(t.values)) - snapshot.values = t.values - t.values = make([]int64, 0, 10) - } - t.sum = 0 - return snapshot -} - -// Time records the duration of the execution of the given function. -func (t *ResettingTimer) Time(f func()) { - ts := time.Now() - f() - t.Update(time.Since(ts)) -} - -// Update records the duration of an event. -func (t *ResettingTimer) Update(d time.Duration) { - if !metricsEnabled { - return - } - t.mutex.Lock() - defer t.mutex.Unlock() - t.values = append(t.values, int64(d)) - t.sum += int64(d) -} - -// UpdateSince records the duration of an event that started at a time and ends now. -func (t *ResettingTimer) UpdateSince(ts time.Time) { - t.Update(time.Since(ts)) -} - -// ResettingTimerSnapshot is a point-in-time copy of another ResettingTimer. -type ResettingTimerSnapshot struct { - values []int64 - mean float64 - max int64 - min int64 - thresholdBoundaries []float64 - calculated bool -} - -// Count return the length of the values from snapshot. -func (t *ResettingTimerSnapshot) Count() int { - return len(t.values) -} - -// Percentiles returns the boundaries for the input percentiles. -// note: this method is not thread safe -func (t *ResettingTimerSnapshot) Percentiles(percentiles []float64) []float64 { - t.calc(percentiles) - return t.thresholdBoundaries -} - -// Mean returns the mean of the snapshotted values -// note: this method is not thread safe -func (t *ResettingTimerSnapshot) Mean() float64 { - if !t.calculated { - t.calc(nil) - } - - return t.mean -} - -// Max returns the max of the snapshotted values -// note: this method is not thread safe -func (t *ResettingTimerSnapshot) Max() int64 { - if !t.calculated { - t.calc(nil) - } - return t.max -} - -// Min returns the min of the snapshotted values -// note: this method is not thread safe -func (t *ResettingTimerSnapshot) Min() int64 { - if !t.calculated { - t.calc(nil) - } - return t.min -} - -func (t *ResettingTimerSnapshot) calc(percentiles []float64) { - scores := CalculatePercentiles(t.values, percentiles) - t.thresholdBoundaries = scores - if len(t.values) == 0 { - return - } - t.min = t.values[0] - t.max = t.values[len(t.values)-1] -} diff --git a/common/metrics/resetting_timer_test.go b/common/metrics/resetting_timer_test.go deleted file mode 100644 index 4571fc8eb05..00000000000 --- a/common/metrics/resetting_timer_test.go +++ /dev/null @@ -1,197 +0,0 @@ -package metrics - -import ( - "testing" - "time" -) - -func TestResettingTimer(t *testing.T) { - tests := []struct { - values []int64 - start int - end int - wantP50 float64 - wantP95 float64 - wantP99 float64 - wantMean float64 - wantMin int64 - wantMax int64 - }{ - { - values: []int64{}, - start: 1, - end: 11, - wantP50: 5.5, wantP95: 10, wantP99: 10, - wantMin: 1, wantMax: 10, wantMean: 5.5, - }, - { - values: []int64{}, - start: 1, - end: 101, - wantP50: 50.5, wantP95: 95.94999999999999, wantP99: 99.99, - wantMin: 1, wantMax: 100, wantMean: 50.5, - }, - { - values: []int64{1}, - start: 0, - end: 0, - wantP50: 1, wantP95: 1, wantP99: 1, - wantMin: 1, wantMax: 1, wantMean: 1, - }, - { - values: []int64{0}, - start: 0, - end: 0, - wantP50: 0, wantP95: 0, wantP99: 0, - wantMin: 0, wantMax: 0, wantMean: 0, - }, - { - values: []int64{}, - start: 0, - end: 0, - wantP50: 0, wantP95: 0, wantP99: 0, - wantMin: 0, wantMax: 0, wantMean: 0, - }, - { - values: []int64{1, 10}, - start: 0, - end: 0, - wantP50: 5.5, wantP95: 10, wantP99: 10, - wantMin: 1, wantMax: 10, wantMean: 5.5, - }, - } - for i, tt := range tests { - timer := NewResettingTimer() - - for i := tt.start; i < tt.end; i++ { - tt.values = append(tt.values, int64(i)) - } - - for _, v := range tt.values { - timer.Update(time.Duration(v)) - } - snap := timer.Snapshot() - - ps := snap.Percentiles([]float64{0.50, 0.95, 0.99}) - - if have, want := snap.Min(), tt.wantMin; have != want { - t.Fatalf("%d: min: have %d, want %d", i, have, want) - } - if have, want := snap.Max(), tt.wantMax; have != want { - t.Fatalf("%d: max: have %d, want %d", i, have, want) - } - if have, want := snap.Mean(), tt.wantMean; have != want { - t.Fatalf("%d: mean: have %v, want %v", i, have, want) - } - if have, want := ps[0], tt.wantP50; have != want { - t.Errorf("%d: p50: have %v, want %v", i, have, want) - } - if have, want := ps[1], tt.wantP95; have != want { - t.Errorf("%d: p95: have %v, want %v", i, have, want) - } - if have, want := ps[2], tt.wantP99; have != want { - t.Errorf("%d: p99: have %v, want %v", i, have, want) - } - } -} - -func TestResettingTimerWithFivePercentiles(t *testing.T) { - tests := []struct { - values []int64 - start int - end int - wantP05 float64 - wantP20 float64 - wantP50 float64 - wantP95 float64 - wantP99 float64 - wantMean float64 - wantMin int64 - wantMax int64 - }{ - { - values: []int64{}, - start: 1, - end: 11, - wantP05: 1, wantP20: 2.2, wantP50: 5.5, wantP95: 10, wantP99: 10, - wantMin: 1, wantMax: 10, wantMean: 5.5, - }, - { - values: []int64{}, - start: 1, - end: 101, - wantP05: 5.050000000000001, wantP20: 20.200000000000003, wantP50: 50.5, wantP95: 95.94999999999999, wantP99: 99.99, - wantMin: 1, wantMax: 100, wantMean: 50.5, - }, - { - values: []int64{1}, - start: 0, - end: 0, - wantP05: 1, wantP20: 1, wantP50: 1, wantP95: 1, wantP99: 1, - wantMin: 1, wantMax: 1, wantMean: 1, - }, - { - values: []int64{0}, - start: 0, - end: 0, - wantP05: 0, wantP20: 0, wantP50: 0, wantP95: 0, wantP99: 0, - wantMin: 0, wantMax: 0, wantMean: 0, - }, - { - values: []int64{}, - start: 0, - end: 0, - wantP05: 0, wantP20: 0, wantP50: 0, wantP95: 0, wantP99: 0, - wantMin: 0, wantMax: 0, wantMean: 0, - }, - { - values: []int64{1, 10}, - start: 0, - end: 0, - wantP05: 1, wantP20: 1, wantP50: 5.5, wantP95: 10, wantP99: 10, - wantMin: 1, wantMax: 10, wantMean: 5.5, - }, - } - for ind, tt := range tests { - timer := NewResettingTimer() - - for i := tt.start; i < tt.end; i++ { - tt.values = append(tt.values, int64(i)) - } - - for _, v := range tt.values { - timer.Update(time.Duration(v)) - } - - snap := timer.Snapshot() - - ps := snap.Percentiles([]float64{0.05, 0.20, 0.50, 0.95, 0.99}) - - if tt.wantMin != snap.Min() { - t.Errorf("%d: min: got %d, want %d", ind, snap.Min(), tt.wantMin) - } - - if tt.wantMax != snap.Max() { - t.Errorf("%d: max: got %d, want %d", ind, snap.Max(), tt.wantMax) - } - - if tt.wantMean != snap.Mean() { - t.Errorf("%d: mean: got %.2f, want %.2f", ind, snap.Mean(), tt.wantMean) - } - if tt.wantP05 != ps[0] { - t.Errorf("%d: p05: got %v, want %v", ind, ps[0], tt.wantP05) - } - if tt.wantP20 != ps[1] { - t.Errorf("%d: p20: got %v, want %v", ind, ps[1], tt.wantP20) - } - if tt.wantP50 != ps[2] { - t.Errorf("%d: p50: got %v, want %v", ind, ps[2], tt.wantP50) - } - if tt.wantP95 != ps[3] { - t.Errorf("%d: p95: got %v, want %v", ind, ps[3], tt.wantP95) - } - if tt.wantP99 != ps[4] { - t.Errorf("%d: p99: got %v, want %v", ind, ps[4], tt.wantP99) - } - } -} diff --git a/common/metrics/runtimehistogram.go b/common/metrics/runtimehistogram.go deleted file mode 100644 index efbed498af8..00000000000 --- a/common/metrics/runtimehistogram.go +++ /dev/null @@ -1,298 +0,0 @@ -package metrics - -import ( - "math" - "runtime/metrics" - "sort" - "sync/atomic" -) - -func getOrRegisterRuntimeHistogram(name string, scale float64, r Registry) *runtimeHistogram { - constructor := func() Histogram { return newRuntimeHistogram(scale) } - return getOrRegister(name, constructor, r).(*runtimeHistogram) -} - -// runtimeHistogram wraps a runtime/metrics histogram. -type runtimeHistogram struct { - v atomic.Pointer[metrics.Float64Histogram] - scaleFactor float64 -} - -func newRuntimeHistogram(scale float64) *runtimeHistogram { - h := &runtimeHistogram{scaleFactor: scale} - h.update(new(metrics.Float64Histogram)) - return h -} - -func RuntimeHistogramFromData(scale float64, hist *metrics.Float64Histogram) *runtimeHistogram { - h := &runtimeHistogram{scaleFactor: scale} - h.update(hist) - return h -} - -func (h *runtimeHistogram) update(mh *metrics.Float64Histogram) { - if mh == nil { - // The update value can be nil if the current Go version doesn't support a - // requested metric. It's just easier to handle nil here than putting - // conditionals everywhere. - return - } - - s := metrics.Float64Histogram{ - Counts: make([]uint64, len(mh.Counts)), - Buckets: make([]float64, len(mh.Buckets)), - } - copy(s.Counts, mh.Counts) - for i, b := range mh.Buckets { - s.Buckets[i] = b * h.scaleFactor - } - h.v.Store(&s) -} - -func (h *runtimeHistogram) Clear() { - panic("runtimeHistogram does not support Clear") -} -func (h *runtimeHistogram) Update(int64) { - panic("runtimeHistogram does not support Update") -} - -// Snapshot returns a non-changing copy of the histogram. -func (h *runtimeHistogram) Snapshot() HistogramSnapshot { - hist := h.v.Load() - return newRuntimeHistogramSnapshot(hist) -} - -type runtimeHistogramSnapshot struct { - internal *metrics.Float64Histogram - calculated bool - // The following fields are (lazily) calculated based on 'internal' - mean float64 - count int64 - min int64 // min is the lowest sample value. - max int64 // max is the highest sample value. - variance float64 -} - -func newRuntimeHistogramSnapshot(h *metrics.Float64Histogram) *runtimeHistogramSnapshot { - return &runtimeHistogramSnapshot{ - internal: h, - } -} - -// calc calculates the values for the snapshot. This method is not threadsafe. -func (h *runtimeHistogramSnapshot) calc() { - h.calculated = true - var ( - count int64 // number of samples - sum float64 // approx sum of all sample values - min int64 - max float64 - ) - if len(h.internal.Counts) == 0 { - return - } - for i, c := range h.internal.Counts { - if c == 0 { - continue - } - if count == 0 { // Set min only first loop iteration - min = int64(math.Floor(h.internal.Buckets[i])) - } - count += int64(c) - sum += h.midpoint(i) * float64(c) - // Set max on every iteration - edge := h.internal.Buckets[i+1] - if math.IsInf(edge, 1) { - edge = h.internal.Buckets[i] - } - if edge > max { - max = edge - } - } - h.min = min - h.max = int64(max) - h.mean = sum / float64(count) - h.count = count -} - -// Count returns the sample count. -func (h *runtimeHistogramSnapshot) Count() int64 { - if !h.calculated { - h.calc() - } - return h.count -} - -// Size returns the size of the sample at the time the snapshot was taken. -func (h *runtimeHistogramSnapshot) Size() int { - return len(h.internal.Counts) -} - -// Mean returns an approximation of the mean. -func (h *runtimeHistogramSnapshot) Mean() float64 { - if !h.calculated { - h.calc() - } - return h.mean -} - -func (h *runtimeHistogramSnapshot) midpoint(bucket int) float64 { - high := h.internal.Buckets[bucket+1] - low := h.internal.Buckets[bucket] - if math.IsInf(high, 1) { - // The edge of the highest bucket can be +Inf, and it's supposed to mean that this - // bucket contains all remaining samples > low. We can't get the middle of an - // infinite range, so just return the lower bound of this bucket instead. - return low - } - if math.IsInf(low, -1) { - // Similarly, we can get -Inf in the left edge of the lowest bucket, - // and it means the bucket contains all remaining values < high. - return high - } - return (low + high) / 2 -} - -// StdDev approximates the standard deviation of the histogram. -func (h *runtimeHistogramSnapshot) StdDev() float64 { - return math.Sqrt(h.Variance()) -} - -// Variance approximates the variance of the histogram. -func (h *runtimeHistogramSnapshot) Variance() float64 { - if len(h.internal.Counts) == 0 { - return 0 - } - if !h.calculated { - h.calc() - } - if h.count <= 1 { - // There is no variance when there are zero or one items. - return 0 - } - // Variance is not calculated in 'calc', because it requires a second iteration. - // Therefore we calculate it lazily in this method, triggered either by - // a direct call to Variance or via StdDev. - if h.variance != 0.0 { - return h.variance - } - var sum float64 - - for i, c := range h.internal.Counts { - midpoint := h.midpoint(i) - d := midpoint - h.mean - sum += float64(c) * (d * d) - } - h.variance = sum / float64(h.count-1) - return h.variance -} - -// Percentile computes the p'th percentile value. -func (h *runtimeHistogramSnapshot) Percentile(p float64) float64 { - threshold := float64(h.Count()) * p - values := [1]float64{threshold} - h.computePercentiles(values[:]) - return values[0] -} - -// Percentiles computes all requested percentile values. -func (h *runtimeHistogramSnapshot) Percentiles(ps []float64) []float64 { - // Compute threshold values. We need these to be sorted - // for the percentile computation, but restore the original - // order later, so keep the indexes as well. - count := float64(h.Count()) - thresholds := make([]float64, len(ps)) - indexes := make([]int, len(ps)) - for i, percentile := range ps { - thresholds[i] = count * max(0, min(1.0, percentile)) - indexes[i] = i - } - sort.Sort(floatsAscendingKeepingIndex{thresholds, indexes}) - - // Now compute. The result is stored back into the thresholds slice. - h.computePercentiles(thresholds) - - // Put the result back into the requested order. - sort.Sort(floatsByIndex{thresholds, indexes}) - return thresholds -} - -func (h *runtimeHistogramSnapshot) computePercentiles(thresh []float64) { - var totalCount float64 - for i, count := range h.internal.Counts { - totalCount += float64(count) - - for len(thresh) > 0 && thresh[0] < totalCount { - thresh[0] = h.internal.Buckets[i] - thresh = thresh[1:] - } - if len(thresh) == 0 { - return - } - } -} - -// Note: runtime/metrics.Float64Histogram is a collection of float64s, but the methods -// below need to return int64 to satisfy the interface. The histogram provided by runtime -// also doesn't keep track of individual samples, so results are approximated. - -// Max returns the highest sample value. -func (h *runtimeHistogramSnapshot) Max() int64 { - if !h.calculated { - h.calc() - } - return h.max -} - -// Min returns the lowest sample value. -func (h *runtimeHistogramSnapshot) Min() int64 { - if !h.calculated { - h.calc() - } - return h.min -} - -// Sum returns the sum of all sample values. -func (h *runtimeHistogramSnapshot) Sum() int64 { - var sum float64 - for i := range h.internal.Counts { - sum += h.internal.Buckets[i] * float64(h.internal.Counts[i]) - } - return int64(math.Ceil(sum)) -} - -type floatsAscendingKeepingIndex struct { - values []float64 - indexes []int -} - -func (s floatsAscendingKeepingIndex) Len() int { - return len(s.values) -} - -func (s floatsAscendingKeepingIndex) Less(i, j int) bool { - return s.values[i] < s.values[j] -} - -func (s floatsAscendingKeepingIndex) Swap(i, j int) { - s.values[i], s.values[j] = s.values[j], s.values[i] - s.indexes[i], s.indexes[j] = s.indexes[j], s.indexes[i] -} - -type floatsByIndex struct { - values []float64 - indexes []int -} - -func (s floatsByIndex) Len() int { - return len(s.values) -} - -func (s floatsByIndex) Less(i, j int) bool { - return s.indexes[i] < s.indexes[j] -} - -func (s floatsByIndex) Swap(i, j int) { - s.values[i], s.values[j] = s.values[j], s.values[i] - s.indexes[i], s.indexes[j] = s.indexes[j], s.indexes[i] -} diff --git a/common/metrics/runtimehistogram_test.go b/common/metrics/runtimehistogram_test.go deleted file mode 100644 index cf7e36420ae..00000000000 --- a/common/metrics/runtimehistogram_test.go +++ /dev/null @@ -1,162 +0,0 @@ -package metrics - -import ( - "bytes" - "encoding/gob" - "fmt" - "math" - "reflect" - "runtime/metrics" - "testing" - "time" -) - -var _ Histogram = (*runtimeHistogram)(nil) - -type runtimeHistogramTest struct { - h metrics.Float64Histogram - - Count int64 - Min int64 - Max int64 - Sum int64 - Mean float64 - Variance float64 - StdDev float64 - Percentiles []float64 // .5 .8 .9 .99 .995 -} - -// This test checks the results of statistical functions implemented -// by runtimeHistogramSnapshot. -func TestRuntimeHistogramStats(t *testing.T) { - tests := []runtimeHistogramTest{ - 0: { - h: metrics.Float64Histogram{ - Counts: []uint64{}, - Buckets: []float64{}, - }, - Count: 0, - Max: 0, - Min: 0, - Sum: 0, - Mean: 0, - Variance: 0, - StdDev: 0, - Percentiles: []float64{0, 0, 0, 0, 0}, - }, - 1: { - // This checks the case where the highest bucket is +Inf. - h: metrics.Float64Histogram{ - Counts: []uint64{0, 1, 2}, - Buckets: []float64{0, 0.5, 1, math.Inf(1)}, - }, - Count: 3, - Max: 1, - Min: 0, - Sum: 3, - Mean: 0.9166666, - Percentiles: []float64{1, 1, 1, 1, 1}, - Variance: 0.020833, - StdDev: 0.144433, - }, - 2: { - h: metrics.Float64Histogram{ - Counts: []uint64{8, 6, 3, 1}, - Buckets: []float64{12, 16, 18, 24, 25}, - }, - Count: 18, - Max: 25, - Min: 12, - Sum: 270, - Mean: 16.75, - Variance: 10.3015, - StdDev: 3.2096, - Percentiles: []float64{16, 18, 18, 24, 24}, - }, - } - - for i, test := range tests { - t.Run(fmt.Sprint(i), func(t *testing.T) { - s := RuntimeHistogramFromData(1.0, &test.h).Snapshot() - - if v := s.Count(); v != test.Count { - t.Errorf("Count() = %v, want %v", v, test.Count) - } - if v := s.Min(); v != test.Min { - t.Errorf("Min() = %v, want %v", v, test.Min) - } - if v := s.Max(); v != test.Max { - t.Errorf("Max() = %v, want %v", v, test.Max) - } - if v := s.Sum(); v != test.Sum { - t.Errorf("Sum() = %v, want %v", v, test.Sum) - } - if v := s.Mean(); !approxEqual(v, test.Mean, 0.0001) { - t.Errorf("Mean() = %v, want %v", v, test.Mean) - } - if v := s.Variance(); !approxEqual(v, test.Variance, 0.0001) { - t.Errorf("Variance() = %v, want %v", v, test.Variance) - } - if v := s.StdDev(); !approxEqual(v, test.StdDev, 0.0001) { - t.Errorf("StdDev() = %v, want %v", v, test.StdDev) - } - ps := []float64{.5, .8, .9, .99, .995} - if v := s.Percentiles(ps); !reflect.DeepEqual(v, test.Percentiles) { - t.Errorf("Percentiles(%v) = %v, want %v", ps, v, test.Percentiles) - } - }) - } -} - -func approxEqual(x, y, ε float64) bool { - if math.IsInf(x, -1) && math.IsInf(y, -1) { - return true - } - if math.IsInf(x, 1) && math.IsInf(y, 1) { - return true - } - if math.IsNaN(x) && math.IsNaN(y) { - return true - } - return math.Abs(x-y) < ε -} - -// This test verifies that requesting Percentiles in unsorted order -// returns them in the requested order. -func TestRuntimeHistogramStatsPercentileOrder(t *testing.T) { - s := RuntimeHistogramFromData(1.0, &metrics.Float64Histogram{ - Counts: []uint64{1, 1, 1, 1, 1, 1, 1, 1, 1, 1}, - Buckets: []float64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, - }).Snapshot() - result := s.Percentiles([]float64{1, 0.2, 0.5, 0.1, 0.2}) - expected := []float64{10, 2, 5, 1, 2} - if !reflect.DeepEqual(result, expected) { - t.Fatal("wrong result:", result) - } -} - -func BenchmarkRuntimeHistogramSnapshotRead(b *testing.B) { - var sLatency = "7\xff\x81\x03\x01\x01\x10Float64Histogram\x01\xff\x82\x00\x01\x02\x01\x06Counts\x01\xff\x84\x00\x01\aBuckets\x01\xff\x86\x00\x00\x00\x16\xff\x83\x02\x01\x01\b[]uint64\x01\xff\x84\x00\x01\x06\x00\x00\x17\xff\x85\x02\x01\x01\t[]float64\x01\xff\x86\x00\x01\b\x00\x00\xfe\x06T\xff\x82\x01\xff\xa2\x00\xfe\r\xef\x00\x01\x02\x02\x04\x05\x04\b\x15\x17 B?6.L;$!2) \x1a? \x190aH7FY6#\x190\x1d\x14\x10\x1b\r\t\x04\x03\x01\x01\x00\x03\x02\x00\x03\x05\x05\x02\x02\x06\x04\v\x06\n\x15\x18\x13'&.\x12=H/L&\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\xff\xa3\xfe\xf0\xff\x00\xf8\x95\xd6&\xe8\v.q>\xf8\x95\xd6&\xe8\v.\x81>\xf8\xdfA:\xdc\x11ʼn>\xf8\x95\xd6&\xe8\v.\x91>\xf8:\x8c0\xe2\x8ey\x95>\xf8\xdfA:\xdc\x11ř>\xf8\x84\xf7C֔\x10\x9e>\xf8\x95\xd6&\xe8\v.\xa1>\xf8:\x8c0\xe2\x8ey\xa5>\xf8\xdfA:\xdc\x11ũ>\xf8\x84\xf7C֔\x10\xae>\xf8\x95\xd6&\xe8\v.\xb1>\xf8:\x8c0\xe2\x8ey\xb5>\xf8\xdfA:\xdc\x11Ź>\xf8\x84\xf7C֔\x10\xbe>\xf8\x95\xd6&\xe8\v.\xc1>\xf8:\x8c0\xe2\x8ey\xc5>\xf8\xdfA:\xdc\x11\xc5\xc9>\xf8\x84\xf7C֔\x10\xce>\xf8\x95\xd6&\xe8\v.\xd1>\xf8:\x8c0\xe2\x8ey\xd5>\xf8\xdfA:\xdc\x11\xc5\xd9>\xf8\x84\xf7C֔\x10\xde>\xf8\x95\xd6&\xe8\v.\xe1>\xf8:\x8c0\xe2\x8ey\xe5>\xf8\xdfA:\xdc\x11\xc5\xe9>\xf8\x84\xf7C֔\x10\xee>\xf8\x95\xd6&\xe8\v.\xf1>\xf8:\x8c0\xe2\x8ey\xf5>\xf8\xdfA:\xdc\x11\xc5\xf9>\xf8\x84\xf7C֔\x10\xfe>\xf8\x95\xd6&\xe8\v.\x01?\xf8:\x8c0\xe2\x8ey\x05?\xf8\xdfA:\xdc\x11\xc5\t?\xf8\x84\xf7C֔\x10\x0e?\xf8\x95\xd6&\xe8\v.\x11?\xf8:\x8c0\xe2\x8ey\x15?\xf8\xdfA:\xdc\x11\xc5\x19?\xf8\x84\xf7C֔\x10\x1e?\xf8\x95\xd6&\xe8\v.!?\xf8:\x8c0\xe2\x8ey%?\xf8\xdfA:\xdc\x11\xc5)?\xf8\x84\xf7C֔\x10.?\xf8\x95\xd6&\xe8\v.1?\xf8:\x8c0\xe2\x8ey5?\xf8\xdfA:\xdc\x11\xc59?\xf8\x84\xf7C֔\x10>?\xf8\x95\xd6&\xe8\v.A?\xf8:\x8c0\xe2\x8eyE?\xf8\xdfA:\xdc\x11\xc5I?\xf8\x84\xf7C֔\x10N?\xf8\x95\xd6&\xe8\v.Q?\xf8:\x8c0\xe2\x8eyU?\xf8\xdfA:\xdc\x11\xc5Y?\xf8\x84\xf7C֔\x10^?\xf8\x95\xd6&\xe8\v.a?\xf8:\x8c0\xe2\x8eye?\xf8\xdfA:\xdc\x11\xc5i?\xf8\x84\xf7C֔\x10n?\xf8\x95\xd6&\xe8\v.q?\xf8:\x8c0\xe2\x8eyu?\xf8\xdfA:\xdc\x11\xc5y?\xf8\x84\xf7C֔\x10~?\xf8\x95\xd6&\xe8\v.\x81?\xf8:\x8c0\xe2\x8ey\x85?\xf8\xdfA:\xdc\x11ʼn?\xf8\x84\xf7C֔\x10\x8e?\xf8\x95\xd6&\xe8\v.\x91?\xf8:\x8c0\xe2\x8ey\x95?\xf8\xdfA:\xdc\x11ř?\xf8\x84\xf7C֔\x10\x9e?\xf8\x95\xd6&\xe8\v.\xa1?\xf8:\x8c0\xe2\x8ey\xa5?\xf8\xdfA:\xdc\x11ũ?\xf8\x84\xf7C֔\x10\xae?\xf8\x95\xd6&\xe8\v.\xb1?\xf8:\x8c0\xe2\x8ey\xb5?\xf8\xdfA:\xdc\x11Ź?\xf8\x84\xf7C֔\x10\xbe?\xf8\x95\xd6&\xe8\v.\xc1?\xf8:\x8c0\xe2\x8ey\xc5?\xf8\xdfA:\xdc\x11\xc5\xc9?\xf8\x84\xf7C֔\x10\xce?\xf8\x95\xd6&\xe8\v.\xd1?\xf8:\x8c0\xe2\x8ey\xd5?\xf8\xdfA:\xdc\x11\xc5\xd9?\xf8\x84\xf7C֔\x10\xde?\xf8\x95\xd6&\xe8\v.\xe1?\xf8:\x8c0\xe2\x8ey\xe5?\xf8\xdfA:\xdc\x11\xc5\xe9?\xf8\x84\xf7C֔\x10\xee?\xf8\x95\xd6&\xe8\v.\xf1?\xf8:\x8c0\xe2\x8ey\xf5?\xf8\xdfA:\xdc\x11\xc5\xf9?\xf8\x84\xf7C֔\x10\xfe?\xf8\x95\xd6&\xe8\v.\x01@\xf8:\x8c0\xe2\x8ey\x05@\xf8\xdfA:\xdc\x11\xc5\t@\xf8\x84\xf7C֔\x10\x0e@\xf8\x95\xd6&\xe8\v.\x11@\xf8:\x8c0\xe2\x8ey\x15@\xf8\xdfA:\xdc\x11\xc5\x19@\xf8\x84\xf7C֔\x10\x1e@\xf8\x95\xd6&\xe8\v.!@\xf8:\x8c0\xe2\x8ey%@\xf8\xdfA:\xdc\x11\xc5)@\xf8\x84\xf7C֔\x10.@\xf8\x95\xd6&\xe8\v.1@\xf8:\x8c0\xe2\x8ey5@\xf8\xdfA:\xdc\x11\xc59@\xf8\x84\xf7C֔\x10>@\xf8\x95\xd6&\xe8\v.A@\xf8:\x8c0\xe2\x8eyE@\xf8\xdfA:\xdc\x11\xc5I@\xf8\x84\xf7C֔\x10N@\xf8\x95\xd6&\xe8\v.Q@\xf8:\x8c0\xe2\x8eyU@\xf8\xdfA:\xdc\x11\xc5Y@\xf8\x84\xf7C֔\x10^@\xf8\x95\xd6&\xe8\v.a@\xf8:\x8c0\xe2\x8eye@\xf8\xdfA:\xdc\x11\xc5i@\xf8\x84\xf7C֔\x10n@\xf8\x95\xd6&\xe8\v.q@\xf8:\x8c0\xe2\x8eyu@\xf8\xdfA:\xdc\x11\xc5y@\xf8\x84\xf7C֔\x10~@\xf8\x95\xd6&\xe8\v.\x81@\xf8:\x8c0\xe2\x8ey\x85@\xf8\xdfA:\xdc\x11ʼn@\xf8\x84\xf7C֔\x10\x8e@\xf8\x95\xd6&\xe8\v.\x91@\xf8:\x8c0\xe2\x8ey\x95@\xf8\xdfA:\xdc\x11ř@\xf8\x84\xf7C֔\x10\x9e@\xf8\x95\xd6&\xe8\v.\xa1@\xf8:\x8c0\xe2\x8ey\xa5@\xf8\xdfA:\xdc\x11ũ@\xf8\x84\xf7C֔\x10\xae@\xf8\x95\xd6&\xe8\v.\xb1@\xf8:\x8c0\xe2\x8ey\xb5@\xf8\xdfA:\xdc\x11Ź@\xf8\x84\xf7C֔\x10\xbe@\xf8\x95\xd6&\xe8\v.\xc1@\xf8:\x8c0\xe2\x8ey\xc5@\xf8\xdfA:\xdc\x11\xc5\xc9@\xf8\x84\xf7C֔\x10\xce@\xf8\x95\xd6&\xe8\v.\xd1@\xf8:\x8c0\xe2\x8ey\xd5@\xf8\xdfA:\xdc\x11\xc5\xd9@\xf8\x84\xf7C֔\x10\xde@\xf8\x95\xd6&\xe8\v.\xe1@\xf8:\x8c0\xe2\x8ey\xe5@\xf8\xdfA:\xdc\x11\xc5\xe9@\xf8\x84\xf7C֔\x10\xee@\xf8\x95\xd6&\xe8\v.\xf1@\xf8:\x8c0\xe2\x8ey\xf5@\xf8\xdfA:\xdc\x11\xc5\xf9@\xf8\x84\xf7C֔\x10\xfe@\xf8\x95\xd6&\xe8\v.\x01A\xfe\xf0\x7f\x00" - - dserialize := func(data string) *metrics.Float64Histogram { - var res metrics.Float64Histogram - if err := gob.NewDecoder(bytes.NewReader([]byte(data))).Decode(&res); err != nil { - panic(err) - } - return &res - } - latency := RuntimeHistogramFromData(float64(time.Second), dserialize(sLatency)) - b.ResetTimer() - b.ReportAllocs() - for i := 0; i < b.N; i++ { - snap := latency.Snapshot() - // These are the fields that influxdb accesses - _ = snap.Count() - _ = snap.Max() - _ = snap.Mean() - _ = snap.Min() - _ = snap.StdDev() - _ = snap.Variance() - _ = snap.Percentiles([]float64{0.25, 0.5, 0.75, 0.95, 0.99, 0.999, 0.9999}) - } -} diff --git a/common/metrics/sample.go b/common/metrics/sample.go deleted file mode 100644 index dc8167809fd..00000000000 --- a/common/metrics/sample.go +++ /dev/null @@ -1,425 +0,0 @@ -package metrics - -import ( - "math" - "math/rand" - "slices" - "sync" - "time" -) - -const rescaleThreshold = time.Hour - -// Sample maintains a statistically-significant selection of values from -// a stream. -type Sample interface { - Snapshot() *sampleSnapshot - Clear() - Update(int64) -} - -var ( - _ Sample = (*ExpDecaySample)(nil) - _ Sample = (*UniformSample)(nil) - _ Sample = (*resettingSample)(nil) -) - -// sampleSnapshot is a read-only copy of a Sample. -type sampleSnapshot struct { - count int64 - values []int64 - - max int64 - min int64 - mean float64 - sum int64 - variance float64 -} - -// newSampleSnapshotPrecalculated creates a read-only sampleSnapShot, using -// precalculated sums to avoid iterating the values -func newSampleSnapshotPrecalculated(count int64, values []int64, min, max, sum int64) *sampleSnapshot { - if len(values) == 0 { - return &sampleSnapshot{ - count: count, - values: values, - } - } - return &sampleSnapshot{ - count: count, - values: values, - max: max, - min: min, - mean: float64(sum) / float64(len(values)), - sum: sum, - } -} - -// newSampleSnapshot creates a read-only sampleSnapShot, and calculates some -// numbers. -func newSampleSnapshot(count int64, values []int64) *sampleSnapshot { - var ( - max int64 = math.MinInt64 - min int64 = math.MaxInt64 - sum int64 - ) - for _, v := range values { - sum += v - if v > max { - max = v - } - if v < min { - min = v - } - } - return newSampleSnapshotPrecalculated(count, values, min, max, sum) -} - -// Count returns the count of inputs at the time the snapshot was taken. -func (s *sampleSnapshot) Count() int64 { return s.count } - -// Max returns the maximal value at the time the snapshot was taken. -func (s *sampleSnapshot) Max() int64 { return s.max } - -// Mean returns the mean value at the time the snapshot was taken. -func (s *sampleSnapshot) Mean() float64 { return s.mean } - -// Min returns the minimal value at the time the snapshot was taken. -func (s *sampleSnapshot) Min() int64 { return s.min } - -// Percentile returns an arbitrary percentile of values at the time the -// snapshot was taken. -func (s *sampleSnapshot) Percentile(p float64) float64 { - return SamplePercentile(s.values, p) -} - -// Percentiles returns a slice of arbitrary percentiles of values at the time -// the snapshot was taken. -func (s *sampleSnapshot) Percentiles(ps []float64) []float64 { - return CalculatePercentiles(s.values, ps) -} - -// Size returns the size of the sample at the time the snapshot was taken. -func (s *sampleSnapshot) Size() int { return len(s.values) } - -// StdDev returns the standard deviation of values at the time the snapshot was -// taken. -func (s *sampleSnapshot) StdDev() float64 { - if s.variance == 0.0 { - s.variance = SampleVariance(s.mean, s.values) - } - return math.Sqrt(s.variance) -} - -// Sum returns the sum of values at the time the snapshot was taken. -func (s *sampleSnapshot) Sum() int64 { return s.sum } - -// Values returns a copy of the values in the sample. -func (s *sampleSnapshot) Values() []int64 { - return slices.Clone(s.values) -} - -// Variance returns the variance of values at the time the snapshot was taken. -func (s *sampleSnapshot) Variance() float64 { - if s.variance == 0.0 { - s.variance = SampleVariance(s.mean, s.values) - } - return s.variance -} - -// ExpDecaySample is an exponentially-decaying sample using a forward-decaying -// priority reservoir. See Cormode et al's "Forward Decay: A Practical Time -// Decay Model for Streaming Systems". -// -// -type ExpDecaySample struct { - alpha float64 - count int64 - mutex sync.Mutex - reservoirSize int - t0, t1 time.Time - values *expDecaySampleHeap - rand *rand.Rand -} - -// NewExpDecaySample constructs a new exponentially-decaying sample with the -// given reservoir size and alpha. -func NewExpDecaySample(reservoirSize int, alpha float64) Sample { - s := &ExpDecaySample{ - alpha: alpha, - reservoirSize: reservoirSize, - t0: time.Now(), - values: newExpDecaySampleHeap(reservoirSize), - } - s.t1 = s.t0.Add(rescaleThreshold) - return s -} - -// SetRand sets the random source (useful in tests) -func (s *ExpDecaySample) SetRand(prng *rand.Rand) Sample { - s.rand = prng - return s -} - -// Clear clears all samples. -func (s *ExpDecaySample) Clear() { - s.mutex.Lock() - defer s.mutex.Unlock() - s.count = 0 - s.t0 = time.Now() - s.t1 = s.t0.Add(rescaleThreshold) - s.values.Clear() -} - -// Snapshot returns a read-only copy of the sample. -func (s *ExpDecaySample) Snapshot() *sampleSnapshot { - s.mutex.Lock() - defer s.mutex.Unlock() - var ( - samples = s.values.Values() - values = make([]int64, len(samples)) - max int64 = math.MinInt64 - min int64 = math.MaxInt64 - sum int64 - ) - for i, item := range samples { - v := item.v - values[i] = v - sum += v - if v > max { - max = v - } - if v < min { - min = v - } - } - return newSampleSnapshotPrecalculated(s.count, values, min, max, sum) -} - -// Update samples a new value. -func (s *ExpDecaySample) Update(v int64) { - if !metricsEnabled { - return - } - s.update(time.Now(), v) -} - -// update samples a new value at a particular timestamp. This is a method all -// its own to facilitate testing. -func (s *ExpDecaySample) update(t time.Time, v int64) { - s.mutex.Lock() - defer s.mutex.Unlock() - s.count++ - if s.values.Size() == s.reservoirSize { - s.values.Pop() - } - var f64 float64 - if s.rand != nil { - f64 = s.rand.Float64() - } else { - f64 = rand.Float64() - } - s.values.Push(expDecaySample{ - k: math.Exp(t.Sub(s.t0).Seconds()*s.alpha) / f64, - v: v, - }) - if t.After(s.t1) { - values := s.values.Values() - t0 := s.t0 - s.values.Clear() - s.t0 = t - s.t1 = s.t0.Add(rescaleThreshold) - for _, v := range values { - v.k = v.k * math.Exp(-s.alpha*s.t0.Sub(t0).Seconds()) - s.values.Push(v) - } - } -} - -// SamplePercentile returns an arbitrary percentile of the slice of int64. -func SamplePercentile(values []int64, p float64) float64 { - return CalculatePercentiles(values, []float64{p})[0] -} - -// CalculatePercentiles returns a slice of arbitrary percentiles of the slice of -// int64. This method returns interpolated results, so e.g. if there are only two -// values, [0, 10], a 50% percentile will land between them. -// -// Note: As a side-effect, this method will also sort the slice of values. -// Note2: The input format for percentiles is NOT percent! To express 50%, use 0.5, not 50. -func CalculatePercentiles(values []int64, ps []float64) []float64 { - scores := make([]float64, len(ps)) - size := len(values) - if size == 0 { - return scores - } - slices.Sort(values) - for i, p := range ps { - pos := p * float64(size+1) - - if pos < 1.0 { - scores[i] = float64(values[0]) - } else if pos >= float64(size) { - scores[i] = float64(values[size-1]) - } else { - lower := float64(values[int(pos)-1]) - upper := float64(values[int(pos)]) - scores[i] = lower + (pos-math.Floor(pos))*(upper-lower) - } - } - return scores -} - -// SampleVariance returns the variance of the slice of int64. -func SampleVariance(mean float64, values []int64) float64 { - if len(values) == 0 { - return 0.0 - } - var sum float64 - for _, v := range values { - d := float64(v) - mean - sum += d * d - } - return sum / float64(len(values)) -} - -// UniformSample implements a uniform sample using Vitter's Algorithm R. -// -// -type UniformSample struct { - count int64 - mutex sync.Mutex - reservoirSize int - values []int64 - rand *rand.Rand -} - -// NewUniformSample constructs a new uniform sample with the given reservoir -// size. -func NewUniformSample(reservoirSize int) Sample { - return &UniformSample{ - reservoirSize: reservoirSize, - values: make([]int64, 0, reservoirSize), - } -} - -// SetRand sets the random source (useful in tests) -func (s *UniformSample) SetRand(prng *rand.Rand) Sample { - s.rand = prng - return s -} - -// Clear clears all samples. -func (s *UniformSample) Clear() { - s.mutex.Lock() - defer s.mutex.Unlock() - s.count = 0 - clear(s.values) -} - -// Snapshot returns a read-only copy of the sample. -func (s *UniformSample) Snapshot() *sampleSnapshot { - s.mutex.Lock() - values := slices.Clone(s.values) - count := s.count - s.mutex.Unlock() - return newSampleSnapshot(count, values) -} - -// Update samples a new value. -func (s *UniformSample) Update(v int64) { - if !metricsEnabled { - return - } - s.mutex.Lock() - defer s.mutex.Unlock() - s.count++ - if len(s.values) < s.reservoirSize { - s.values = append(s.values, v) - return - } - var r int64 - if s.rand != nil { - r = s.rand.Int63n(s.count) - } else { - r = rand.Int63n(s.count) - } - if r < int64(len(s.values)) { - s.values[int(r)] = v - } -} - -// expDecaySample represents an individual sample in a heap. -type expDecaySample struct { - k float64 - v int64 -} - -func newExpDecaySampleHeap(reservoirSize int) *expDecaySampleHeap { - return &expDecaySampleHeap{make([]expDecaySample, 0, reservoirSize)} -} - -// expDecaySampleHeap is a min-heap of expDecaySamples. -// The internal implementation is copied from the standard library's container/heap -type expDecaySampleHeap struct { - s []expDecaySample -} - -func (h *expDecaySampleHeap) Clear() { - h.s = h.s[:0] -} - -func (h *expDecaySampleHeap) Push(s expDecaySample) { - n := len(h.s) - h.s = h.s[0 : n+1] - h.s[n] = s - h.up(n) -} - -func (h *expDecaySampleHeap) Pop() expDecaySample { - n := len(h.s) - 1 - h.s[0], h.s[n] = h.s[n], h.s[0] - h.down(0, n) - - n = len(h.s) - s := h.s[n-1] - h.s = h.s[0 : n-1] - return s -} - -func (h *expDecaySampleHeap) Size() int { - return len(h.s) -} - -func (h *expDecaySampleHeap) Values() []expDecaySample { - return h.s -} - -func (h *expDecaySampleHeap) up(j int) { - for { - i := (j - 1) / 2 // parent - if i == j || !(h.s[j].k < h.s[i].k) { - break - } - h.s[i], h.s[j] = h.s[j], h.s[i] - j = i - } -} - -func (h *expDecaySampleHeap) down(i, n int) { - for { - j1 := 2*i + 1 - if j1 >= n || j1 < 0 { // j1 < 0 after int overflow - break - } - j := j1 // left child - if j2 := j1 + 1; j2 < n && !(h.s[j1].k < h.s[j2].k) { - j = j2 // = 2*i + 2 // right child - } - if !(h.s[j].k < h.s[i].k) { - break - } - h.s[i], h.s[j] = h.s[j], h.s[i] - i = j - } -} diff --git a/common/metrics/sample_test.go b/common/metrics/sample_test.go deleted file mode 100644 index 6619eb1e9eb..00000000000 --- a/common/metrics/sample_test.go +++ /dev/null @@ -1,324 +0,0 @@ -package metrics - -import ( - "math" - "math/rand" - "testing" - "time" -) - -const epsilonPercentile = .00000000001 - -// Benchmark{Compute,Copy}{1000,1000000} demonstrate that, even for relatively -// expensive computations like Variance, the cost of copying the Sample, as -// approximated by a make and copy, is much greater than the cost of the -// computation for small samples and only slightly less for large samples. -func BenchmarkCompute1000(b *testing.B) { - s := make([]int64, 1000) - var sum int64 - for i := 0; i < len(s); i++ { - s[i] = int64(i) - sum += int64(i) - } - mean := float64(sum) / float64(len(s)) - b.ResetTimer() - for i := 0; i < b.N; i++ { - SampleVariance(mean, s) - } -} - -func BenchmarkCompute1000000(b *testing.B) { - s := make([]int64, 1000000) - var sum int64 - for i := 0; i < len(s); i++ { - s[i] = int64(i) - sum += int64(i) - } - mean := float64(sum) / float64(len(s)) - b.ResetTimer() - for i := 0; i < b.N; i++ { - SampleVariance(mean, s) - } -} - -func BenchmarkExpDecaySample257(b *testing.B) { - benchmarkSample(b, NewExpDecaySample(257, 0.015)) -} - -func BenchmarkExpDecaySample514(b *testing.B) { - benchmarkSample(b, NewExpDecaySample(514, 0.015)) -} - -func BenchmarkExpDecaySample1028(b *testing.B) { - benchmarkSample(b, NewExpDecaySample(1028, 0.015)) -} - -func BenchmarkUniformSample257(b *testing.B) { - benchmarkSample(b, NewUniformSample(257)) -} - -func BenchmarkUniformSample514(b *testing.B) { - benchmarkSample(b, NewUniformSample(514)) -} - -func BenchmarkUniformSample1028(b *testing.B) { - benchmarkSample(b, NewUniformSample(1028)) -} - -func TestExpDecaySample(t *testing.T) { - for _, tc := range []struct { - reservoirSize int - alpha float64 - updates int - }{ - {100, 0.99, 10}, - {1000, 0.01, 100}, - {100, 0.99, 1000}, - } { - sample := NewExpDecaySample(tc.reservoirSize, tc.alpha) - for i := 0; i < tc.updates; i++ { - sample.Update(int64(i)) - } - snap := sample.Snapshot() - if have, want := int(snap.Count()), tc.updates; have != want { - t.Errorf("unexpected count: have %d want %d", have, want) - } - if have, want := snap.Size(), min(tc.updates, tc.reservoirSize); have != want { - t.Errorf("unexpected size: have %d want %d", have, want) - } - values := snap.values - if have, want := len(values), min(tc.updates, tc.reservoirSize); have != want { - t.Errorf("unexpected values length: have %d want %d", have, want) - } - for _, v := range values { - if v > int64(tc.updates) || v < 0 { - t.Errorf("out of range [0, %d]: %v", tc.updates, v) - } - } - } -} - -// This test makes sure that the sample's priority is not amplified by using -// nanosecond duration since start rather than second duration since start. -// The priority becomes +Inf quickly after starting if this is done, -// effectively freezing the set of samples until a rescale step happens. -func TestExpDecaySampleNanosecondRegression(t *testing.T) { - sw := NewExpDecaySample(1000, 0.99) - for i := 0; i < 1000; i++ { - sw.Update(10) - } - time.Sleep(1 * time.Millisecond) - for i := 0; i < 1000; i++ { - sw.Update(20) - } - v := sw.Snapshot().values - avg := float64(0) - for i := 0; i < len(v); i++ { - avg += float64(v[i]) - } - avg /= float64(len(v)) - if avg > 16 || avg < 14 { - t.Errorf("out of range [14, 16]: %v\n", avg) - } -} - -func TestExpDecaySampleRescale(t *testing.T) { - s := NewExpDecaySample(2, 0.001).(*ExpDecaySample) - s.update(time.Now(), 1) - s.update(time.Now().Add(time.Hour+time.Microsecond), 1) - for _, v := range s.values.Values() { - if v.k == 0.0 { - t.Fatal("v.k == 0.0") - } - } -} - -func TestExpDecaySampleSnapshot(t *testing.T) { - now := time.Now() - s := NewExpDecaySample(100, 0.99).(*ExpDecaySample).SetRand(rand.New(rand.NewSource(1))) - for i := 1; i <= 10000; i++ { - s.(*ExpDecaySample).update(now.Add(time.Duration(i)), int64(i)) - } - snapshot := s.Snapshot() - s.Update(1) - testExpDecaySampleStatistics(t, snapshot) -} - -func TestExpDecaySampleStatistics(t *testing.T) { - now := time.Now() - s := NewExpDecaySample(100, 0.99).(*ExpDecaySample).SetRand(rand.New(rand.NewSource(1))) - for i := 1; i <= 10000; i++ { - s.(*ExpDecaySample).update(now.Add(time.Duration(i)), int64(i)) - } - testExpDecaySampleStatistics(t, s.Snapshot()) -} - -func TestUniformSample(t *testing.T) { - sw := NewUniformSample(100) - for i := 0; i < 1000; i++ { - sw.Update(int64(i)) - } - s := sw.Snapshot() - if size := s.Count(); size != 1000 { - t.Errorf("s.Count(): 1000 != %v\n", size) - } - if size := s.Size(); size != 100 { - t.Errorf("s.Size(): 100 != %v\n", size) - } - values := s.values - - if l := len(values); l != 100 { - t.Errorf("len(s.Values()): 100 != %v\n", l) - } - for _, v := range values { - if v > 1000 || v < 0 { - t.Errorf("out of range [0, 1000]: %v\n", v) - } - } -} - -func TestUniformSampleIncludesTail(t *testing.T) { - sw := NewUniformSample(100) - max := 100 - for i := 0; i < max; i++ { - sw.Update(int64(i)) - } - v := sw.Snapshot().values - sum := 0 - exp := (max - 1) * max / 2 - for i := 0; i < len(v); i++ { - sum += int(v[i]) - } - if exp != sum { - t.Errorf("sum: %v != %v\n", exp, sum) - } -} - -func TestUniformSampleSnapshot(t *testing.T) { - s := NewUniformSample(100).(*UniformSample).SetRand(rand.New(rand.NewSource(1))) - for i := 1; i <= 10000; i++ { - s.Update(int64(i)) - } - snapshot := s.Snapshot() - s.Update(1) - testUniformSampleStatistics(t, snapshot) -} - -func TestUniformSampleStatistics(t *testing.T) { - s := NewUniformSample(100).(*UniformSample).SetRand(rand.New(rand.NewSource(1))) - for i := 1; i <= 10000; i++ { - s.Update(int64(i)) - } - testUniformSampleStatistics(t, s.Snapshot()) -} - -func benchmarkSample(b *testing.B, s Sample) { - for i := 0; i < b.N; i++ { - s.Update(1) - } -} - -func testExpDecaySampleStatistics(t *testing.T, s *sampleSnapshot) { - if sum := s.Sum(); sum != 496598 { - t.Errorf("s.Sum(): 496598 != %v\n", sum) - } - if count := s.Count(); count != 10000 { - t.Errorf("s.Count(): 10000 != %v\n", count) - } - if min := s.Min(); min != 107 { - t.Errorf("s.Min(): 107 != %v\n", min) - } - if max := s.Max(); max != 10000 { - t.Errorf("s.Max(): 10000 != %v\n", max) - } - if mean := s.Mean(); mean != 4965.98 { - t.Errorf("s.Mean(): 4965.98 != %v\n", mean) - } - if stdDev := s.StdDev(); stdDev != 2959.825156930727 { - t.Errorf("s.StdDev(): 2959.825156930727 != %v\n", stdDev) - } - ps := s.Percentiles([]float64{0.5, 0.75, 0.99}) - if ps[0] != 4615 { - t.Errorf("median: 4615 != %v\n", ps[0]) - } - if ps[1] != 7672 { - t.Errorf("75th percentile: 7672 != %v\n", ps[1]) - } - if ps[2] != 9998.99 { - t.Errorf("99th percentile: 9998.99 != %v\n", ps[2]) - } -} - -func testUniformSampleStatistics(t *testing.T, s *sampleSnapshot) { - if count := s.Count(); count != 10000 { - t.Errorf("s.Count(): 10000 != %v\n", count) - } - if min := s.Min(); min != 37 { - t.Errorf("s.Min(): 37 != %v\n", min) - } - if max := s.Max(); max != 9989 { - t.Errorf("s.Max(): 9989 != %v\n", max) - } - if mean := s.Mean(); mean != 4748.14 { - t.Errorf("s.Mean(): 4748.14 != %v\n", mean) - } - if stdDev := s.StdDev(); stdDev != 2826.684117548333 { - t.Errorf("s.StdDev(): 2826.684117548333 != %v\n", stdDev) - } - ps := s.Percentiles([]float64{0.5, 0.75, 0.99}) - if ps[0] != 4599 { - t.Errorf("median: 4599 != %v\n", ps[0]) - } - if ps[1] != 7380.5 { - t.Errorf("75th percentile: 7380.5 != %v\n", ps[1]) - } - if math.Abs(9986.429999999998-ps[2]) > epsilonPercentile { - t.Errorf("99th percentile: 9986.429999999998 != %v\n", ps[2]) - } -} - -// TestUniformSampleConcurrentUpdateCount would expose data race problems with -// concurrent Update and Count calls on Sample when test is called with -race -// argument -func TestUniformSampleConcurrentUpdateCount(t *testing.T) { - if testing.Short() { - t.Skip("skipping in short mode") - } - s := NewUniformSample(100) - for i := 0; i < 100; i++ { - s.Update(int64(i)) - } - quit := make(chan struct{}) - go func() { - t := time.NewTicker(10 * time.Millisecond) - defer t.Stop() - for { - select { - case <-t.C: - s.Update(rand.Int63()) - case <-quit: - t.Stop() - return - } - } - }() - for i := 0; i < 1000; i++ { - s.Snapshot().Count() - time.Sleep(5 * time.Millisecond) - } - quit <- struct{}{} -} - -func BenchmarkCalculatePercentiles(b *testing.B) { - pss := []float64{0.5, 0.75, 0.95, 0.99, 0.999, 0.9999} - var vals []int64 - for i := 0; i < 1000; i++ { - vals = append(vals, int64(rand.Int31())) - } - v := make([]int64, len(vals)) - b.ResetTimer() - for i := 0; i < b.N; i++ { - copy(v, vals) - _ = CalculatePercentiles(v, pss) - } -} diff --git a/common/metrics/syslog.go b/common/metrics/syslog.go deleted file mode 100644 index b265328f876..00000000000 --- a/common/metrics/syslog.go +++ /dev/null @@ -1,83 +0,0 @@ -//go:build !windows -// +build !windows - -package metrics - -import ( - "fmt" - "log/syslog" - "time" -) - -// Syslog outputs each metric in the given registry to syslog periodically using -// the given syslogger. -func Syslog(r Registry, d time.Duration, w *syslog.Writer) { - for range time.Tick(d) { - r.Each(func(name string, i interface{}) { - switch metric := i.(type) { - case *Counter: - w.Info(fmt.Sprintf("counter %s: count: %d", name, metric.Snapshot().Count())) - case *CounterFloat64: - w.Info(fmt.Sprintf("counter %s: count: %f", name, metric.Snapshot().Count())) - case *Gauge: - w.Info(fmt.Sprintf("gauge %s: value: %d", name, metric.Snapshot().Value())) - case *GaugeFloat64: - w.Info(fmt.Sprintf("gauge %s: value: %f", name, metric.Snapshot().Value())) - case *GaugeInfo: - w.Info(fmt.Sprintf("gauge %s: value: %s", name, metric.Snapshot().Value())) - case *Healthcheck: - metric.Check() - w.Info(fmt.Sprintf("healthcheck %s: error: %v", name, metric.Error())) - case Histogram: - h := metric.Snapshot() - ps := h.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999}) - w.Info(fmt.Sprintf( - "histogram %s: count: %d min: %d max: %d mean: %.2f stddev: %.2f median: %.2f 75%%: %.2f 95%%: %.2f 99%%: %.2f 99.9%%: %.2f", - name, - h.Count(), - h.Min(), - h.Max(), - h.Mean(), - h.StdDev(), - ps[0], - ps[1], - ps[2], - ps[3], - ps[4], - )) - case *Meter: - m := metric.Snapshot() - w.Info(fmt.Sprintf( - "meter %s: count: %d 1-min: %.2f 5-min: %.2f 15-min: %.2f mean: %.2f", - name, - m.Count(), - m.Rate1(), - m.Rate5(), - m.Rate15(), - m.RateMean(), - )) - case *Timer: - t := metric.Snapshot() - ps := t.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999}) - w.Info(fmt.Sprintf( - "timer %s: count: %d min: %d max: %d mean: %.2f stddev: %.2f median: %.2f 75%%: %.2f 95%%: %.2f 99%%: %.2f 99.9%%: %.2f 1-min: %.2f 5-min: %.2f 15-min: %.2f mean-rate: %.2f", - name, - t.Count(), - t.Min(), - t.Max(), - t.Mean(), - t.StdDev(), - ps[0], - ps[1], - ps[2], - ps[3], - ps[4], - t.Rate1(), - t.Rate5(), - t.Rate15(), - t.RateMean(), - )) - } - }) - } -} diff --git a/common/metrics/testdata/opentsb.want b/common/metrics/testdata/opentsb.want deleted file mode 100644 index 43fe1b2ac27..00000000000 --- a/common/metrics/testdata/opentsb.want +++ /dev/null @@ -1,23 +0,0 @@ -put pre.elite.count 978307200 1337 host=hal9000 -put pre.elite.one-minute 978307200 0.00 host=hal9000 -put pre.elite.five-minute 978307200 0.00 host=hal9000 -put pre.elite.fifteen-minute 978307200 0.00 host=hal9000 -put pre.elite.mean 978307200 0.00 host=hal9000 -put pre.foo.value 978307200 {"chain_id":"5"} host=hal9000 -put pre.months.count 978307200 12 host=hal9000 -put pre.pi.value 978307200 3.140000 host=hal9000 -put pre.second.count 978307200 1 host=hal9000 -put pre.second.min 978307200 1000 host=hal9000 -put pre.second.max 978307200 1000 host=hal9000 -put pre.second.mean 978307200 1000.00 host=hal9000 -put pre.second.std-dev 978307200 0.00 host=hal9000 -put pre.second.50-percentile 978307200 1000.00 host=hal9000 -put pre.second.75-percentile 978307200 1000.00 host=hal9000 -put pre.second.95-percentile 978307200 1000.00 host=hal9000 -put pre.second.99-percentile 978307200 1000.00 host=hal9000 -put pre.second.999-percentile 978307200 1000.00 host=hal9000 -put pre.second.one-minute 978307200 0.00 host=hal9000 -put pre.second.five-minute 978307200 0.00 host=hal9000 -put pre.second.fifteen-minute 978307200 0.00 host=hal9000 -put pre.second.mean-rate 978307200 0.00 host=hal9000 -put pre.tau.count 978307200 1.570000 host=hal9000 diff --git a/common/metrics/timer.go b/common/metrics/timer.go deleted file mode 100644 index 894bdfc3273..00000000000 --- a/common/metrics/timer.go +++ /dev/null @@ -1,149 +0,0 @@ -package metrics - -import ( - "sync" - "time" -) - -// GetOrRegisterTimer returns an existing Timer or constructs and registers a -// new Timer. -// Be sure to unregister the meter from the registry once it is of no use to -// allow for garbage collection. -func GetOrRegisterTimer(name string, r Registry) *Timer { - return getOrRegister(name, NewTimer, r) -} - -// NewCustomTimer constructs a new Timer from a Histogram and a Meter. -// Be sure to call Stop() once the timer is of no use to allow for garbage collection. -func NewCustomTimer(h Histogram, m *Meter) *Timer { - return &Timer{ - histogram: h, - meter: m, - } -} - -// NewRegisteredTimer constructs and registers a new Timer. -// Be sure to unregister the meter from the registry once it is of no use to -// allow for garbage collection. -func NewRegisteredTimer(name string, r Registry) *Timer { - c := NewTimer() - if nil == r { - r = DefaultRegistry - } - r.Register(name, c) - return c -} - -// NewTimer constructs a new Timer using an exponentially-decaying -// sample with the same reservoir size and alpha as UNIX load averages. -// Be sure to call Stop() once the timer is of no use to allow for garbage collection. -func NewTimer() *Timer { - return &Timer{ - histogram: NewHistogram(NewExpDecaySample(1028, 0.015)), - meter: NewMeter(), - } -} - -// Timer captures the duration and rate of events, using a Histogram and a Meter. -type Timer struct { - histogram Histogram - meter *Meter - mutex sync.Mutex -} - -// Snapshot returns a read-only copy of the timer. -func (t *Timer) Snapshot() *TimerSnapshot { - t.mutex.Lock() - defer t.mutex.Unlock() - return &TimerSnapshot{ - histogram: t.histogram.Snapshot(), - meter: t.meter.Snapshot(), - } -} - -// Stop stops the meter. -func (t *Timer) Stop() { - t.meter.Stop() -} - -// Time record the duration of the execution of the given function. -func (t *Timer) Time(f func()) { - ts := time.Now() - f() - t.Update(time.Since(ts)) -} - -// Update the duration of an event, in nanoseconds. -func (t *Timer) Update(d time.Duration) { - t.mutex.Lock() - defer t.mutex.Unlock() - t.histogram.Update(d.Nanoseconds()) - t.meter.Mark(1) -} - -// UpdateSince update the duration of an event that started at a time and ends now. -// The record uses nanoseconds. -func (t *Timer) UpdateSince(ts time.Time) { - t.Update(time.Since(ts)) -} - -// TimerSnapshot is a read-only copy of another Timer. -type TimerSnapshot struct { - histogram HistogramSnapshot - meter *MeterSnapshot -} - -// Count returns the number of events recorded at the time the snapshot was -// taken. -func (t *TimerSnapshot) Count() int64 { return t.histogram.Count() } - -// Max returns the maximum value at the time the snapshot was taken. -func (t *TimerSnapshot) Max() int64 { return t.histogram.Max() } - -// Size returns the size of the sample at the time the snapshot was taken. -func (t *TimerSnapshot) Size() int { return t.histogram.Size() } - -// Mean returns the mean value at the time the snapshot was taken. -func (t *TimerSnapshot) Mean() float64 { return t.histogram.Mean() } - -// Min returns the minimum value at the time the snapshot was taken. -func (t *TimerSnapshot) Min() int64 { return t.histogram.Min() } - -// Percentile returns an arbitrary percentile of sampled values at the time the -// snapshot was taken. -func (t *TimerSnapshot) Percentile(p float64) float64 { - return t.histogram.Percentile(p) -} - -// Percentiles returns a slice of arbitrary percentiles of sampled values at -// the time the snapshot was taken. -func (t *TimerSnapshot) Percentiles(ps []float64) []float64 { - return t.histogram.Percentiles(ps) -} - -// Rate1 returns the one-minute moving average rate of events per second at the -// time the snapshot was taken. -func (t *TimerSnapshot) Rate1() float64 { return t.meter.Rate1() } - -// Rate5 returns the five-minute moving average rate of events per second at -// the time the snapshot was taken. -func (t *TimerSnapshot) Rate5() float64 { return t.meter.Rate5() } - -// Rate15 returns the fifteen-minute moving average rate of events per second -// at the time the snapshot was taken. -func (t *TimerSnapshot) Rate15() float64 { return t.meter.Rate15() } - -// RateMean returns the meter's mean rate of events per second at the time the -// snapshot was taken. -func (t *TimerSnapshot) RateMean() float64 { return t.meter.RateMean() } - -// StdDev returns the standard deviation of the values at the time the snapshot -// was taken. -func (t *TimerSnapshot) StdDev() float64 { return t.histogram.StdDev() } - -// Sum returns the sum at the time the snapshot was taken. -func (t *TimerSnapshot) Sum() int64 { return t.histogram.Sum() } - -// Variance returns the variance of the values at the time the snapshot was -// taken. -func (t *TimerSnapshot) Variance() float64 { return t.histogram.Variance() } diff --git a/common/metrics/timer_test.go b/common/metrics/timer_test.go deleted file mode 100644 index f10de16c9c2..00000000000 --- a/common/metrics/timer_test.go +++ /dev/null @@ -1,114 +0,0 @@ -package metrics - -import ( - "fmt" - "math" - "testing" - "time" -) - -func BenchmarkTimer(b *testing.B) { - tm := NewTimer() - b.ResetTimer() - for i := 0; i < b.N; i++ { - tm.Update(1) - } -} - -func TestGetOrRegisterTimer(t *testing.T) { - r := NewRegistry() - NewRegisteredTimer("foo", r).Update(47) - if tm := GetOrRegisterTimer("foo", r).Snapshot(); tm.Count() != 1 { - t.Fatal(tm) - } -} - -func TestTimerExtremes(t *testing.T) { - tm := NewTimer() - tm.Update(math.MaxInt64) - tm.Update(0) - if stdDev := tm.Snapshot().StdDev(); stdDev != 4.611686018427388e+18 { - t.Errorf("tm.StdDev(): 4.611686018427388e+18 != %v\n", stdDev) - } -} - -func TestTimerStop(t *testing.T) { - l := len(arbiter.meters) - tm := NewTimer() - if l+1 != len(arbiter.meters) { - t.Errorf("arbiter.meters: %d != %d\n", l+1, len(arbiter.meters)) - } - tm.Stop() - if l != len(arbiter.meters) { - t.Errorf("arbiter.meters: %d != %d\n", l, len(arbiter.meters)) - } -} - -func TestTimerFunc(t *testing.T) { - var ( - tm = NewTimer() - testStart = time.Now() - actualTime time.Duration - ) - tm.Time(func() { - time.Sleep(50 * time.Millisecond) - actualTime = time.Since(testStart) - }) - var ( - drift = time.Millisecond * 2 - measured = time.Duration(tm.Snapshot().Max()) - ceil = actualTime + drift - floor = actualTime - drift - ) - if measured > ceil || measured < floor { - t.Errorf("tm.Max(): %v > %v || %v > %v\n", measured, ceil, measured, floor) - } -} - -func TestTimerZero(t *testing.T) { - tm := NewTimer().Snapshot() - if count := tm.Count(); count != 0 { - t.Errorf("tm.Count(): 0 != %v\n", count) - } - if min := tm.Min(); min != 0 { - t.Errorf("tm.Min(): 0 != %v\n", min) - } - if max := tm.Max(); max != 0 { - t.Errorf("tm.Max(): 0 != %v\n", max) - } - if mean := tm.Mean(); mean != 0.0 { - t.Errorf("tm.Mean(): 0.0 != %v\n", mean) - } - if stdDev := tm.StdDev(); stdDev != 0.0 { - t.Errorf("tm.StdDev(): 0.0 != %v\n", stdDev) - } - ps := tm.Percentiles([]float64{0.5, 0.75, 0.99}) - if ps[0] != 0.0 { - t.Errorf("median: 0.0 != %v\n", ps[0]) - } - if ps[1] != 0.0 { - t.Errorf("75th percentile: 0.0 != %v\n", ps[1]) - } - if ps[2] != 0.0 { - t.Errorf("99th percentile: 0.0 != %v\n", ps[2]) - } - if rate1 := tm.Rate1(); rate1 != 0.0 { - t.Errorf("tm.Rate1(): 0.0 != %v\n", rate1) - } - if rate5 := tm.Rate5(); rate5 != 0.0 { - t.Errorf("tm.Rate5(): 0.0 != %v\n", rate5) - } - if rate15 := tm.Rate15(); rate15 != 0.0 { - t.Errorf("tm.Rate15(): 0.0 != %v\n", rate15) - } - if rateMean := tm.RateMean(); rateMean != 0.0 { - t.Errorf("tm.RateMean(): 0.0 != %v\n", rateMean) - } -} - -func ExampleGetOrRegisterTimer() { - m := "account.create.latency" - t := GetOrRegisterTimer(m, nil) - t.Update(47) - fmt.Println(t.Snapshot().Max()) // Output: 47 -} diff --git a/common/metrics/validate.sh b/common/metrics/validate.sh deleted file mode 100755 index c4ae91e642d..00000000000 --- a/common/metrics/validate.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -set -e - -# check there are no formatting issues -GOFMT_LINES=`gofmt -l . | wc -l | xargs` -test $GOFMT_LINES -eq 0 || echo "gofmt needs to be run, ${GOFMT_LINES} files have issues" - -# run the tests for the root package -go test -race . diff --git a/common/metrics/writer.go b/common/metrics/writer.go deleted file mode 100644 index 2a41f8e1fe3..00000000000 --- a/common/metrics/writer.go +++ /dev/null @@ -1,99 +0,0 @@ -package metrics - -import ( - "fmt" - "io" - "slices" - "strings" - "time" -) - -// Write sorts writes each metric in the given registry periodically to the -// given io.Writer. -func Write(r Registry, d time.Duration, w io.Writer) { - for range time.Tick(d) { - WriteOnce(r, w) - } -} - -// WriteOnce sorts and writes metrics in the given registry to the given -// io.Writer. -func WriteOnce(r Registry, w io.Writer) { - var namedMetrics []namedMetric - r.Each(func(name string, i interface{}) { - namedMetrics = append(namedMetrics, namedMetric{name, i}) - }) - slices.SortFunc(namedMetrics, namedMetric.cmp) - for _, namedMetric := range namedMetrics { - switch metric := namedMetric.m.(type) { - case *Counter: - fmt.Fprintf(w, "counter %s\n", namedMetric.name) - fmt.Fprintf(w, " count: %9d\n", metric.Snapshot().Count()) - case *CounterFloat64: - fmt.Fprintf(w, "counter %s\n", namedMetric.name) - fmt.Fprintf(w, " count: %f\n", metric.Snapshot().Count()) - case *Gauge: - fmt.Fprintf(w, "gauge %s\n", namedMetric.name) - fmt.Fprintf(w, " value: %9d\n", metric.Snapshot().Value()) - case *GaugeFloat64: - fmt.Fprintf(w, "gauge %s\n", namedMetric.name) - fmt.Fprintf(w, " value: %f\n", metric.Snapshot().Value()) - case *GaugeInfo: - fmt.Fprintf(w, "gauge %s\n", namedMetric.name) - fmt.Fprintf(w, " value: %s\n", metric.Snapshot().Value().String()) - case *Healthcheck: - metric.Check() - fmt.Fprintf(w, "healthcheck %s\n", namedMetric.name) - fmt.Fprintf(w, " error: %v\n", metric.Error()) - case Histogram: - h := metric.Snapshot() - ps := h.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999}) - fmt.Fprintf(w, "histogram %s\n", namedMetric.name) - fmt.Fprintf(w, " count: %9d\n", h.Count()) - fmt.Fprintf(w, " min: %9d\n", h.Min()) - fmt.Fprintf(w, " max: %9d\n", h.Max()) - fmt.Fprintf(w, " mean: %12.2f\n", h.Mean()) - fmt.Fprintf(w, " stddev: %12.2f\n", h.StdDev()) - fmt.Fprintf(w, " median: %12.2f\n", ps[0]) - fmt.Fprintf(w, " 75%%: %12.2f\n", ps[1]) - fmt.Fprintf(w, " 95%%: %12.2f\n", ps[2]) - fmt.Fprintf(w, " 99%%: %12.2f\n", ps[3]) - fmt.Fprintf(w, " 99.9%%: %12.2f\n", ps[4]) - case *Meter: - m := metric.Snapshot() - fmt.Fprintf(w, "meter %s\n", namedMetric.name) - fmt.Fprintf(w, " count: %9d\n", m.Count()) - fmt.Fprintf(w, " 1-min rate: %12.2f\n", m.Rate1()) - fmt.Fprintf(w, " 5-min rate: %12.2f\n", m.Rate5()) - fmt.Fprintf(w, " 15-min rate: %12.2f\n", m.Rate15()) - fmt.Fprintf(w, " mean rate: %12.2f\n", m.RateMean()) - case *Timer: - t := metric.Snapshot() - ps := t.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999}) - fmt.Fprintf(w, "timer %s\n", namedMetric.name) - fmt.Fprintf(w, " count: %9d\n", t.Count()) - fmt.Fprintf(w, " min: %9d\n", t.Min()) - fmt.Fprintf(w, " max: %9d\n", t.Max()) - fmt.Fprintf(w, " mean: %12.2f\n", t.Mean()) - fmt.Fprintf(w, " stddev: %12.2f\n", t.StdDev()) - fmt.Fprintf(w, " median: %12.2f\n", ps[0]) - fmt.Fprintf(w, " 75%%: %12.2f\n", ps[1]) - fmt.Fprintf(w, " 95%%: %12.2f\n", ps[2]) - fmt.Fprintf(w, " 99%%: %12.2f\n", ps[3]) - fmt.Fprintf(w, " 99.9%%: %12.2f\n", ps[4]) - fmt.Fprintf(w, " 1-min rate: %12.2f\n", t.Rate1()) - fmt.Fprintf(w, " 5-min rate: %12.2f\n", t.Rate5()) - fmt.Fprintf(w, " 15-min rate: %12.2f\n", t.Rate15()) - fmt.Fprintf(w, " mean rate: %12.2f\n", t.RateMean()) - } - } -} - -type namedMetric struct { - name string - m interface{} -} - -func (m namedMetric) cmp(other namedMetric) int { - return strings.Compare(m.name, other.name) -} diff --git a/common/metrics/writer_test.go b/common/metrics/writer_test.go deleted file mode 100644 index edcfe955abc..00000000000 --- a/common/metrics/writer_test.go +++ /dev/null @@ -1,22 +0,0 @@ -package metrics - -import ( - "slices" - "testing" -) - -func TestMetricsSorting(t *testing.T) { - var namedMetrics = []namedMetric{ - {name: "zzz"}, - {name: "bbb"}, - {name: "fff"}, - {name: "ggg"}, - } - - slices.SortFunc(namedMetrics, namedMetric.cmp) - for i, name := range []string{"bbb", "fff", "ggg", "zzz"} { - if namedMetrics[i].name != name { - t.Fail() - } - } -} diff --git a/common/metrics/block_metrics.go b/diagnostics/metrics/block_metrics.go similarity index 51% rename from common/metrics/block_metrics.go rename to diagnostics/metrics/block_metrics.go index 7413a064aeb..c32580d1791 100644 --- a/common/metrics/block_metrics.go +++ b/diagnostics/metrics/block_metrics.go @@ -1,26 +1,9 @@ -// Copyright 2024 The Erigon Authors -// This file is part of Erigon. -// -// Erigon is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// Erigon is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with Erigon. If not, see . - package metrics import ( "time" "github.com/erigontech/erigon/common/log/v3" - "github.com/erigontech/erigon/diagnostics/metrics" ) var ( @@ -28,18 +11,18 @@ var ( DelayLoggingEnabled bool - BlockConsumerHeaderDownloadDelay = metrics.NewSummary(`block_consumer_delay{type="header_download"}`) - BlockConsumerBodyDownloadDelay = metrics.NewSummary(`block_consumer_delay{type="body_download"}`) - BlockConsumerPreExecutionDelay = metrics.NewSummary(`block_consumer_delay{type="pre_execution"}`) - BlockConsumerPostExecutionDelay = metrics.NewSummary(`block_consumer_delay{type="post_execution"}`) - BlockProducerProductionDelay = metrics.NewSummary(`block_producer_delay{type="production"}`) + BlockConsumerHeaderDownloadDelay = NewSummary(`block_consumer_delay{type="header_download"}`) + BlockConsumerBodyDownloadDelay = NewSummary(`block_consumer_delay{type="body_download"}`) + BlockConsumerPreExecutionDelay = NewSummary(`block_consumer_delay{type="pre_execution"}`) + BlockConsumerPostExecutionDelay = NewSummary(`block_consumer_delay{type="post_execution"}`) + BlockProducerProductionDelay = NewSummary(`block_producer_delay{type="production"}`) - ChainTipMgasPerSec = metrics.NewGauge(`chain_tip_mgas_per_sec`) + ChainTipMgasPerSec = NewGauge(`chain_tip_mgas_per_sec`) - BlockConsumerHeaderDownloadDelayHistogram = metrics.NewHistogram(`block_consumer_delay_hist{type="header_download"}`, delayBuckets) - BlockConsumerBodyDownloadDelayHistogram = metrics.NewHistogram(`block_consumer_delay_hist{type="body_download"}`, delayBuckets) - BlockConsumerPreExecutionDelayHistogram = metrics.NewHistogram(`block_consumer_delay_hist{type="pre_execution"}`, delayBuckets) - BlockConsumerPostExecutionDelayHistogram = metrics.NewHistogram(`block_consumer_delay_hist{type="post_execution"}`, delayBuckets) + BlockConsumerHeaderDownloadDelayHistogram = NewHistogram(`block_consumer_delay_hist{type="header_download"}`, delayBuckets) + BlockConsumerBodyDownloadDelayHistogram = NewHistogram(`block_consumer_delay_hist{type="body_download"}`, delayBuckets) + BlockConsumerPreExecutionDelayHistogram = NewHistogram(`block_consumer_delay_hist{type="pre_execution"}`, delayBuckets) + BlockConsumerPostExecutionDelayHistogram = NewHistogram(`block_consumer_delay_hist{type="post_execution"}`, delayBuckets) ) func UpdateBlockConsumerHeaderDownloadDelay(blockTime uint64, blockNumber uint64, log log.Logger) { diff --git a/diagnostics/metrics/config.go b/diagnostics/metrics/config.go new file mode 100644 index 00000000000..dbd93dd5b3e --- /dev/null +++ b/diagnostics/metrics/config.go @@ -0,0 +1,17 @@ +package metrics + +// Config contains the configuration for the metric collection. +type Config struct { + Enabled bool `toml:",omitempty"` + EnabledExpensive bool `toml:",omitempty"` + HTTP string `toml:",omitempty"` + Port int `toml:",omitempty"` +} + +// DefaultConfig is the default config for metrics. +var DefaultConfig = Config{ + Enabled: false, + EnabledExpensive: false, + HTTP: "127.0.0.1", + Port: 6061, +} diff --git a/execution/execmodule/forkchoice.go b/execution/execmodule/forkchoice.go index 4b1160f7ba7..6708c2d8564 100644 --- a/execution/execmodule/forkchoice.go +++ b/execution/execmodule/forkchoice.go @@ -28,7 +28,7 @@ import ( "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/common/dbg" "github.com/erigontech/erigon/common/log/v3" - "github.com/erigontech/erigon/common/metrics" + "github.com/erigontech/erigon/diagnostics/metrics" "github.com/erigontech/erigon/db/consensuschain" "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/kv/rawdbv3" diff --git a/execution/execmodule/inserters.go b/execution/execmodule/inserters.go index aba5d772730..f6b9b0839ad 100644 --- a/execution/execmodule/inserters.go +++ b/execution/execmodule/inserters.go @@ -21,7 +21,7 @@ import ( "fmt" "math/big" - "github.com/erigontech/erigon/common/metrics" + "github.com/erigontech/erigon/diagnostics/metrics" "github.com/erigontech/erigon/db/rawdb" "github.com/erigontech/erigon/execution/execmodule/moduleutil" "github.com/erigontech/erigon/execution/types" diff --git a/execution/stagedsync/headerdownload/header_algos.go b/execution/stagedsync/headerdownload/header_algos.go index 335061a577b..1087e90c60d 100644 --- a/execution/stagedsync/headerdownload/header_algos.go +++ b/execution/stagedsync/headerdownload/header_algos.go @@ -35,7 +35,7 @@ import ( "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/common/dbg" "github.com/erigontech/erigon/common/log/v3" - "github.com/erigontech/erigon/common/metrics" + "github.com/erigontech/erigon/diagnostics/metrics" "github.com/erigontech/erigon/db/config3" "github.com/erigontech/erigon/db/etl" "github.com/erigontech/erigon/db/kv" diff --git a/execution/stagedsync/stage_bodies.go b/execution/stagedsync/stage_bodies.go index 1081c106a78..6ed36ce4b0c 100644 --- a/execution/stagedsync/stage_bodies.go +++ b/execution/stagedsync/stage_bodies.go @@ -25,7 +25,7 @@ import ( "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/common/dbg" "github.com/erigontech/erigon/common/log/v3" - "github.com/erigontech/erigon/common/metrics" + "github.com/erigontech/erigon/diagnostics/metrics" "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/rawdb" "github.com/erigontech/erigon/db/rawdb/blockio" diff --git a/execution/stagedsync/stage_mining_exec.go b/execution/stagedsync/stage_mining_exec.go index 0eb31fffed4..d961112bc29 100644 --- a/execution/stagedsync/stage_mining_exec.go +++ b/execution/stagedsync/stage_mining_exec.go @@ -29,7 +29,7 @@ import ( "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/common/dbg" "github.com/erigontech/erigon/common/log/v3" - "github.com/erigontech/erigon/common/metrics" + "github.com/erigontech/erigon/diagnostics/metrics" "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/kv/membatchwithdb" "github.com/erigontech/erigon/db/kv/temporal" diff --git a/execution/stagedsync/stageloop/stageloop.go b/execution/stagedsync/stageloop/stageloop.go index 678d33f6656..29448c28cdd 100644 --- a/execution/stagedsync/stageloop/stageloop.go +++ b/execution/stagedsync/stageloop/stageloop.go @@ -26,7 +26,7 @@ import ( "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/common/dbg" "github.com/erigontech/erigon/common/log/v3" - "github.com/erigontech/erigon/common/metrics" + "github.com/erigontech/erigon/diagnostics/metrics" "github.com/erigontech/erigon/db/datadir" "github.com/erigontech/erigon/db/downloader" "github.com/erigontech/erigon/db/kv" diff --git a/node/logging/logging.go b/node/logging/logging.go index 08ed9debdf3..f04491070a0 100644 --- a/node/logging/logging.go +++ b/node/logging/logging.go @@ -27,7 +27,7 @@ import ( "gopkg.in/natefinch/lumberjack.v2" "github.com/erigontech/erigon/common/log/v3" - "github.com/erigontech/erigon/common/metrics" + "github.com/erigontech/erigon/diagnostics/metrics" ) // Determine the log dir path based on the given urfave context diff --git a/p2p/discover/metrics.go b/p2p/discover/metrics.go index ed802430ea0..1accb017801 100644 --- a/p2p/discover/metrics.go +++ b/p2p/discover/metrics.go @@ -21,27 +21,27 @@ import ( "net" "net/netip" - "github.com/erigontech/erigon/common/metrics" + "github.com/erigontech/erigon/diagnostics/metrics" ) const ( moduleName = "discover" // ingressMeterName is the prefix of the per-packet inbound metrics. - ingressMeterName = moduleName + "/ingress" + ingressMeterName = moduleName + "_ingress" // egressMeterName is the prefix of the per-packet outbound metrics. - egressMeterName = moduleName + "/egress" + egressMeterName = moduleName + "_egress" ) var ( - bucketsCounter []*metrics.Counter - ingressTrafficMeter = metrics.NewRegisteredMeter(ingressMeterName, nil) - egressTrafficMeter = metrics.NewRegisteredMeter(egressMeterName, nil) + bucketsCounter []metrics.Gauge + ingressTrafficMeter = metrics.NewCounter(ingressMeterName) + egressTrafficMeter = metrics.NewCounter(egressMeterName) ) func init() { for i := 0; i < nBuckets; i++ { - bucketsCounter = append(bucketsCounter, metrics.NewRegisteredCounter(fmt.Sprintf("%s/bucket/%d/count", moduleName, i), nil)) + bucketsCounter = append(bucketsCounter, metrics.NewGauge(fmt.Sprintf("%s_bucket_%d_count", moduleName, i))) } } @@ -52,10 +52,6 @@ type meteredUdpConn struct { } func newMeteredConn(conn UDPConn) UDPConn { - // Short circuit if metrics are disabled - if !metrics.Enabled() { - return conn - } return &meteredUdpConn{udpConn: conn} } @@ -70,13 +66,13 @@ func (c *meteredUdpConn) LocalAddr() net.Addr { // ReadFromUDPAddrPort delegates a network read to the underlying connection, bumping the udp ingress traffic meter along the way. func (c *meteredUdpConn) ReadFromUDPAddrPort(b []byte) (n int, addr netip.AddrPort, err error) { n, addr, err = c.udpConn.ReadFromUDPAddrPort(b) - ingressTrafficMeter.Mark(int64(n)) + ingressTrafficMeter.Add(float64(n)) return n, addr, err } // WriteToUDPAddrPort delegates a network write to the underlying connection, bumping the udp egress traffic meter along the way. func (c *meteredUdpConn) WriteToUDPAddrPort(b []byte, addr netip.AddrPort) (n int, err error) { n, err = c.udpConn.WriteToUDPAddrPort(b, addr) - egressTrafficMeter.Mark(int64(n)) + egressTrafficMeter.Add(float64(n)) return n, err } diff --git a/p2p/discover/table.go b/p2p/discover/table.go index feaa752649d..384cf18ca96 100644 --- a/p2p/discover/table.go +++ b/p2p/discover/table.go @@ -33,7 +33,6 @@ import ( "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/common/mclock" - "github.com/erigontech/erigon/common/metrics" "github.com/erigontech/erigon/p2p/enode" "github.com/erigontech/erigon/p2p/event" "github.com/erigontech/erigon/p2p/netutil" @@ -573,9 +572,7 @@ func (tab *Table) nodeAdded(b *bucket, n *tableNode) { if tab.nodeAddedHook != nil { tab.nodeAddedHook(b, n) } - if metrics.Enabled() { - bucketsCounter[b.index].Inc(1) - } + bucketsCounter[b.index].Inc() } func (tab *Table) nodeRemoved(b *bucket, n *tableNode) { @@ -583,9 +580,7 @@ func (tab *Table) nodeRemoved(b *bucket, n *tableNode) { if tab.nodeRemovedHook != nil { tab.nodeRemovedHook(b, n) } - if metrics.Enabled() { - bucketsCounter[b.index].Dec(1) - } + bucketsCounter[b.index].Dec() } // deleteInBucket removes node n from the table. From 545a045d5d4819ae2347dfd6a2721b4bd5c1d226 Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Tue, 3 Feb 2026 18:22:10 +1100 Subject: [PATCH 07/16] Move diagnostics/metrics back into common --- cl/monitor/metrics.go | 2 +- cl/monitor/shuffling_metrics/shuffling_metrics.go | 2 +- cl/phase1/core/state/lru/lru.go | 2 +- cl/phase1/core/state/ssz.go | 2 +- cmd/capcli/cli.go | 2 +- cmd/erigon/main.go | 2 +- cmd/utils/flags.go | 2 +- common/disk/disk.go | 2 +- common/disk/disk_darwin.go | 2 +- common/disk/disk_linux.go | 2 +- {diagnostics => common}/metrics/block_metrics.go | 0 {diagnostics => common}/metrics/config.go | 0 {diagnostics => common}/metrics/counter.go | 0 {diagnostics => common}/metrics/duration_observer.go | 0 {diagnostics => common}/metrics/ema.go | 0 {diagnostics => common}/metrics/gauge.go | 0 {diagnostics => common}/metrics/gaugevec.go | 0 {diagnostics => common}/metrics/histogram.go | 0 {diagnostics => common}/metrics/parsing.go | 0 {diagnostics => common}/metrics/register.go | 0 {diagnostics => common}/metrics/set.go | 0 {diagnostics => common}/metrics/setup.go | 0 {diagnostics => common}/metrics/summary.go | 0 {diagnostics => common}/metrics/timer.go | 0 {diagnostics => common}/metrics/value_getter.go | 0 db/kv/kv_interface.go | 2 +- db/kv/kvcache/cache.go | 2 +- db/kv/table_sizes.go | 2 +- db/rawdb/blockio/block_writer.go | 2 +- db/snapshotsync/freezeblocks/block_snapshots.go | 2 +- db/state/domain.go | 2 +- db/state/execctx/domain_shared.go | 2 +- db/state/inverted_index.go | 2 +- db/state/metrics.go | 2 +- diagnostics/mem/mem_linux.go | 2 +- execution/commitment/commitment.go | 2 +- execution/commitment/commitmentdb/commitment_context.go | 2 +- execution/exec/state.go | 2 +- execution/execmodule/forkchoice.go | 2 +- execution/execmodule/inserters.go | 2 +- execution/execmodule/metrics.go | 2 +- execution/protocol/block_exec.go | 2 +- execution/stagedsync/exec3_metrics.go | 2 +- execution/stagedsync/exec3_parallel.go | 2 +- execution/stagedsync/headerdownload/header_algos.go | 2 +- execution/stagedsync/metrics.go | 2 +- execution/stagedsync/stage_bodies.go | 2 +- execution/stagedsync/stage_execute.go | 2 +- execution/stagedsync/stage_mining_exec.go | 2 +- execution/stagedsync/stageloop/stageloop.go | 2 +- execution/stagedsync/stages/metrics.go | 2 +- node/debug/flags.go | 2 +- node/logging/logging.go | 2 +- node/shards/state_cache.go | 2 +- p2p/discover/metrics.go | 2 +- p2p/metrics.go | 2 +- p2p/peer.go | 2 +- polygon/heimdall/poshttp/http.go | 2 +- polygon/heimdall/poshttp/metrics.go | 2 +- polygon/sync/metrics.go | 2 +- rpc/metrics.go | 2 +- rpc/rpchelper/metrics.go | 2 +- txnprovider/shutter/metrics.go | 2 +- txnprovider/txpool/metrics.go | 2 +- 64 files changed, 49 insertions(+), 49 deletions(-) rename {diagnostics => common}/metrics/block_metrics.go (100%) rename {diagnostics => common}/metrics/config.go (100%) rename {diagnostics => common}/metrics/counter.go (100%) rename {diagnostics => common}/metrics/duration_observer.go (100%) rename {diagnostics => common}/metrics/ema.go (100%) rename {diagnostics => common}/metrics/gauge.go (100%) rename {diagnostics => common}/metrics/gaugevec.go (100%) rename {diagnostics => common}/metrics/histogram.go (100%) rename {diagnostics => common}/metrics/parsing.go (100%) rename {diagnostics => common}/metrics/register.go (100%) rename {diagnostics => common}/metrics/set.go (100%) rename {diagnostics => common}/metrics/setup.go (100%) rename {diagnostics => common}/metrics/summary.go (100%) rename {diagnostics => common}/metrics/timer.go (100%) rename {diagnostics => common}/metrics/value_getter.go (100%) diff --git a/cl/monitor/metrics.go b/cl/monitor/metrics.go index f9050aa47c7..9bc8975544d 100644 --- a/cl/monitor/metrics.go +++ b/cl/monitor/metrics.go @@ -5,7 +5,7 @@ import ( "sync" "time" - "github.com/erigontech/erigon/diagnostics/metrics" + "github.com/erigontech/erigon/common/metrics" ) var ( diff --git a/cl/monitor/shuffling_metrics/shuffling_metrics.go b/cl/monitor/shuffling_metrics/shuffling_metrics.go index 5e8bdd15001..c0ac3999b03 100644 --- a/cl/monitor/shuffling_metrics/shuffling_metrics.go +++ b/cl/monitor/shuffling_metrics/shuffling_metrics.go @@ -3,7 +3,7 @@ package shuffling_metrics import ( "time" - "github.com/erigontech/erigon/diagnostics/metrics" + "github.com/erigontech/erigon/common/metrics" ) var ( diff --git a/cl/phase1/core/state/lru/lru.go b/cl/phase1/core/state/lru/lru.go index 5268476f897..e427a2e0403 100644 --- a/cl/phase1/core/state/lru/lru.go +++ b/cl/phase1/core/state/lru/lru.go @@ -23,7 +23,7 @@ import ( lru "github.com/hashicorp/golang-lru/v2" "github.com/hashicorp/golang-lru/v2/expirable" - "github.com/erigontech/erigon/diagnostics/metrics" + "github.com/erigontech/erigon/common/metrics" ) // Cache is a wrapper around hashicorp lru but with metric for Get diff --git a/cl/phase1/core/state/ssz.go b/cl/phase1/core/state/ssz.go index 7e61baaae39..684c1b47e35 100644 --- a/cl/phase1/core/state/ssz.go +++ b/cl/phase1/core/state/ssz.go @@ -18,7 +18,7 @@ package state import ( "github.com/erigontech/erigon/common/clonable" - "github.com/erigontech/erigon/diagnostics/metrics" + "github.com/erigontech/erigon/common/metrics" ) func (b *CachingBeaconState) EncodeSSZ(buf []byte) ([]byte, error) { diff --git a/cmd/capcli/cli.go b/cmd/capcli/cli.go index 689dc29a921..f94482ea2ae 100644 --- a/cmd/capcli/cli.go +++ b/cmd/capcli/cli.go @@ -64,7 +64,7 @@ import ( "github.com/erigontech/erigon/db/snapshotsync" "github.com/erigontech/erigon/db/snapshotsync/freezeblocks" "github.com/erigontech/erigon/db/snaptype" - "github.com/erigontech/erigon/diagnostics/metrics" + "github.com/erigontech/erigon/common/metrics" "github.com/erigontech/erigon/node/debug" "github.com/erigontech/erigon/node/ethconfig" "github.com/erigontech/erigon/node/gointerfaces/sentinelproto" diff --git a/cmd/erigon/main.go b/cmd/erigon/main.go index dd94b23857e..9dcc7b2ad6a 100644 --- a/cmd/erigon/main.go +++ b/cmd/erigon/main.go @@ -31,7 +31,7 @@ import ( "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/db/datadir" "github.com/erigontech/erigon/db/version" - "github.com/erigontech/erigon/diagnostics/metrics" + "github.com/erigontech/erigon/common/metrics" "github.com/erigontech/erigon/diagnostics/syscheck" erigoncli "github.com/erigontech/erigon/node/cli" "github.com/erigontech/erigon/node/debug" diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 02b550e4d56..15d7972ccec 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -44,7 +44,7 @@ import ( "github.com/erigontech/erigon/common" libkzg "github.com/erigontech/erigon/common/crypto/kzg" "github.com/erigontech/erigon/common/log/v3" - "github.com/erigontech/erigon/diagnostics/metrics" + "github.com/erigontech/erigon/common/metrics" "github.com/erigontech/erigon/db/datadir" "github.com/erigontech/erigon/db/downloader/downloadercfg" "github.com/erigontech/erigon/db/snapcfg" diff --git a/common/disk/disk.go b/common/disk/disk.go index 9d052e887f8..c027edd5a00 100644 --- a/common/disk/disk.go +++ b/common/disk/disk.go @@ -24,7 +24,7 @@ import ( "github.com/shirou/gopsutil/v4/process" - "github.com/erigontech/erigon/diagnostics/metrics" + "github.com/erigontech/erigon/common/metrics" ) var ( diff --git a/common/disk/disk_darwin.go b/common/disk/disk_darwin.go index 959875e2f7c..ddd162b0296 100644 --- a/common/disk/disk_darwin.go +++ b/common/disk/disk_darwin.go @@ -21,7 +21,7 @@ package disk import ( "runtime" - "github.com/erigontech/erigon/diagnostics/metrics" + "github.com/erigontech/erigon/common/metrics" ) var cgoCount = metrics.NewGauge(`go_cgo_calls_count`) diff --git a/common/disk/disk_linux.go b/common/disk/disk_linux.go index f1444290cab..0d3aeefe566 100644 --- a/common/disk/disk_linux.go +++ b/common/disk/disk_linux.go @@ -24,7 +24,7 @@ import ( "github.com/shirou/gopsutil/v4/process" - "github.com/erigontech/erigon/diagnostics/metrics" + "github.com/erigontech/erigon/common/metrics" ) var ( diff --git a/diagnostics/metrics/block_metrics.go b/common/metrics/block_metrics.go similarity index 100% rename from diagnostics/metrics/block_metrics.go rename to common/metrics/block_metrics.go diff --git a/diagnostics/metrics/config.go b/common/metrics/config.go similarity index 100% rename from diagnostics/metrics/config.go rename to common/metrics/config.go diff --git a/diagnostics/metrics/counter.go b/common/metrics/counter.go similarity index 100% rename from diagnostics/metrics/counter.go rename to common/metrics/counter.go diff --git a/diagnostics/metrics/duration_observer.go b/common/metrics/duration_observer.go similarity index 100% rename from diagnostics/metrics/duration_observer.go rename to common/metrics/duration_observer.go diff --git a/diagnostics/metrics/ema.go b/common/metrics/ema.go similarity index 100% rename from diagnostics/metrics/ema.go rename to common/metrics/ema.go diff --git a/diagnostics/metrics/gauge.go b/common/metrics/gauge.go similarity index 100% rename from diagnostics/metrics/gauge.go rename to common/metrics/gauge.go diff --git a/diagnostics/metrics/gaugevec.go b/common/metrics/gaugevec.go similarity index 100% rename from diagnostics/metrics/gaugevec.go rename to common/metrics/gaugevec.go diff --git a/diagnostics/metrics/histogram.go b/common/metrics/histogram.go similarity index 100% rename from diagnostics/metrics/histogram.go rename to common/metrics/histogram.go diff --git a/diagnostics/metrics/parsing.go b/common/metrics/parsing.go similarity index 100% rename from diagnostics/metrics/parsing.go rename to common/metrics/parsing.go diff --git a/diagnostics/metrics/register.go b/common/metrics/register.go similarity index 100% rename from diagnostics/metrics/register.go rename to common/metrics/register.go diff --git a/diagnostics/metrics/set.go b/common/metrics/set.go similarity index 100% rename from diagnostics/metrics/set.go rename to common/metrics/set.go diff --git a/diagnostics/metrics/setup.go b/common/metrics/setup.go similarity index 100% rename from diagnostics/metrics/setup.go rename to common/metrics/setup.go diff --git a/diagnostics/metrics/summary.go b/common/metrics/summary.go similarity index 100% rename from diagnostics/metrics/summary.go rename to common/metrics/summary.go diff --git a/diagnostics/metrics/timer.go b/common/metrics/timer.go similarity index 100% rename from diagnostics/metrics/timer.go rename to common/metrics/timer.go diff --git a/diagnostics/metrics/value_getter.go b/common/metrics/value_getter.go similarity index 100% rename from diagnostics/metrics/value_getter.go rename to common/metrics/value_getter.go diff --git a/db/kv/kv_interface.go b/db/kv/kv_interface.go index cdb8e9aa6d2..e91a61672ff 100644 --- a/db/kv/kv_interface.go +++ b/db/kv/kv_interface.go @@ -30,7 +30,7 @@ import ( "github.com/erigontech/erigon/db/kv/order" "github.com/erigontech/erigon/db/kv/stream" "github.com/erigontech/erigon/db/version" - "github.com/erigontech/erigon/diagnostics/metrics" + "github.com/erigontech/erigon/common/metrics" ) /* diff --git a/db/kv/kvcache/cache.go b/db/kv/kvcache/cache.go index 8be45fe6699..6041faf1dbb 100644 --- a/db/kv/kvcache/cache.go +++ b/db/kv/kvcache/cache.go @@ -33,7 +33,7 @@ import ( "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/db/kv" - "github.com/erigontech/erigon/diagnostics/metrics" + "github.com/erigontech/erigon/common/metrics" "github.com/erigontech/erigon/node/gointerfaces" "github.com/erigontech/erigon/node/gointerfaces/remoteproto" ) diff --git a/db/kv/table_sizes.go b/db/kv/table_sizes.go index f1fb81cf5cc..76b03f5da4b 100644 --- a/db/kv/table_sizes.go +++ b/db/kv/table_sizes.go @@ -9,7 +9,7 @@ import ( "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/common/dbg" "github.com/erigontech/erigon/common/log/v3" - "github.com/erigontech/erigon/diagnostics/metrics" + "github.com/erigontech/erigon/common/metrics" ) var ( diff --git a/db/rawdb/blockio/block_writer.go b/db/rawdb/blockio/block_writer.go index 6631d5a3dae..216387ac308 100644 --- a/db/rawdb/blockio/block_writer.go +++ b/db/rawdb/blockio/block_writer.go @@ -31,7 +31,7 @@ import ( "github.com/erigontech/erigon/db/kv/dbutils" "github.com/erigontech/erigon/db/kv/rawdbv3" "github.com/erigontech/erigon/db/rawdb" - "github.com/erigontech/erigon/diagnostics/metrics" + "github.com/erigontech/erigon/common/metrics" ) //Naming: diff --git a/db/snapshotsync/freezeblocks/block_snapshots.go b/db/snapshotsync/freezeblocks/block_snapshots.go index e8295f1854f..448d8b6302a 100644 --- a/db/snapshotsync/freezeblocks/block_snapshots.go +++ b/db/snapshotsync/freezeblocks/block_snapshots.go @@ -51,7 +51,7 @@ import ( "github.com/erigontech/erigon/db/snapshotsync" "github.com/erigontech/erigon/db/snaptype" "github.com/erigontech/erigon/db/snaptype2" - "github.com/erigontech/erigon/diagnostics/metrics" + "github.com/erigontech/erigon/common/metrics" "github.com/erigontech/erigon/execution/chain" "github.com/erigontech/erigon/execution/rlp" "github.com/erigontech/erigon/execution/stagedsync/stages" diff --git a/db/state/domain.go b/db/state/domain.go index 42c46ceb837..8067f6d8c99 100644 --- a/db/state/domain.go +++ b/db/state/domain.go @@ -51,7 +51,7 @@ import ( "github.com/erigontech/erigon/db/state/changeset" "github.com/erigontech/erigon/db/state/statecfg" "github.com/erigontech/erigon/db/version" - "github.com/erigontech/erigon/diagnostics/metrics" + "github.com/erigontech/erigon/common/metrics" ) var ( diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 614b7b250ac..de48f510588 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -33,7 +33,7 @@ import ( "github.com/erigontech/erigon/db/kv/order" "github.com/erigontech/erigon/db/state/changeset" "github.com/erigontech/erigon/db/state/statecfg" - "github.com/erigontech/erigon/diagnostics/metrics" + "github.com/erigontech/erigon/common/metrics" "github.com/erigontech/erigon/execution/cache" "github.com/erigontech/erigon/execution/commitment" "github.com/erigontech/erigon/execution/commitment/commitmentdb" diff --git a/db/state/inverted_index.go b/db/state/inverted_index.go index 88cdb493a39..568db2d42fc 100644 --- a/db/state/inverted_index.go +++ b/db/state/inverted_index.go @@ -33,7 +33,7 @@ import ( "time" "github.com/erigontech/erigon/db/kv/prune" - "github.com/erigontech/erigon/diagnostics/metrics" + "github.com/erigontech/erigon/common/metrics" "github.com/spaolacci/murmur3" btree2 "github.com/tidwall/btree" "golang.org/x/sync/errgroup" diff --git a/db/state/metrics.go b/db/state/metrics.go index 1b368d034e2..2514d9fbce7 100644 --- a/db/state/metrics.go +++ b/db/state/metrics.go @@ -18,7 +18,7 @@ package state import ( "github.com/erigontech/erigon/db/kv" - "github.com/erigontech/erigon/diagnostics/metrics" + "github.com/erigontech/erigon/common/metrics" ) var ( diff --git a/diagnostics/mem/mem_linux.go b/diagnostics/mem/mem_linux.go index 8fcaa04e707..b2bd5a8f82b 100644 --- a/diagnostics/mem/mem_linux.go +++ b/diagnostics/mem/mem_linux.go @@ -24,7 +24,7 @@ import ( "github.com/shirou/gopsutil/v4/process" - "github.com/erigontech/erigon/diagnostics/metrics" + "github.com/erigontech/erigon/common/metrics" ) var ( diff --git a/execution/commitment/commitment.go b/execution/commitment/commitment.go index d4c54c144b2..6176e997de4 100644 --- a/execution/commitment/commitment.go +++ b/execution/commitment/commitment.go @@ -43,7 +43,7 @@ import ( "github.com/erigontech/erigon/common/maphash" "github.com/erigontech/erigon/db/etl" "github.com/erigontech/erigon/db/kv" - "github.com/erigontech/erigon/diagnostics/metrics" + "github.com/erigontech/erigon/common/metrics" "github.com/erigontech/erigon/execution/types/accounts" ) diff --git a/execution/commitment/commitmentdb/commitment_context.go b/execution/commitment/commitmentdb/commitment_context.go index 95dc869b41f..9d45a198957 100644 --- a/execution/commitment/commitmentdb/commitment_context.go +++ b/execution/commitment/commitmentdb/commitment_context.go @@ -18,7 +18,7 @@ import ( "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/kv/rawdbv3" - "github.com/erigontech/erigon/diagnostics/metrics" + "github.com/erigontech/erigon/common/metrics" "github.com/erigontech/erigon/execution/commitment" "github.com/erigontech/erigon/execution/commitment/trie" witnesstypes "github.com/erigontech/erigon/execution/commitment/witness" diff --git a/execution/exec/state.go b/execution/exec/state.go index 6893a026ce8..e4d7cb2c157 100644 --- a/execution/exec/state.go +++ b/execution/exec/state.go @@ -31,7 +31,7 @@ import ( "github.com/erigontech/erigon/db/datadir" "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/services" - "github.com/erigontech/erigon/diagnostics/metrics" + "github.com/erigontech/erigon/common/metrics" "github.com/erigontech/erigon/execution/chain" "github.com/erigontech/erigon/execution/protocol/rules" "github.com/erigontech/erigon/execution/state" diff --git a/execution/execmodule/forkchoice.go b/execution/execmodule/forkchoice.go index 6708c2d8564..4b1160f7ba7 100644 --- a/execution/execmodule/forkchoice.go +++ b/execution/execmodule/forkchoice.go @@ -28,7 +28,7 @@ import ( "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/common/dbg" "github.com/erigontech/erigon/common/log/v3" - "github.com/erigontech/erigon/diagnostics/metrics" + "github.com/erigontech/erigon/common/metrics" "github.com/erigontech/erigon/db/consensuschain" "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/kv/rawdbv3" diff --git a/execution/execmodule/inserters.go b/execution/execmodule/inserters.go index f6b9b0839ad..aba5d772730 100644 --- a/execution/execmodule/inserters.go +++ b/execution/execmodule/inserters.go @@ -21,7 +21,7 @@ import ( "fmt" "math/big" - "github.com/erigontech/erigon/diagnostics/metrics" + "github.com/erigontech/erigon/common/metrics" "github.com/erigontech/erigon/db/rawdb" "github.com/erigontech/erigon/execution/execmodule/moduleutil" "github.com/erigontech/erigon/execution/types" diff --git a/execution/execmodule/metrics.go b/execution/execmodule/metrics.go index 62cea112f5b..ca475ce981b 100644 --- a/execution/execmodule/metrics.go +++ b/execution/execmodule/metrics.go @@ -19,7 +19,7 @@ package execmodule import ( "time" - "github.com/erigontech/erigon/diagnostics/metrics" + "github.com/erigontech/erigon/common/metrics" ) var ( diff --git a/execution/protocol/block_exec.go b/execution/protocol/block_exec.go index cdff1ca653a..30f11b8d677 100644 --- a/execution/protocol/block_exec.go +++ b/execution/protocol/block_exec.go @@ -33,7 +33,7 @@ import ( "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/common/math" "github.com/erigontech/erigon/common/u256" - "github.com/erigontech/erigon/diagnostics/metrics" + "github.com/erigontech/erigon/common/metrics" "github.com/erigontech/erigon/execution/chain" "github.com/erigontech/erigon/execution/protocol/params" "github.com/erigontech/erigon/execution/protocol/rules" diff --git a/execution/stagedsync/exec3_metrics.go b/execution/stagedsync/exec3_metrics.go index 18f972ea169..a98a6df4821 100644 --- a/execution/stagedsync/exec3_metrics.go +++ b/execution/stagedsync/exec3_metrics.go @@ -12,7 +12,7 @@ import ( "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/state/changeset" - "github.com/erigontech/erigon/diagnostics/metrics" + "github.com/erigontech/erigon/common/metrics" "github.com/erigontech/erigon/execution/commitment" "github.com/erigontech/erigon/execution/protocol/rules" "github.com/erigontech/erigon/execution/state" diff --git a/execution/stagedsync/exec3_parallel.go b/execution/stagedsync/exec3_parallel.go index a7c6d3889b9..bb663453e33 100644 --- a/execution/stagedsync/exec3_parallel.go +++ b/execution/stagedsync/exec3_parallel.go @@ -24,7 +24,7 @@ import ( "github.com/erigontech/erigon/db/kv/temporal" "github.com/erigontech/erigon/db/rawdb/rawdbhelpers" "github.com/erigontech/erigon/db/state/changeset" - "github.com/erigontech/erigon/diagnostics/metrics" + "github.com/erigontech/erigon/common/metrics" "github.com/erigontech/erigon/execution/chain" "github.com/erigontech/erigon/execution/commitment" "github.com/erigontech/erigon/execution/exec" diff --git a/execution/stagedsync/headerdownload/header_algos.go b/execution/stagedsync/headerdownload/header_algos.go index 1087e90c60d..335061a577b 100644 --- a/execution/stagedsync/headerdownload/header_algos.go +++ b/execution/stagedsync/headerdownload/header_algos.go @@ -35,7 +35,7 @@ import ( "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/common/dbg" "github.com/erigontech/erigon/common/log/v3" - "github.com/erigontech/erigon/diagnostics/metrics" + "github.com/erigontech/erigon/common/metrics" "github.com/erigontech/erigon/db/config3" "github.com/erigontech/erigon/db/etl" "github.com/erigontech/erigon/db/kv" diff --git a/execution/stagedsync/metrics.go b/execution/stagedsync/metrics.go index ea47c888d6a..4452c7a1b72 100644 --- a/execution/stagedsync/metrics.go +++ b/execution/stagedsync/metrics.go @@ -1,7 +1,7 @@ package stagedsync import ( - "github.com/erigontech/erigon/diagnostics/metrics" + "github.com/erigontech/erigon/common/metrics" "github.com/erigontech/erigon/execution/stagedsync/stages" ) diff --git a/execution/stagedsync/stage_bodies.go b/execution/stagedsync/stage_bodies.go index 6ed36ce4b0c..1081c106a78 100644 --- a/execution/stagedsync/stage_bodies.go +++ b/execution/stagedsync/stage_bodies.go @@ -25,7 +25,7 @@ import ( "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/common/dbg" "github.com/erigontech/erigon/common/log/v3" - "github.com/erigontech/erigon/diagnostics/metrics" + "github.com/erigontech/erigon/common/metrics" "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/rawdb" "github.com/erigontech/erigon/db/rawdb/blockio" diff --git a/execution/stagedsync/stage_execute.go b/execution/stagedsync/stage_execute.go index 2e55314b533..19b4c517056 100644 --- a/execution/stagedsync/stage_execute.go +++ b/execution/stagedsync/stage_execute.go @@ -42,7 +42,7 @@ import ( "github.com/erigontech/erigon/db/state" "github.com/erigontech/erigon/db/state/changeset" "github.com/erigontech/erigon/db/state/execctx" - "github.com/erigontech/erigon/diagnostics/metrics" + "github.com/erigontech/erigon/common/metrics" "github.com/erigontech/erigon/execution/chain" "github.com/erigontech/erigon/execution/protocol/rules" "github.com/erigontech/erigon/execution/stagedsync/stages" diff --git a/execution/stagedsync/stage_mining_exec.go b/execution/stagedsync/stage_mining_exec.go index d961112bc29..0eb31fffed4 100644 --- a/execution/stagedsync/stage_mining_exec.go +++ b/execution/stagedsync/stage_mining_exec.go @@ -29,7 +29,7 @@ import ( "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/common/dbg" "github.com/erigontech/erigon/common/log/v3" - "github.com/erigontech/erigon/diagnostics/metrics" + "github.com/erigontech/erigon/common/metrics" "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/kv/membatchwithdb" "github.com/erigontech/erigon/db/kv/temporal" diff --git a/execution/stagedsync/stageloop/stageloop.go b/execution/stagedsync/stageloop/stageloop.go index 29448c28cdd..678d33f6656 100644 --- a/execution/stagedsync/stageloop/stageloop.go +++ b/execution/stagedsync/stageloop/stageloop.go @@ -26,7 +26,7 @@ import ( "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/common/dbg" "github.com/erigontech/erigon/common/log/v3" - "github.com/erigontech/erigon/diagnostics/metrics" + "github.com/erigontech/erigon/common/metrics" "github.com/erigontech/erigon/db/datadir" "github.com/erigontech/erigon/db/downloader" "github.com/erigontech/erigon/db/kv" diff --git a/execution/stagedsync/stages/metrics.go b/execution/stagedsync/stages/metrics.go index b0d03fee05e..51b78f05a39 100644 --- a/execution/stagedsync/stages/metrics.go +++ b/execution/stagedsync/stages/metrics.go @@ -22,7 +22,7 @@ import ( "github.com/huandu/xstrings" "github.com/erigontech/erigon/db/kv" - "github.com/erigontech/erigon/diagnostics/metrics" + "github.com/erigontech/erigon/common/metrics" ) var SyncMetrics = map[SyncStage]metrics.Gauge{} diff --git a/node/debug/flags.go b/node/debug/flags.go index 2050dde7914..e0ce954d215 100644 --- a/node/debug/flags.go +++ b/node/debug/flags.go @@ -39,7 +39,7 @@ import ( "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/db/downloader" "github.com/erigontech/erigon/diagnostics/mem" - "github.com/erigontech/erigon/diagnostics/metrics" + "github.com/erigontech/erigon/common/metrics" "github.com/erigontech/erigon/execution/tracing/tracers" "github.com/erigontech/erigon/node/logging" ) diff --git a/node/logging/logging.go b/node/logging/logging.go index f04491070a0..08ed9debdf3 100644 --- a/node/logging/logging.go +++ b/node/logging/logging.go @@ -27,7 +27,7 @@ import ( "gopkg.in/natefinch/lumberjack.v2" "github.com/erigontech/erigon/common/log/v3" - "github.com/erigontech/erigon/diagnostics/metrics" + "github.com/erigontech/erigon/common/metrics" ) // Determine the log dir path based on the given urfave context diff --git a/node/shards/state_cache.go b/node/shards/state_cache.go index c4a3e772072..19ff92bb314 100644 --- a/node/shards/state_cache.go +++ b/node/shards/state_cache.go @@ -27,7 +27,7 @@ import ( "github.com/holiman/uint256" "github.com/erigontech/erigon/common" - "github.com/erigontech/erigon/diagnostics/metrics" + "github.com/erigontech/erigon/common/metrics" "github.com/erigontech/erigon/execution/types/accounts" ) diff --git a/p2p/discover/metrics.go b/p2p/discover/metrics.go index 1accb017801..58fa38ed042 100644 --- a/p2p/discover/metrics.go +++ b/p2p/discover/metrics.go @@ -21,7 +21,7 @@ import ( "net" "net/netip" - "github.com/erigontech/erigon/diagnostics/metrics" + "github.com/erigontech/erigon/common/metrics" ) const ( diff --git a/p2p/metrics.go b/p2p/metrics.go index 9ee2519983f..e78d32ebf2f 100644 --- a/p2p/metrics.go +++ b/p2p/metrics.go @@ -24,7 +24,7 @@ package p2p import ( "net" - "github.com/erigontech/erigon/diagnostics/metrics" + "github.com/erigontech/erigon/common/metrics" ) const ( diff --git a/p2p/peer.go b/p2p/peer.go index ee4f4ab6b48..015d8450aa3 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -33,7 +33,7 @@ import ( "github.com/erigontech/erigon/common/dbg" "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/common/mclock" - "github.com/erigontech/erigon/diagnostics/metrics" + "github.com/erigontech/erigon/common/metrics" "github.com/erigontech/erigon/execution/rlp" "github.com/erigontech/erigon/p2p/enode" "github.com/erigontech/erigon/p2p/enr" diff --git a/polygon/heimdall/poshttp/http.go b/polygon/heimdall/poshttp/http.go index 83c0aae6df6..f50a862bf8a 100644 --- a/polygon/heimdall/poshttp/http.go +++ b/polygon/heimdall/poshttp/http.go @@ -14,7 +14,7 @@ import ( "time" "github.com/erigontech/erigon/common/log/v3" - "github.com/erigontech/erigon/diagnostics/metrics" + "github.com/erigontech/erigon/common/metrics" ) var ( diff --git a/polygon/heimdall/poshttp/metrics.go b/polygon/heimdall/poshttp/metrics.go index c8eb248da27..8815de9bd2a 100644 --- a/polygon/heimdall/poshttp/metrics.go +++ b/polygon/heimdall/poshttp/metrics.go @@ -20,7 +20,7 @@ import ( "context" "time" - "github.com/erigontech/erigon/diagnostics/metrics" + "github.com/erigontech/erigon/common/metrics" ) type ( diff --git a/polygon/sync/metrics.go b/polygon/sync/metrics.go index 22adabf9bab..fb0200e7da6 100644 --- a/polygon/sync/metrics.go +++ b/polygon/sync/metrics.go @@ -19,7 +19,7 @@ package sync import ( "time" - "github.com/erigontech/erigon/diagnostics/metrics" + "github.com/erigontech/erigon/common/metrics" ) var ( diff --git a/rpc/metrics.go b/rpc/metrics.go index 9a43075b750..7e318d90f4f 100644 --- a/rpc/metrics.go +++ b/rpc/metrics.go @@ -24,7 +24,7 @@ import ( "reflect" "strings" - "github.com/erigontech/erigon/diagnostics/metrics" + "github.com/erigontech/erigon/common/metrics" ) var ( diff --git a/rpc/rpchelper/metrics.go b/rpc/rpchelper/metrics.go index a91dd46609a..7294dc331bf 100644 --- a/rpc/rpchelper/metrics.go +++ b/rpc/rpchelper/metrics.go @@ -17,7 +17,7 @@ package rpchelper import ( - "github.com/erigontech/erigon/diagnostics/metrics" + "github.com/erigontech/erigon/common/metrics" ) const ( diff --git a/txnprovider/shutter/metrics.go b/txnprovider/shutter/metrics.go index 3e27491fc76..d1bb6a11254 100644 --- a/txnprovider/shutter/metrics.go +++ b/txnprovider/shutter/metrics.go @@ -1,6 +1,6 @@ package shutter -import "github.com/erigontech/erigon/diagnostics/metrics" +import "github.com/erigontech/erigon/common/metrics" var ( encryptedTxnsPoolAdded = metrics.GetOrCreateCounter("shutter_encrypted_txns_pool_added") diff --git a/txnprovider/txpool/metrics.go b/txnprovider/txpool/metrics.go index e6d950321ae..7d54cd0b29e 100644 --- a/txnprovider/txpool/metrics.go +++ b/txnprovider/txpool/metrics.go @@ -16,7 +16,7 @@ package txpool -import "github.com/erigontech/erigon/diagnostics/metrics" +import "github.com/erigontech/erigon/common/metrics" var ( processBatchTxnsTimer = metrics.NewSummary(`pool_process_remote_txs`) From e910eaffdf91579f5b03ca56d6d66a5dc93e5fa4 Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Tue, 3 Feb 2026 18:28:47 +1100 Subject: [PATCH 08/16] Fix tracer tests --- execution/tracing/tracers/js/tracer_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/execution/tracing/tracers/js/tracer_test.go b/execution/tracing/tracers/js/tracer_test.go index 351f122696e..176b36146e5 100644 --- a/execution/tracing/tracers/js/tracer_test.go +++ b/execution/tracing/tracers/js/tracer_test.go @@ -95,15 +95,15 @@ func TestTracer(t *testing.T) { { // tests that we don't panic on bad arguments to memory access code: "{depths: [], step: function(log) { this.depths.push(log.memory.slice(-1,-2)); }, fault: function() {}, result: function() { return this.depths; }}", want: ``, - fail: "tracer accessed out of bound memory: offset -1, end -2 at step (:1:53(15)) in server-side tracer function 'step'", + fail: "tracer accessed out of bound memory: offset -1, end -2 at step (:1:53(13)) in server-side tracer function 'step'", }, { // tests that we don't panic on bad arguments to stack peeks code: "{depths: [], step: function(log) { this.depths.push(log.stack.peek(-1)); }, fault: function() {}, result: function() { return this.depths; }}", want: ``, - fail: "tracer accessed out of bound stack: size 0, index -1 at step (:1:53(13)) in server-side tracer function 'step'", + fail: "tracer accessed out of bound stack: size 0, index -1 at step (:1:53(11)) in server-side tracer function 'step'", }, { // tests that we don't panic on bad arguments to memory getUint code: "{ depths: [], step: function(log, db) { this.depths.push(log.memory.getUint(-64));}, fault: function() {}, result: function() { return this.depths; }}", want: ``, - fail: "tracer accessed out of bound memory: available 0, offset -64, size 32 at step (:1:58(13)) in server-side tracer function 'step'", + fail: "tracer accessed out of bound memory: available 0, offset -64, size 32 at step (:1:58(11)) in server-side tracer function 'step'", }, { // tests some general counting code: "{count: 0, step: function() { this.count += 1; }, fault: function() {}, result: function() { return this.count; }}", want: `3`, @@ -138,7 +138,7 @@ func TestTracer(t *testing.T) { }, { code: "{res: [], step: function(log) { if (log.op.toString() === 'STOP') { this.res.push(log.memory.slice(5, 1025 * 1024)) } }, fault: function() {}, result: function() { return this.res }}", want: "", - fail: "reached limit for padding memory slice: 1049568 at step (:1:83(23)) in server-side tracer function 'step'", + fail: "reached limit for padding memory slice: 1049568 at step (:1:83(20)) in server-side tracer function 'step'", contract: []byte{byte(vm.PUSH1), byte(0xff), byte(vm.PUSH1), byte(0x00), byte(vm.MSTORE8), byte(vm.STOP)}, }, { // tests ctx.coinbase code: "{lengths: [], step: function(log) { }, fault: function() {}, result: function(ctx) { var coinbase = ctx.coinbase; return toAddress(coinbase); }}", From 8fb9390e439454c6169d4fd112b26489d1d85d9c Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Tue, 3 Feb 2026 18:39:27 +1100 Subject: [PATCH 09/16] Restore errors helper usage --- common/crypto/crypto.go | 3 ++- execution/rlp/decode.go | 40 ++++++++++++++++++++------------------- execution/rlp/raw_test.go | 6 +++--- p2p/enr/enr.go | 8 ++++---- 4 files changed, 30 insertions(+), 27 deletions(-) diff --git a/common/crypto/crypto.go b/common/crypto/crypto.go index adbd7900192..c79b3751d51 100644 --- a/common/crypto/crypto.go +++ b/common/crypto/crypto.go @@ -226,7 +226,8 @@ func FromECDSAPub(pub *ecdsa.PublicKey) []byte { // HexToECDSA parses a secp256k1 private key. func HexToECDSA(hexkey string) (*ecdsa.PrivateKey, error) { b, err := hex.DecodeString(hexkey) - if byteErr, ok := err.(hex.InvalidByteError); ok { + var byteErr hex.InvalidByteError + if errors.As(err, &byteErr) { return nil, fmt.Errorf("invalid hex character %q in private key", byte(byteErr)) } else if err != nil { return nil, errors.New("invalid hex data for private key") diff --git a/execution/rlp/decode.go b/execution/rlp/decode.go index b5b95a0a8e7..dfc345832c5 100644 --- a/execution/rlp/decode.go +++ b/execution/rlp/decode.go @@ -151,20 +151,20 @@ func (err *decodeError) Error() string { } func wrapStreamError(err error, typ reflect.Type) error { - switch err { - case ErrCanonInt: + switch { + case errors.Is(err, ErrCanonInt): return &decodeError{msg: "non-canonical integer (leading zero bytes)", typ: typ} - case ErrCanonSize: + case errors.Is(err, ErrCanonSize): return &decodeError{msg: "non-canonical size information", typ: typ} - case ErrExpectedList: + case errors.Is(err, ErrExpectedList): return &decodeError{msg: "expected input list", typ: typ} - case ErrExpectedString: + case errors.Is(err, ErrExpectedString): return &decodeError{msg: "expected input string or byte", typ: typ} - case errUintOverflow: + case errors.Is(err, errUintOverflow): return &decodeError{msg: "input string too long", typ: typ} - case errUint256Large: + case errors.Is(err, errUint256Large): return &decodeError{msg: "input string too long", typ: typ} - case errNotAtEOL: + case errors.Is(err, errNotAtEOL): return &decodeError{msg: "input list has too many elements", typ: typ} } return err @@ -176,7 +176,8 @@ func WrapStreamError(err error, typ reflect.Type) error { } func addErrorContext(err error, ctx string) error { - if decErr, ok := err.(*decodeError); ok { + var decErr *decodeError + if errors.As(err, &decErr) { decErr.ctx = append(decErr.ctx, ctx) } return err @@ -385,7 +386,7 @@ func decodeSliceElems(s *Stream, val reflect.Value, elemdec decoder) error { val.SetLen(i + 1) } // decode into element - if err := elemdec(s, val.Index(i)); err == EOL { + if err := elemdec(s, val.Index(i)); errors.Is(err, EOL) { break } else if err != nil { return addErrorContext(err, fmt.Sprint("[", i, "]")) @@ -404,7 +405,7 @@ func decodeListArray(s *Stream, val reflect.Value, elemdec decoder) error { vlen := val.Len() i := 0 for ; i < vlen; i++ { - if err := elemdec(s, val.Index(i)); err == EOL { + if err := elemdec(s, val.Index(i)); errors.Is(err, EOL) { break } else if err != nil { return addErrorContext(err, fmt.Sprint("[", i, "]")) @@ -476,7 +477,7 @@ func makeStructDecoder(typ reflect.Type) (decoder, error) { } for i, f := range fields { err := f.info.decoder(s, val.Field(f.index)) - if err == EOL { + if errors.Is(err, EOL) { if f.optional { // The field is optional, so reaching the end of the list before // reaching the last field is acceptable. All remaining undecoded @@ -830,7 +831,7 @@ func (s *Stream) uint(maxbits int) (uint64, error) { } v, err := s.readUint(byte(size)) switch { - case err == ErrCanonSize: + case errors.Is(err, ErrCanonSize): // Adjust error because we're not reading a size right now. return 0, ErrCanonInt case err != nil: @@ -1055,7 +1056,8 @@ func (s *Stream) Decode(val interface{}) error { } err = decoder(s, rval.Elem()) - if decErr, ok := err.(*decodeError); ok && len(decErr.ctx) > 0 { + var decErr *decodeError + if errors.As(err, &decErr) && len(decErr.ctx) > 0 { // Add decode target type to error so context has more meaning. decErr.ctx = append(decErr.ctx, fmt.Sprint("(", rtyp.Elem(), ")")) } @@ -1148,10 +1150,10 @@ func (s *Stream) readKind() (kind Kind, size uint64, err error) { if len(s.stack) == 0 { // At toplevel, Adjust the error to actual EOF. io.EOF is // used by callers to determine when to stop decoding. - switch err { - case io.ErrUnexpectedEOF: + switch { + case errors.Is(err, io.ErrUnexpectedEOF): err = io.EOF - case ErrValueTooLarge: + case errors.Is(err, ErrValueTooLarge): err = io.EOF } } @@ -1236,7 +1238,7 @@ func (s *Stream) readFull(buf []byte) (err error) { nn, err = s.r.Read(buf[n:]) n += nn } - if err == io.EOF { + if errors.Is(err, io.EOF) { if n < len(buf) { err = io.ErrUnexpectedEOF } else { @@ -1254,7 +1256,7 @@ func (s *Stream) readByte() (byte, error) { return 0, err } b, err := s.r.ReadByte() - if err == io.EOF { + if errors.Is(err, io.EOF) { err = io.ErrUnexpectedEOF } return b, err diff --git a/execution/rlp/raw_test.go b/execution/rlp/raw_test.go index 6b6fbe7345e..a70b121a1bb 100644 --- a/execution/rlp/raw_test.go +++ b/execution/rlp/raw_test.go @@ -132,7 +132,7 @@ func TestSplitUint64(t *testing.T) { if !bytes.Equal(rest, unhex(test.rest)) { t.Errorf("test %d: rest mismatch: got %x, want %s (input %q)", i, rest, test.rest, test.input) } - if err != test.err { + if !errors.Is(err, test.err) { t.Errorf("test %d: error mismatch: got %q, want %q", i, err, test.err) } } @@ -216,7 +216,7 @@ func TestSplit(t *testing.T) { if !bytes.Equal(rest, unhex(test.rest)) { t.Errorf("test %d: rest mismatch: got %x, want %s", i, rest, test.rest) } - if err != test.err { + if !errors.Is(err, test.err) { t.Errorf("test %d: error mismatch: got %q, want %q", i, err, test.err) } } @@ -254,7 +254,7 @@ func TestReadSize(t *testing.T) { for _, test := range tests { size, err := readSize(unhex(test.input), test.slen) - if err != test.err { + if !errors.Is(err, test.err) { t.Errorf("readSize(%s, %d): error mismatch: got %q, want %q", test.input, test.slen, err, test.err) continue } diff --git a/p2p/enr/enr.go b/p2p/enr/enr.go index 42e8a22adfb..d18a4c2bbc4 100644 --- a/p2p/enr/enr.go +++ b/p2p/enr/enr.go @@ -231,13 +231,13 @@ func decodeRecord(s *rlp.Stream) (dec Record, raw []byte, err error) { return dec, raw, err } if err = s.Decode(&dec.signature); err != nil { - if err == rlp.EOL { + if errors.Is(err, rlp.EOL) { err = errIncompleteList } return dec, raw, err } if err = s.Decode(&dec.seq); err != nil { - if err == rlp.EOL { + if errors.Is(err, rlp.EOL) { err = errIncompleteList } return dec, raw, err @@ -247,13 +247,13 @@ func decodeRecord(s *rlp.Stream) (dec Record, raw []byte, err error) { for i := 0; ; i++ { var kv pair if err := s.Decode(&kv.k); err != nil { - if err == rlp.EOL { + if errors.Is(err, rlp.EOL) { break } return dec, raw, err } if err := s.Decode(&kv.v); err != nil { - if err == rlp.EOL { + if errors.Is(err, rlp.EOL) { return dec, raw, errIncompletePair } return dec, raw, err From ed853a47ef12ebeda01087f9146e683de67bbb9e Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Tue, 3 Feb 2026 18:52:28 +1100 Subject: [PATCH 10/16] Use mclock from upstream --- common/mclock/alarm.go | 106 -------------- common/mclock/alarm_test.go | 116 --------------- common/mclock/mclock.go | 129 ---------------- common/mclock/mclock.s | 1 - common/mclock/simclock.go | 212 --------------------------- common/mclock/simclock_test.go | 165 --------------------- p2p/config.go | 2 +- p2p/dial.go | 2 +- p2p/dial_test.go | 2 +- p2p/discover/common.go | 2 +- p2p/discover/table.go | 2 +- p2p/discover/table_reval.go | 2 +- p2p/discover/table_reval_test.go | 2 +- p2p/discover/table_test.go | 2 +- p2p/discover/v5_udp.go | 2 +- p2p/discover/v5wire/encoding.go | 2 +- p2p/discover/v5wire/encoding_test.go | 2 +- p2p/discover/v5wire/msg.go | 2 +- p2p/discover/v5wire/session.go | 2 +- p2p/dnsdisc/client.go | 2 +- p2p/dnsdisc/client_test.go | 2 +- p2p/dnsdisc/sync.go | 2 +- p2p/event/subscription.go | 2 +- p2p/netutil/iptrack.go | 2 +- p2p/netutil/iptrack_test.go | 2 +- p2p/peer.go | 2 +- p2p/server.go | 2 +- p2p/util.go | 2 +- p2p/util_test.go | 2 +- 29 files changed, 23 insertions(+), 752 deletions(-) delete mode 100644 common/mclock/alarm.go delete mode 100644 common/mclock/alarm_test.go delete mode 100644 common/mclock/mclock.go delete mode 100644 common/mclock/mclock.s delete mode 100644 common/mclock/simclock.go delete mode 100644 common/mclock/simclock_test.go diff --git a/common/mclock/alarm.go b/common/mclock/alarm.go deleted file mode 100644 index e83810a6a07..00000000000 --- a/common/mclock/alarm.go +++ /dev/null @@ -1,106 +0,0 @@ -// Copyright 2022 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package mclock - -import ( - "time" -) - -// Alarm sends timed notifications on a channel. This is very similar to a regular timer, -// but is easier to use in code that needs to re-schedule the same timer over and over. -// -// When scheduling an Alarm, the channel returned by C() will receive a value no later -// than the scheduled time. An Alarm can be reused after it has fired and can also be -// canceled by calling Stop. -type Alarm struct { - ch chan struct{} - clock Clock - timer Timer - deadline AbsTime -} - -// NewAlarm creates an Alarm. -func NewAlarm(clock Clock) *Alarm { - if clock == nil { - panic("nil clock") - } - return &Alarm{ - ch: make(chan struct{}, 1), - clock: clock, - } -} - -// C returns the alarm notification channel. This channel remains identical for -// the entire lifetime of the alarm, and is never closed. -func (e *Alarm) C() <-chan struct{} { - return e.ch -} - -// Stop cancels the alarm and drains the channel. -// This method is not safe for concurrent use. -func (e *Alarm) Stop() { - // Clear timer. - if e.timer != nil { - e.timer.Stop() - } - e.deadline = 0 - - // Drain the channel. - select { - case <-e.ch: - default: - } -} - -// Schedule sets the alarm to fire no later than the given time. If the alarm was already -// scheduled but has not fired yet, it may fire earlier than the newly-scheduled time. -func (e *Alarm) Schedule(time AbsTime) { - now := e.clock.Now() - e.schedule(now, time) -} - -func (e *Alarm) schedule(now, newDeadline AbsTime) { - if e.timer != nil { - if e.deadline > now && e.deadline <= newDeadline { - // Here, the current timer can be reused because it is already scheduled to - // occur earlier than the new deadline. - // - // The e.deadline > now part of the condition is important. If the old - // deadline lies in the past, we assume the timer has already fired and needs - // to be rescheduled. - return - } - e.timer.Stop() - } - - // Set the timer. - d := time.Duration(0) - if newDeadline < now { - newDeadline = now - } else { - d = newDeadline.Sub(now) - } - e.timer = e.clock.AfterFunc(d, e.send) - e.deadline = newDeadline -} - -func (e *Alarm) send() { - select { - case e.ch <- struct{}{}: - default: - } -} diff --git a/common/mclock/alarm_test.go b/common/mclock/alarm_test.go deleted file mode 100644 index d2ad9913fd2..00000000000 --- a/common/mclock/alarm_test.go +++ /dev/null @@ -1,116 +0,0 @@ -// Copyright 2022 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package mclock - -import "testing" - -// This test checks basic functionality of Alarm. -func TestAlarm(t *testing.T) { - clk := new(Simulated) - clk.Run(20) - a := NewAlarm(clk) - - a.Schedule(clk.Now() + 10) - if recv(a.C()) { - t.Fatal("Alarm fired before scheduled deadline") - } - if ntimers := clk.ActiveTimers(); ntimers != 1 { - t.Fatal("clock has", ntimers, "active timers, want", 1) - } - clk.Run(5) - if recv(a.C()) { - t.Fatal("Alarm fired too early") - } - - clk.Run(5) - if !recv(a.C()) { - t.Fatal("Alarm did not fire") - } - if recv(a.C()) { - t.Fatal("Alarm fired twice") - } - if ntimers := clk.ActiveTimers(); ntimers != 0 { - t.Fatal("clock has", ntimers, "active timers, want", 0) - } - - a.Schedule(clk.Now() + 5) - if recv(a.C()) { - t.Fatal("Alarm fired before scheduled deadline when scheduling the second event") - } - - clk.Run(5) - if !recv(a.C()) { - t.Fatal("Alarm did not fire when scheduling the second event") - } - if recv(a.C()) { - t.Fatal("Alarm fired twice when scheduling the second event") - } -} - -// This test checks that scheduling an Alarm to an earlier time than the -// one already scheduled works properly. -func TestAlarmScheduleEarlier(t *testing.T) { - clk := new(Simulated) - clk.Run(20) - a := NewAlarm(clk) - - a.Schedule(clk.Now() + 50) - clk.Run(5) - a.Schedule(clk.Now() + 1) - clk.Run(3) - if !recv(a.C()) { - t.Fatal("Alarm did not fire") - } -} - -// This test checks that scheduling an Alarm to a later time than the -// one already scheduled works properly. -func TestAlarmScheduleLater(t *testing.T) { - clk := new(Simulated) - clk.Run(20) - a := NewAlarm(clk) - - a.Schedule(clk.Now() + 50) - clk.Run(5) - a.Schedule(clk.Now() + 100) - clk.Run(50) - if !recv(a.C()) { - t.Fatal("Alarm did not fire") - } -} - -// This test checks that scheduling an Alarm in the past makes it fire immediately. -func TestAlarmNegative(t *testing.T) { - clk := new(Simulated) - clk.Run(50) - a := NewAlarm(clk) - - a.Schedule(-1) - clk.Run(1) // needed to process timers - if !recv(a.C()) { - t.Fatal("Alarm did not fire for negative time") - } -} - -func recv(ch <-chan struct{}) bool { - select { - case <-ch: - return true - default: - return false - } -} diff --git a/common/mclock/mclock.go b/common/mclock/mclock.go deleted file mode 100644 index add14bf55b8..00000000000 --- a/common/mclock/mclock.go +++ /dev/null @@ -1,129 +0,0 @@ -// Copyright 2016 The go-ethereum Authors -// (original work) -// Copyright 2024 The Erigon Authors -// (modifications) -// This file is part of Erigon. -// -// Erigon is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// Erigon is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with Erigon. If not, see . - -// Package mclock is a wrapper for a monotonic clock source -package mclock - -import ( - "time" - _ "unsafe" // for go:linkname -) - -//go:noescape -//go:linkname nanotime runtime.nanotime -func nanotime() int64 - -// AbsTime represents absolute monotonic time. -type AbsTime int64 - -// Now returns the current absolute monotonic time. -func Now() AbsTime { - return AbsTime(nanotime()) -} - -// Add returns t + d as absolute time. -func (t AbsTime) Add(d time.Duration) AbsTime { - return t + AbsTime(d) -} - -// Sub returns t - t2 as a duration. -func (t AbsTime) Sub(t2 AbsTime) time.Duration { - return time.Duration(t - t2) -} - -// The Clock interface makes it possible to replace the monotonic system clock with -// a simulated clock. -type Clock interface { - Now() AbsTime - Sleep(time.Duration) - NewTimer(time.Duration) ChanTimer - After(time.Duration) <-chan AbsTime - AfterFunc(d time.Duration, f func()) Timer -} - -// Timer is a cancellable event created by AfterFunc. -type Timer interface { - // Stop cancels the timer. It returns false if the timer has already - // expired or been stopped. - Stop() bool -} - -// ChanTimer is a cancellable event created by NewTimer. -type ChanTimer interface { - Timer - - // The channel returned by C receives a value when the timer expires. - C() <-chan AbsTime - // Reset reschedules the timer with a new timeout. - // It should be invoked only on stopped or expired timers with drained channels. - Reset(time.Duration) -} - -// System implements Clock using the system clock. -type System struct{} - -// Now returns the current monotonic time. -func (c System) Now() AbsTime { - return Now() -} - -// Sleep blocks for the given duration. -func (c System) Sleep(d time.Duration) { - time.Sleep(d) -} - -// NewTimer creates a timer which can be rescheduled. -func (c System) NewTimer(d time.Duration) ChanTimer { - ch := make(chan AbsTime, 1) - t := time.AfterFunc(d, func() { - // This send is non-blocking because that's how time.Timer - // behaves. It doesn't matter in the happy case, but does - // when Reset is misused. - select { - case ch <- c.Now(): - default: - } - }) - return &systemTimer{t, ch} -} - -// After returns a channel which receives the current time after d has elapsed. -func (c System) After(d time.Duration) <-chan AbsTime { - ch := make(chan AbsTime, 1) - time.AfterFunc(d, func() { ch <- c.Now() }) - return ch -} - -// AfterFunc runs f on a new goroutine after the duration has elapsed. -func (c System) AfterFunc(d time.Duration, f func()) Timer { - return time.AfterFunc(d, f) -} - -type systemTimer struct { - *time.Timer - ch <-chan AbsTime -} - -func (st *systemTimer) Reset(d time.Duration) { - st.Timer.Reset(d) -} - -func (st *systemTimer) C() <-chan AbsTime { - return st.ch -} diff --git a/common/mclock/mclock.s b/common/mclock/mclock.s deleted file mode 100644 index 99a7a878f0d..00000000000 --- a/common/mclock/mclock.s +++ /dev/null @@ -1 +0,0 @@ -// This file exists in order to be able to use go:linkname. diff --git a/common/mclock/simclock.go b/common/mclock/simclock.go deleted file mode 100644 index 24ef865216a..00000000000 --- a/common/mclock/simclock.go +++ /dev/null @@ -1,212 +0,0 @@ -// Copyright 2018 The go-ethereum Authors -// (original work) -// Copyright 2024 The Erigon Authors -// (modifications) -// This file is part of Erigon. -// -// Erigon is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// Erigon is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with Erigon. If not, see . - -package mclock - -import ( - "container/heap" - "sync" - "time" -) - -// Simulated implements a virtual Clock for reproducible time-sensitive tests. It -// simulates a scheduler on a virtual timescale where actual processing takes zero time. -// -// The virtual clock doesn't advance on its own, call Run to advance it and execute timers. -// Since there is no way to influence the Go scheduler, testing timeout behaviour involving -// goroutines needs special care. A good way to test such timeouts is as follows: First -// perform the action that is supposed to time out. Ensure that the timer you want to test -// is created. Then run the clock until after the timeout. Finally observe the effect of -// the timeout using a channel or semaphore. -type Simulated struct { - now AbsTime - scheduled simTimerHeap - mu sync.RWMutex - cond *sync.Cond -} - -// simTimer implements ChanTimer on the virtual clock. -type simTimer struct { - at AbsTime - index int // position in s.scheduled - s *Simulated - do func() - ch <-chan AbsTime -} - -func (s *Simulated) init() { - if s.cond == nil { - s.cond = sync.NewCond(&s.mu) - } -} - -// Run moves the clock by the given duration, executing all timers before that duration. -func (s *Simulated) Run(d time.Duration) { - s.mu.Lock() - s.init() - - end := s.now.Add(d) - var do []func() - for len(s.scheduled) > 0 && s.scheduled[0].at <= end { - ev := heap.Pop(&s.scheduled).(*simTimer) - do = append(do, ev.do) - } - s.now = end - s.mu.Unlock() - - for _, fn := range do { - fn() - } -} - -// ActiveTimers returns the number of timers that haven't fired. -func (s *Simulated) ActiveTimers() int { - s.mu.RLock() - defer s.mu.RUnlock() - - return len(s.scheduled) -} - -// WaitForTimers waits until the clock has at least n scheduled timers. -func (s *Simulated) WaitForTimers(n int) { - s.mu.Lock() - defer s.mu.Unlock() - s.init() - - for len(s.scheduled) < n { - s.cond.Wait() - } -} - -// Now returns the current virtual time. -func (s *Simulated) Now() AbsTime { - s.mu.RLock() - defer s.mu.RUnlock() - - return s.now -} - -// Sleep blocks until the clock has advanced by d. -func (s *Simulated) Sleep(d time.Duration) { - <-s.After(d) -} - -// NewTimer creates a timer which fires when the clock has advanced by d. -func (s *Simulated) NewTimer(d time.Duration) ChanTimer { - s.mu.Lock() - defer s.mu.Unlock() - - ch := make(chan AbsTime, 1) - var timer *simTimer - timer = s.schedule(d, func() { ch <- timer.at }) - timer.ch = ch - return timer -} - -// After returns a channel which receives the current time after the clock -// has advanced by d. -func (s *Simulated) After(d time.Duration) <-chan AbsTime { - return s.NewTimer(d).C() -} - -// AfterFunc runs fn after the clock has advanced by d. Unlike with the system -// clock, fn runs on the goroutine that calls Run. -func (s *Simulated) AfterFunc(d time.Duration, fn func()) Timer { - s.mu.Lock() - defer s.mu.Unlock() - - return s.schedule(d, fn) -} - -func (s *Simulated) schedule(d time.Duration, fn func()) *simTimer { - s.init() - - at := s.now.Add(d) - ev := &simTimer{do: fn, at: at, s: s} - heap.Push(&s.scheduled, ev) - s.cond.Broadcast() - return ev -} - -func (ev *simTimer) Stop() bool { - ev.s.mu.Lock() - defer ev.s.mu.Unlock() - - if ev.index < 0 { - return false - } - heap.Remove(&ev.s.scheduled, ev.index) - ev.s.cond.Broadcast() - ev.index = -1 - return true -} - -func (ev *simTimer) Reset(d time.Duration) { - if ev.ch == nil { - panic("mclock: Reset() on timer created by AfterFunc") - } - - ev.s.mu.Lock() - defer ev.s.mu.Unlock() - ev.at = ev.s.now.Add(d) - if ev.index < 0 { - heap.Push(&ev.s.scheduled, ev) // already expired - } else { - heap.Fix(&ev.s.scheduled, ev.index) // hasn't fired yet, reschedule - } - ev.s.cond.Broadcast() -} - -func (ev *simTimer) C() <-chan AbsTime { - if ev.ch == nil { - panic("mclock: C() on timer created by AfterFunc") - } - return ev.ch -} - -type simTimerHeap []*simTimer - -func (h *simTimerHeap) Len() int { - return len(*h) -} - -func (h *simTimerHeap) Less(i, j int) bool { - return (*h)[i].at < (*h)[j].at -} - -func (h *simTimerHeap) Swap(i, j int) { - (*h)[i], (*h)[j] = (*h)[j], (*h)[i] - (*h)[i].index = i - (*h)[j].index = j -} - -func (h *simTimerHeap) Push(x any) { - t := x.(*simTimer) - t.index = len(*h) - *h = append(*h, t) -} - -func (h *simTimerHeap) Pop() any { - end := len(*h) - 1 - t := (*h)[end] - t.index = -1 - (*h)[end] = nil - *h = (*h)[:end] - return t -} diff --git a/common/mclock/simclock_test.go b/common/mclock/simclock_test.go deleted file mode 100644 index 0eb1bd71d82..00000000000 --- a/common/mclock/simclock_test.go +++ /dev/null @@ -1,165 +0,0 @@ -// Copyright 2018 The go-ethereum Authors -// (original work) -// Copyright 2024 The Erigon Authors -// (modifications) -// This file is part of Erigon. -// -// Erigon is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// Erigon is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with Erigon. If not, see . - -package mclock - -import ( - "testing" - "time" -) - -var _ Clock = System{} -var _ Clock = new(Simulated) - -func TestSimulatedAfter(t *testing.T) { - var ( - timeout = 30 * time.Minute - offset = 99 * time.Hour - adv = 11 * time.Minute - c Simulated - ) - c.Run(offset) - - end := c.Now().Add(timeout) - ch := c.After(timeout) - for c.Now() < end.Add(-adv) { - c.Run(adv) - select { - case <-ch: - t.Fatal("Timer fired early") - default: - } - } - - c.Run(adv) - select { - case stamp := <-ch: - want := AbsTime(0).Add(offset).Add(timeout) - if stamp != want { - t.Errorf("Wrong time sent on timer channel: got %v, want %v", stamp, want) - } - default: - t.Fatal("Timer didn't fire") - } -} - -func TestSimulatedAfterFunc(t *testing.T) { - var c Simulated - - called1 := false - timer1 := c.AfterFunc(100*time.Millisecond, func() { called1 = true }) - if c.ActiveTimers() != 1 { - t.Fatalf("%d active timers, want one", c.ActiveTimers()) - } - if fired := timer1.Stop(); !fired { - t.Fatal("Stop returned false even though timer didn't fire") - } - if c.ActiveTimers() != 0 { - t.Fatalf("%d active timers, want zero", c.ActiveTimers()) - } - if called1 { - t.Fatal("timer 1 called") - } - if fired := timer1.Stop(); fired { - t.Fatal("Stop returned true after timer was already stopped") - } - - called2 := false - timer2 := c.AfterFunc(100*time.Millisecond, func() { called2 = true }) - c.Run(50 * time.Millisecond) - if called2 { - t.Fatal("timer 2 called") - } - c.Run(51 * time.Millisecond) - if !called2 { - t.Fatal("timer 2 not called") - } - if fired := timer2.Stop(); fired { - t.Fatal("Stop returned true after timer has fired") - } -} - -func TestSimulatedSleep(t *testing.T) { - var ( - c Simulated - timeout = 1 * time.Hour - done = make(chan AbsTime, 1) - ) - go func() { - c.Sleep(timeout) - done <- c.Now() - }() - - c.WaitForTimers(1) - c.Run(2 * timeout) - select { - case stamp := <-done: - want := AbsTime(2 * timeout) - if stamp != want { - t.Errorf("Wrong time after sleep: got %v, want %v", stamp, want) - } - case <-time.After(5 * time.Second): - t.Fatal("Sleep didn't return in time") - } -} - -func TestSimulatedTimerReset(t *testing.T) { - var ( - c Simulated - timeout = 1 * time.Hour - ) - timer := c.NewTimer(timeout) - c.Run(2 * timeout) - select { - case ftime := <-timer.C(): - if ftime != AbsTime(timeout) { - t.Fatalf("wrong time %v sent on timer channel, want %v", ftime, AbsTime(timeout)) - } - default: - t.Fatal("timer didn't fire") - } - - timer.Reset(timeout) - c.Run(2 * timeout) - select { - case ftime := <-timer.C(): - if ftime != AbsTime(3*timeout) { - t.Fatalf("wrong time %v sent on timer channel, want %v", ftime, AbsTime(3*timeout)) - } - default: - t.Fatal("timer didn't fire again") - } -} - -func TestSimulatedTimerStop(t *testing.T) { - var ( - c Simulated - timeout = 1 * time.Hour - ) - timer := c.NewTimer(timeout) - c.Run(2 * timeout) - if timer.Stop() { - t.Errorf("Stop returned true for fired timer") - } - select { - case <-timer.C(): - default: - t.Fatal("timer didn't fire") - } -} diff --git a/p2p/config.go b/p2p/config.go index a9196ba8df5..472e9f70b6f 100644 --- a/p2p/config.go +++ b/p2p/config.go @@ -5,7 +5,7 @@ import ( "net" "strconv" - "github.com/erigontech/erigon/common/mclock" + "github.com/ethereum/go-ethereum/common/mclock" "github.com/erigontech/erigon/p2p/enode" "github.com/erigontech/erigon/p2p/nat" "github.com/erigontech/erigon/p2p/netutil" diff --git a/p2p/dial.go b/p2p/dial.go index c261d4d3186..d7ccd3c483e 100644 --- a/p2p/dial.go +++ b/p2p/dial.go @@ -33,7 +33,7 @@ import ( "github.com/erigontech/erigon/common/dbg" "github.com/erigontech/erigon/common/log/v3" - "github.com/erigontech/erigon/common/mclock" + "github.com/ethereum/go-ethereum/common/mclock" "github.com/erigontech/erigon/p2p/enode" "github.com/erigontech/erigon/p2p/netutil" ) diff --git a/p2p/dial_test.go b/p2p/dial_test.go index 0726877491a..7d69e25171e 100644 --- a/p2p/dial_test.go +++ b/p2p/dial_test.go @@ -31,7 +31,7 @@ import ( "time" "github.com/erigontech/erigon/common/log/v3" - "github.com/erigontech/erigon/common/mclock" + "github.com/ethereum/go-ethereum/common/mclock" "github.com/erigontech/erigon/common/testlog" "github.com/erigontech/erigon/p2p/enode" "github.com/erigontech/erigon/p2p/netutil" diff --git a/p2p/discover/common.go b/p2p/discover/common.go index c3de9e9fb22..a3fbb6a2826 100644 --- a/p2p/discover/common.go +++ b/p2p/discover/common.go @@ -26,7 +26,7 @@ import ( "sync" "time" - "github.com/erigontech/erigon/common/mclock" + "github.com/ethereum/go-ethereum/common/mclock" "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/p2p/enode" "github.com/erigontech/erigon/p2p/enr" diff --git a/p2p/discover/table.go b/p2p/discover/table.go index 384cf18ca96..ff137c3b4c8 100644 --- a/p2p/discover/table.go +++ b/p2p/discover/table.go @@ -32,7 +32,7 @@ import ( "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/common/log/v3" - "github.com/erigontech/erigon/common/mclock" + "github.com/ethereum/go-ethereum/common/mclock" "github.com/erigontech/erigon/p2p/enode" "github.com/erigontech/erigon/p2p/event" "github.com/erigontech/erigon/p2p/netutil" diff --git a/p2p/discover/table_reval.go b/p2p/discover/table_reval.go index 9cb1254a04c..00b96d74e8a 100644 --- a/p2p/discover/table_reval.go +++ b/p2p/discover/table_reval.go @@ -22,7 +22,7 @@ import ( "slices" "time" - "github.com/erigontech/erigon/common/mclock" + "github.com/ethereum/go-ethereum/common/mclock" "github.com/erigontech/erigon/p2p/enode" ) diff --git a/p2p/discover/table_reval_test.go b/p2p/discover/table_reval_test.go index fb18e19f499..3831bcaba8e 100644 --- a/p2p/discover/table_reval_test.go +++ b/p2p/discover/table_reval_test.go @@ -21,7 +21,7 @@ import ( "testing" "time" - "github.com/erigontech/erigon/common/mclock" + "github.com/ethereum/go-ethereum/common/mclock" "github.com/erigontech/erigon/p2p/enode" "github.com/erigontech/erigon/p2p/enr" ) diff --git a/p2p/discover/table_test.go b/p2p/discover/table_test.go index 2762a68784e..18383dc6cfb 100644 --- a/p2p/discover/table_test.go +++ b/p2p/discover/table_test.go @@ -29,7 +29,7 @@ import ( "github.com/erigontech/erigon/common/crypto" "github.com/erigontech/erigon/common/log/v3" - "github.com/erigontech/erigon/common/mclock" + "github.com/ethereum/go-ethereum/common/mclock" "github.com/erigontech/erigon/common/testlog" "github.com/erigontech/erigon/p2p/enode" "github.com/erigontech/erigon/p2p/enr" diff --git a/p2p/discover/v5_udp.go b/p2p/discover/v5_udp.go index 5943ca0d3d6..c5e154f2c8d 100644 --- a/p2p/discover/v5_udp.go +++ b/p2p/discover/v5_udp.go @@ -30,7 +30,7 @@ import ( "sync" "time" - "github.com/erigontech/erigon/common/mclock" + "github.com/ethereum/go-ethereum/common/mclock" "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/p2p/discover/v5wire" "github.com/erigontech/erigon/p2p/enode" diff --git a/p2p/discover/v5wire/encoding.go b/p2p/discover/v5wire/encoding.go index 03ee4c627ac..a72c9ba736a 100644 --- a/p2p/discover/v5wire/encoding.go +++ b/p2p/discover/v5wire/encoding.go @@ -29,7 +29,7 @@ import ( "hash" "slices" - "github.com/erigontech/erigon/common/mclock" + "github.com/ethereum/go-ethereum/common/mclock" "github.com/erigontech/erigon/p2p/enode" "github.com/erigontech/erigon/p2p/enr" "github.com/erigontech/erigon/execution/rlp" diff --git a/p2p/discover/v5wire/encoding_test.go b/p2p/discover/v5wire/encoding_test.go index 7cf5ae8909e..a1c4844dae7 100644 --- a/p2p/discover/v5wire/encoding_test.go +++ b/p2p/discover/v5wire/encoding_test.go @@ -32,7 +32,7 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/erigontech/erigon/common/hexutil" - "github.com/erigontech/erigon/common/mclock" + "github.com/ethereum/go-ethereum/common/mclock" "github.com/erigontech/erigon/common/crypto" "github.com/erigontech/erigon/p2p/enode" ) diff --git a/p2p/discover/v5wire/msg.go b/p2p/discover/v5wire/msg.go index 3c74cbb4bf4..792ae158e39 100644 --- a/p2p/discover/v5wire/msg.go +++ b/p2p/discover/v5wire/msg.go @@ -21,7 +21,7 @@ import ( "net" "github.com/erigontech/erigon/common/hexutil" - "github.com/erigontech/erigon/common/mclock" + "github.com/ethereum/go-ethereum/common/mclock" "github.com/erigontech/erigon/p2p/enode" "github.com/erigontech/erigon/p2p/enr" "github.com/erigontech/erigon/execution/rlp" diff --git a/p2p/discover/v5wire/session.go b/p2p/discover/v5wire/session.go index 2857128d973..ec50a0493f3 100644 --- a/p2p/discover/v5wire/session.go +++ b/p2p/discover/v5wire/session.go @@ -23,7 +23,7 @@ import ( "time" "github.com/ethereum/go-ethereum/common/lru" - "github.com/erigontech/erigon/common/mclock" + "github.com/ethereum/go-ethereum/common/mclock" "github.com/erigontech/erigon/common/crypto" "github.com/erigontech/erigon/p2p/enode" ) diff --git a/p2p/dnsdisc/client.go b/p2p/dnsdisc/client.go index d0d1c2abe9b..3eeabe0aa68 100644 --- a/p2p/dnsdisc/client.go +++ b/p2p/dnsdisc/client.go @@ -36,7 +36,7 @@ import ( "github.com/erigontech/erigon/common/crypto" "github.com/erigontech/erigon/common/log/v3" - "github.com/erigontech/erigon/common/mclock" + "github.com/ethereum/go-ethereum/common/mclock" "github.com/erigontech/erigon/p2p/enode" "github.com/erigontech/erigon/p2p/enr" ) diff --git a/p2p/dnsdisc/client_test.go b/p2p/dnsdisc/client_test.go index 691381584f1..a2e2c89c923 100644 --- a/p2p/dnsdisc/client_test.go +++ b/p2p/dnsdisc/client_test.go @@ -33,7 +33,7 @@ import ( "github.com/erigontech/erigon/common/crypto" "github.com/erigontech/erigon/common/hexutil" "github.com/erigontech/erigon/common/log/v3" - "github.com/erigontech/erigon/common/mclock" + "github.com/ethereum/go-ethereum/common/mclock" "github.com/erigontech/erigon/common/testlog" "github.com/erigontech/erigon/p2p/enode" "github.com/erigontech/erigon/p2p/enr" diff --git a/p2p/dnsdisc/sync.go b/p2p/dnsdisc/sync.go index 36eb8e7a7a9..c619f15f394 100644 --- a/p2p/dnsdisc/sync.go +++ b/p2p/dnsdisc/sync.go @@ -24,7 +24,7 @@ import ( "math/rand" "time" - "github.com/erigontech/erigon/common/mclock" + "github.com/ethereum/go-ethereum/common/mclock" "github.com/erigontech/erigon/p2p/enode" ) diff --git a/p2p/event/subscription.go b/p2p/event/subscription.go index f0875ddc624..4c03097cf76 100644 --- a/p2p/event/subscription.go +++ b/p2p/event/subscription.go @@ -24,7 +24,7 @@ import ( "sync" "time" - "github.com/erigontech/erigon/common/mclock" + "github.com/ethereum/go-ethereum/common/mclock" ) // Subscription represents a stream of events. The carrier of the events is typically a diff --git a/p2p/netutil/iptrack.go b/p2p/netutil/iptrack.go index 598b7def608..f6433585052 100644 --- a/p2p/netutil/iptrack.go +++ b/p2p/netutil/iptrack.go @@ -23,7 +23,7 @@ import ( "net/netip" "time" - "github.com/erigontech/erigon/common/mclock" + "github.com/ethereum/go-ethereum/common/mclock" ) // IPTracker predicts the external endpoint, i.e. IP address and port, of the local host diff --git a/p2p/netutil/iptrack_test.go b/p2p/netutil/iptrack_test.go index 7d1b98de1bc..596a43aad2c 100644 --- a/p2p/netutil/iptrack_test.go +++ b/p2p/netutil/iptrack_test.go @@ -25,7 +25,7 @@ import ( "testing" "time" - "github.com/erigontech/erigon/common/mclock" + "github.com/ethereum/go-ethereum/common/mclock" ) const ( diff --git a/p2p/peer.go b/p2p/peer.go index 015d8450aa3..5b20afce4a2 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -32,7 +32,7 @@ import ( "github.com/erigontech/erigon/common/dbg" "github.com/erigontech/erigon/common/log/v3" - "github.com/erigontech/erigon/common/mclock" + "github.com/ethereum/go-ethereum/common/mclock" "github.com/erigontech/erigon/common/metrics" "github.com/erigontech/erigon/execution/rlp" "github.com/erigontech/erigon/p2p/enode" diff --git a/p2p/server.go b/p2p/server.go index 2e63f66b61d..250940971b2 100644 --- a/p2p/server.go +++ b/p2p/server.go @@ -41,7 +41,7 @@ import ( "github.com/erigontech/erigon/common/crypto" "github.com/erigontech/erigon/common/dbg" "github.com/erigontech/erigon/common/log/v3" - "github.com/erigontech/erigon/common/mclock" + "github.com/ethereum/go-ethereum/common/mclock" "github.com/erigontech/erigon/p2p/discover" "github.com/erigontech/erigon/p2p/enode" "github.com/erigontech/erigon/p2p/enr" diff --git a/p2p/util.go b/p2p/util.go index b6b1fb5e594..6d67f5d2a43 100644 --- a/p2p/util.go +++ b/p2p/util.go @@ -22,7 +22,7 @@ package p2p import ( "container/heap" - "github.com/erigontech/erigon/common/mclock" + "github.com/ethereum/go-ethereum/common/mclock" ) // expHeap tracks strings and their expiry time. diff --git a/p2p/util_test.go b/p2p/util_test.go index 956aa72d011..30b05f3cdf5 100644 --- a/p2p/util_test.go +++ b/p2p/util_test.go @@ -23,7 +23,7 @@ import ( "testing" "time" - "github.com/erigontech/erigon/common/mclock" + "github.com/ethereum/go-ethereum/common/mclock" ) func TestExpHeap(t *testing.T) { From add2df1ec92b46567baa4746f0b6d06df74ccf97 Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Tue, 3 Feb 2026 22:15:26 +1100 Subject: [PATCH 11/16] Fix licensing --- common/metrics/block_metrics.go | 16 ++++++++++++++++ p2p/discover/common.go | 11 +++++++---- p2p/discover/lookup.go | 11 +++++++---- p2p/discover/node.go | 11 +++++++---- p2p/discover/ntp.go | 11 +++++++---- p2p/discover/table.go | 11 +++++++---- p2p/discover/table_test.go | 11 +++++++---- p2p/discover/table_util_test.go | 11 +++++++---- p2p/discover/v4_lookup_test.go | 11 +++++++---- p2p/discover/v4_udp.go | 11 +++++++---- p2p/discover/v4_udp_test.go | 11 +++++++---- p2p/discover/v4wire/v4wire.go | 11 +++++++---- p2p/discover/v4wire/v4wire_test.go | 11 +++++++---- p2p/discover/v5_udp.go | 11 +++++++---- p2p/discover/v5_udp_test.go | 11 +++++++---- p2p/discover/v5wire/crypto.go | 11 +++++++---- p2p/discover/v5wire/crypto_test.go | 11 +++++++---- p2p/discover/v5wire/encoding.go | 11 +++++++---- p2p/discover/v5wire/encoding_test.go | 11 +++++++---- p2p/discover/v5wire/msg.go | 11 +++++++---- p2p/discover/v5wire/session.go | 11 +++++++---- 21 files changed, 156 insertions(+), 80 deletions(-) diff --git a/common/metrics/block_metrics.go b/common/metrics/block_metrics.go index c32580d1791..a5d2d564ee8 100644 --- a/common/metrics/block_metrics.go +++ b/common/metrics/block_metrics.go @@ -1,3 +1,19 @@ +// Copyright 2024 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with Erigon. If not, see . + package metrics import ( diff --git a/p2p/discover/common.go b/p2p/discover/common.go index a3fbb6a2826..f8149dacbeb 100644 --- a/p2p/discover/common.go +++ b/p2p/discover/common.go @@ -1,18 +1,21 @@ // Copyright 2019 The go-ethereum Authors -// This file is part of the go-ethereum library. +// (original work) +// Copyright 2024 The Erigon Authors +// (modifications) +// This file is part of Erigon. // -// The go-ethereum library is free software: you can redistribute it and/or modify +// Erigon is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // -// The go-ethereum library is distributed in the hope that it will be useful, +// Erigon is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Lesser General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . +// along with Erigon. If not, see . package discover diff --git a/p2p/discover/lookup.go b/p2p/discover/lookup.go index 692f35b0764..30998ab53dd 100644 --- a/p2p/discover/lookup.go +++ b/p2p/discover/lookup.go @@ -1,18 +1,21 @@ // Copyright 2019 The go-ethereum Authors -// This file is part of the go-ethereum library. +// (original work) +// Copyright 2024 The Erigon Authors +// (modifications) +// This file is part of Erigon. // -// The go-ethereum library is free software: you can redistribute it and/or modify +// Erigon is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // -// The go-ethereum library is distributed in the hope that it will be useful, +// Erigon is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Lesser General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . +// along with Erigon. If not, see . package discover diff --git a/p2p/discover/node.go b/p2p/discover/node.go index 7a3f59cf42c..6533e2e94a2 100644 --- a/p2p/discover/node.go +++ b/p2p/discover/node.go @@ -1,18 +1,21 @@ // Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. +// (original work) +// Copyright 2024 The Erigon Authors +// (modifications) +// This file is part of Erigon. // -// The go-ethereum library is free software: you can redistribute it and/or modify +// Erigon is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // -// The go-ethereum library is distributed in the hope that it will be useful, +// Erigon is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Lesser General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . +// along with Erigon. If not, see . package discover diff --git a/p2p/discover/ntp.go b/p2p/discover/ntp.go index d8b4f184688..4e4c85ca992 100644 --- a/p2p/discover/ntp.go +++ b/p2p/discover/ntp.go @@ -1,18 +1,21 @@ // Copyright 2016 The go-ethereum Authors -// This file is part of the go-ethereum library. +// (original work) +// Copyright 2024 The Erigon Authors +// (modifications) +// This file is part of Erigon. // -// The go-ethereum library is free software: you can redistribute it and/or modify +// Erigon is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // -// The go-ethereum library is distributed in the hope that it will be useful, +// Erigon is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Lesser General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . +// along with Erigon. If not, see . // Contains the NTP time drift detection via the SNTP protocol: // https://tools.ietf.org/html/rfc4330 diff --git a/p2p/discover/table.go b/p2p/discover/table.go index ff137c3b4c8..915c5490bc2 100644 --- a/p2p/discover/table.go +++ b/p2p/discover/table.go @@ -1,18 +1,21 @@ // Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. +// (original work) +// Copyright 2024 The Erigon Authors +// (modifications) +// This file is part of Erigon. // -// The go-ethereum library is free software: you can redistribute it and/or modify +// Erigon is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // -// The go-ethereum library is distributed in the hope that it will be useful, +// Erigon is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Lesser General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . +// along with Erigon. If not, see . // Package discover implements the Node Discovery Protocol. // diff --git a/p2p/discover/table_test.go b/p2p/discover/table_test.go index 18383dc6cfb..3419a7ed749 100644 --- a/p2p/discover/table_test.go +++ b/p2p/discover/table_test.go @@ -1,18 +1,21 @@ // Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. +// (original work) +// Copyright 2024 The Erigon Authors +// (modifications) +// This file is part of Erigon. // -// The go-ethereum library is free software: you can redistribute it and/or modify +// Erigon is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // -// The go-ethereum library is distributed in the hope that it will be useful, +// Erigon is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Lesser General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . +// along with Erigon. If not, see . package discover diff --git a/p2p/discover/table_util_test.go b/p2p/discover/table_util_test.go index b9d42303a59..106e15d2e4f 100644 --- a/p2p/discover/table_util_test.go +++ b/p2p/discover/table_util_test.go @@ -1,18 +1,21 @@ // Copyright 2018 The go-ethereum Authors -// This file is part of the go-ethereum library. +// (original work) +// Copyright 2024 The Erigon Authors +// (modifications) +// This file is part of Erigon. // -// The go-ethereum library is free software: you can redistribute it and/or modify +// Erigon is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // -// The go-ethereum library is distributed in the hope that it will be useful, +// Erigon is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Lesser General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . +// along with Erigon. If not, see . package discover diff --git a/p2p/discover/v4_lookup_test.go b/p2p/discover/v4_lookup_test.go index 49e0f07462d..29d1d173020 100644 --- a/p2p/discover/v4_lookup_test.go +++ b/p2p/discover/v4_lookup_test.go @@ -1,18 +1,21 @@ // Copyright 2019 The go-ethereum Authors -// This file is part of the go-ethereum library. +// (original work) +// Copyright 2024 The Erigon Authors +// (modifications) +// This file is part of Erigon. // -// The go-ethereum library is free software: you can redistribute it and/or modify +// Erigon is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // -// The go-ethereum library is distributed in the hope that it will be useful, +// Erigon is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Lesser General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . +// along with Erigon. If not, see . package discover diff --git a/p2p/discover/v4_udp.go b/p2p/discover/v4_udp.go index 3332739dc2d..9c785a87a9b 100644 --- a/p2p/discover/v4_udp.go +++ b/p2p/discover/v4_udp.go @@ -1,18 +1,21 @@ // Copyright 2019 The go-ethereum Authors -// This file is part of the go-ethereum library. +// (original work) +// Copyright 2024 The Erigon Authors +// (modifications) +// This file is part of Erigon. // -// The go-ethereum library is free software: you can redistribute it and/or modify +// Erigon is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // -// The go-ethereum library is distributed in the hope that it will be useful, +// Erigon is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Lesser General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . +// along with Erigon. If not, see . package discover diff --git a/p2p/discover/v4_udp_test.go b/p2p/discover/v4_udp_test.go index 57143defc17..7a95ce356d0 100644 --- a/p2p/discover/v4_udp_test.go +++ b/p2p/discover/v4_udp_test.go @@ -1,18 +1,21 @@ // Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. +// (original work) +// Copyright 2024 The Erigon Authors +// (modifications) +// This file is part of Erigon. // -// The go-ethereum library is free software: you can redistribute it and/or modify +// Erigon is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // -// The go-ethereum library is distributed in the hope that it will be useful, +// Erigon is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Lesser General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . +// along with Erigon. If not, see . package discover diff --git a/p2p/discover/v4wire/v4wire.go b/p2p/discover/v4wire/v4wire.go index ecff4d1909d..baad4fae555 100644 --- a/p2p/discover/v4wire/v4wire.go +++ b/p2p/discover/v4wire/v4wire.go @@ -1,18 +1,21 @@ // Copyright 2020 The go-ethereum Authors -// This file is part of the go-ethereum library. +// (original work) +// Copyright 2024 The Erigon Authors +// (modifications) +// This file is part of Erigon. // -// The go-ethereum library is free software: you can redistribute it and/or modify +// Erigon is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // -// The go-ethereum library is distributed in the hope that it will be useful, +// Erigon is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Lesser General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . +// along with Erigon. If not, see . // Package v4wire implements the Discovery v4 Wire Protocol. package v4wire diff --git a/p2p/discover/v4wire/v4wire_test.go b/p2p/discover/v4wire/v4wire_test.go index 5d1119b8419..7adbed5dfed 100644 --- a/p2p/discover/v4wire/v4wire_test.go +++ b/p2p/discover/v4wire/v4wire_test.go @@ -1,18 +1,21 @@ // Copyright 2020 The go-ethereum Authors -// This file is part of the go-ethereum library. +// (original work) +// Copyright 2024 The Erigon Authors +// (modifications) +// This file is part of Erigon. // -// The go-ethereum library is free software: you can redistribute it and/or modify +// Erigon is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // -// The go-ethereum library is distributed in the hope that it will be useful, +// Erigon is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Lesser General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . +// along with Erigon. If not, see . package v4wire diff --git a/p2p/discover/v5_udp.go b/p2p/discover/v5_udp.go index c5e154f2c8d..ca38cc538f9 100644 --- a/p2p/discover/v5_udp.go +++ b/p2p/discover/v5_udp.go @@ -1,18 +1,21 @@ // Copyright 2020 The go-ethereum Authors -// This file is part of the go-ethereum library. +// (original work) +// Copyright 2024 The Erigon Authors +// (modifications) +// This file is part of Erigon. // -// The go-ethereum library is free software: you can redistribute it and/or modify +// Erigon is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // -// The go-ethereum library is distributed in the hope that it will be useful, +// Erigon is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Lesser General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . +// along with Erigon. If not, see . package discover diff --git a/p2p/discover/v5_udp_test.go b/p2p/discover/v5_udp_test.go index 8fea29df6dc..64934704a4d 100644 --- a/p2p/discover/v5_udp_test.go +++ b/p2p/discover/v5_udp_test.go @@ -1,18 +1,21 @@ // Copyright 2020 The go-ethereum Authors -// This file is part of the go-ethereum library. +// (original work) +// Copyright 2024 The Erigon Authors +// (modifications) +// This file is part of Erigon. // -// The go-ethereum library is free software: you can redistribute it and/or modify +// Erigon is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // -// The go-ethereum library is distributed in the hope that it will be useful, +// Erigon is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Lesser General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . +// along with Erigon. If not, see . package discover diff --git a/p2p/discover/v5wire/crypto.go b/p2p/discover/v5wire/crypto.go index d946207a0f9..e651946c46f 100644 --- a/p2p/discover/v5wire/crypto.go +++ b/p2p/discover/v5wire/crypto.go @@ -1,18 +1,21 @@ // Copyright 2020 The go-ethereum Authors -// This file is part of the go-ethereum library. +// (original work) +// Copyright 2024 The Erigon Authors +// (modifications) +// This file is part of Erigon. // -// The go-ethereum library is free software: you can redistribute it and/or modify +// Erigon is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // -// The go-ethereum library is distributed in the hope that it will be useful, +// Erigon is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Lesser General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . +// along with Erigon. If not, see . package v5wire diff --git a/p2p/discover/v5wire/crypto_test.go b/p2p/discover/v5wire/crypto_test.go index 771112db1fb..8a673245da3 100644 --- a/p2p/discover/v5wire/crypto_test.go +++ b/p2p/discover/v5wire/crypto_test.go @@ -1,18 +1,21 @@ // Copyright 2020 The go-ethereum Authors -// This file is part of the go-ethereum library. +// (original work) +// Copyright 2024 The Erigon Authors +// (modifications) +// This file is part of Erigon. // -// The go-ethereum library is free software: you can redistribute it and/or modify +// Erigon is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // -// The go-ethereum library is distributed in the hope that it will be useful, +// Erigon is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Lesser General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . +// along with Erigon. If not, see . package v5wire diff --git a/p2p/discover/v5wire/encoding.go b/p2p/discover/v5wire/encoding.go index a72c9ba736a..87334a9f1d0 100644 --- a/p2p/discover/v5wire/encoding.go +++ b/p2p/discover/v5wire/encoding.go @@ -1,18 +1,21 @@ // Copyright 2020 The go-ethereum Authors -// This file is part of the go-ethereum library. +// (original work) +// Copyright 2024 The Erigon Authors +// (modifications) +// This file is part of Erigon. // -// The go-ethereum library is free software: you can redistribute it and/or modify +// Erigon is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // -// The go-ethereum library is distributed in the hope that it will be useful, +// Erigon is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Lesser General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . +// along with Erigon. If not, see . package v5wire diff --git a/p2p/discover/v5wire/encoding_test.go b/p2p/discover/v5wire/encoding_test.go index a1c4844dae7..b58f3f12af6 100644 --- a/p2p/discover/v5wire/encoding_test.go +++ b/p2p/discover/v5wire/encoding_test.go @@ -1,18 +1,21 @@ // Copyright 2020 The go-ethereum Authors -// This file is part of the go-ethereum library. +// (original work) +// Copyright 2024 The Erigon Authors +// (modifications) +// This file is part of Erigon. // -// The go-ethereum library is free software: you can redistribute it and/or modify +// Erigon is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // -// The go-ethereum library is distributed in the hope that it will be useful, +// Erigon is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Lesser General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . +// along with Erigon. If not, see . package v5wire diff --git a/p2p/discover/v5wire/msg.go b/p2p/discover/v5wire/msg.go index 792ae158e39..d7948cff153 100644 --- a/p2p/discover/v5wire/msg.go +++ b/p2p/discover/v5wire/msg.go @@ -1,18 +1,21 @@ // Copyright 2020 The go-ethereum Authors -// This file is part of the go-ethereum library. +// (original work) +// Copyright 2024 The Erigon Authors +// (modifications) +// This file is part of Erigon. // -// The go-ethereum library is free software: you can redistribute it and/or modify +// Erigon is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // -// The go-ethereum library is distributed in the hope that it will be useful, +// Erigon is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Lesser General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . +// along with Erigon. If not, see . package v5wire diff --git a/p2p/discover/v5wire/session.go b/p2p/discover/v5wire/session.go index ec50a0493f3..c01461ddb75 100644 --- a/p2p/discover/v5wire/session.go +++ b/p2p/discover/v5wire/session.go @@ -1,18 +1,21 @@ // Copyright 2020 The go-ethereum Authors -// This file is part of the go-ethereum library. +// (original work) +// Copyright 2024 The Erigon Authors +// (modifications) +// This file is part of Erigon. // -// The go-ethereum library is free software: you can redistribute it and/or modify +// Erigon is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // -// The go-ethereum library is distributed in the hope that it will be useful, +// Erigon is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Lesser General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . +// along with Erigon. If not, see . package v5wire From 5cc3c3f04e913bdaacf854ad5b694394e725bbdd Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Wed, 4 Feb 2026 11:24:33 +1100 Subject: [PATCH 12/16] Move back to diagnostics/metrics to keep diff smaller --- cl/monitor/metrics.go | 2 +- cl/monitor/shuffling_metrics/shuffling_metrics.go | 2 +- cl/phase1/core/state/lru/lru.go | 2 +- cl/phase1/core/state/ssz.go | 2 +- cmd/capcli/cli.go | 2 +- cmd/erigon/main.go | 2 +- cmd/utils/flags.go | 2 +- common/disk/disk.go | 2 +- common/disk/disk_darwin.go | 2 +- common/disk/disk_linux.go | 2 +- db/kv/kv_interface.go | 2 +- db/kv/kvcache/cache.go | 2 +- db/kv/table_sizes.go | 2 +- db/rawdb/blockio/block_writer.go | 2 +- db/snapshotsync/freezeblocks/block_snapshots.go | 2 +- db/state/domain.go | 2 +- db/state/execctx/domain_shared.go | 2 +- db/state/inverted_index.go | 2 +- db/state/metrics.go | 2 +- diagnostics/mem/mem_linux.go | 2 +- {common => diagnostics}/metrics/block_metrics.go | 0 {common => diagnostics}/metrics/config.go | 0 {common => diagnostics}/metrics/counter.go | 0 {common => diagnostics}/metrics/duration_observer.go | 0 {common => diagnostics}/metrics/ema.go | 0 {common => diagnostics}/metrics/gauge.go | 0 {common => diagnostics}/metrics/gaugevec.go | 0 {common => diagnostics}/metrics/histogram.go | 0 {common => diagnostics}/metrics/parsing.go | 0 {common => diagnostics}/metrics/register.go | 0 {common => diagnostics}/metrics/set.go | 0 {common => diagnostics}/metrics/setup.go | 0 {common => diagnostics}/metrics/summary.go | 0 {common => diagnostics}/metrics/timer.go | 0 {common => diagnostics}/metrics/value_getter.go | 0 execution/commitment/commitment.go | 2 +- execution/commitment/commitmentdb/commitment_context.go | 2 +- execution/exec/state.go | 2 +- execution/execmodule/forkchoice.go | 2 +- execution/execmodule/inserters.go | 2 +- execution/execmodule/metrics.go | 2 +- execution/protocol/block_exec.go | 2 +- execution/stagedsync/exec3_metrics.go | 2 +- execution/stagedsync/exec3_parallel.go | 2 +- execution/stagedsync/headerdownload/header_algos.go | 2 +- execution/stagedsync/metrics.go | 2 +- execution/stagedsync/stage_bodies.go | 2 +- execution/stagedsync/stage_execute.go | 2 +- execution/stagedsync/stage_mining_exec.go | 2 +- execution/stagedsync/stageloop/stageloop.go | 2 +- execution/stagedsync/stages/metrics.go | 2 +- node/debug/flags.go | 2 +- node/logging/logging.go | 2 +- node/shards/state_cache.go | 2 +- p2p/discover/metrics.go | 2 +- p2p/metrics.go | 2 +- p2p/peer.go | 2 +- polygon/heimdall/poshttp/http.go | 2 +- polygon/heimdall/poshttp/metrics.go | 2 +- polygon/sync/metrics.go | 2 +- rpc/metrics.go | 2 +- rpc/rpchelper/metrics.go | 2 +- txnprovider/shutter/metrics.go | 2 +- txnprovider/txpool/metrics.go | 2 +- 64 files changed, 49 insertions(+), 49 deletions(-) rename {common => diagnostics}/metrics/block_metrics.go (100%) rename {common => diagnostics}/metrics/config.go (100%) rename {common => diagnostics}/metrics/counter.go (100%) rename {common => diagnostics}/metrics/duration_observer.go (100%) rename {common => diagnostics}/metrics/ema.go (100%) rename {common => diagnostics}/metrics/gauge.go (100%) rename {common => diagnostics}/metrics/gaugevec.go (100%) rename {common => diagnostics}/metrics/histogram.go (100%) rename {common => diagnostics}/metrics/parsing.go (100%) rename {common => diagnostics}/metrics/register.go (100%) rename {common => diagnostics}/metrics/set.go (100%) rename {common => diagnostics}/metrics/setup.go (100%) rename {common => diagnostics}/metrics/summary.go (100%) rename {common => diagnostics}/metrics/timer.go (100%) rename {common => diagnostics}/metrics/value_getter.go (100%) diff --git a/cl/monitor/metrics.go b/cl/monitor/metrics.go index 9bc8975544d..f9050aa47c7 100644 --- a/cl/monitor/metrics.go +++ b/cl/monitor/metrics.go @@ -5,7 +5,7 @@ import ( "sync" "time" - "github.com/erigontech/erigon/common/metrics" + "github.com/erigontech/erigon/diagnostics/metrics" ) var ( diff --git a/cl/monitor/shuffling_metrics/shuffling_metrics.go b/cl/monitor/shuffling_metrics/shuffling_metrics.go index c0ac3999b03..5e8bdd15001 100644 --- a/cl/monitor/shuffling_metrics/shuffling_metrics.go +++ b/cl/monitor/shuffling_metrics/shuffling_metrics.go @@ -3,7 +3,7 @@ package shuffling_metrics import ( "time" - "github.com/erigontech/erigon/common/metrics" + "github.com/erigontech/erigon/diagnostics/metrics" ) var ( diff --git a/cl/phase1/core/state/lru/lru.go b/cl/phase1/core/state/lru/lru.go index e427a2e0403..5268476f897 100644 --- a/cl/phase1/core/state/lru/lru.go +++ b/cl/phase1/core/state/lru/lru.go @@ -23,7 +23,7 @@ import ( lru "github.com/hashicorp/golang-lru/v2" "github.com/hashicorp/golang-lru/v2/expirable" - "github.com/erigontech/erigon/common/metrics" + "github.com/erigontech/erigon/diagnostics/metrics" ) // Cache is a wrapper around hashicorp lru but with metric for Get diff --git a/cl/phase1/core/state/ssz.go b/cl/phase1/core/state/ssz.go index 684c1b47e35..7e61baaae39 100644 --- a/cl/phase1/core/state/ssz.go +++ b/cl/phase1/core/state/ssz.go @@ -18,7 +18,7 @@ package state import ( "github.com/erigontech/erigon/common/clonable" - "github.com/erigontech/erigon/common/metrics" + "github.com/erigontech/erigon/diagnostics/metrics" ) func (b *CachingBeaconState) EncodeSSZ(buf []byte) ([]byte, error) { diff --git a/cmd/capcli/cli.go b/cmd/capcli/cli.go index f94482ea2ae..689dc29a921 100644 --- a/cmd/capcli/cli.go +++ b/cmd/capcli/cli.go @@ -64,7 +64,7 @@ import ( "github.com/erigontech/erigon/db/snapshotsync" "github.com/erigontech/erigon/db/snapshotsync/freezeblocks" "github.com/erigontech/erigon/db/snaptype" - "github.com/erigontech/erigon/common/metrics" + "github.com/erigontech/erigon/diagnostics/metrics" "github.com/erigontech/erigon/node/debug" "github.com/erigontech/erigon/node/ethconfig" "github.com/erigontech/erigon/node/gointerfaces/sentinelproto" diff --git a/cmd/erigon/main.go b/cmd/erigon/main.go index 9dcc7b2ad6a..dd94b23857e 100644 --- a/cmd/erigon/main.go +++ b/cmd/erigon/main.go @@ -31,7 +31,7 @@ import ( "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/db/datadir" "github.com/erigontech/erigon/db/version" - "github.com/erigontech/erigon/common/metrics" + "github.com/erigontech/erigon/diagnostics/metrics" "github.com/erigontech/erigon/diagnostics/syscheck" erigoncli "github.com/erigontech/erigon/node/cli" "github.com/erigontech/erigon/node/debug" diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 15d7972ccec..02b550e4d56 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -44,7 +44,7 @@ import ( "github.com/erigontech/erigon/common" libkzg "github.com/erigontech/erigon/common/crypto/kzg" "github.com/erigontech/erigon/common/log/v3" - "github.com/erigontech/erigon/common/metrics" + "github.com/erigontech/erigon/diagnostics/metrics" "github.com/erigontech/erigon/db/datadir" "github.com/erigontech/erigon/db/downloader/downloadercfg" "github.com/erigontech/erigon/db/snapcfg" diff --git a/common/disk/disk.go b/common/disk/disk.go index c027edd5a00..9d052e887f8 100644 --- a/common/disk/disk.go +++ b/common/disk/disk.go @@ -24,7 +24,7 @@ import ( "github.com/shirou/gopsutil/v4/process" - "github.com/erigontech/erigon/common/metrics" + "github.com/erigontech/erigon/diagnostics/metrics" ) var ( diff --git a/common/disk/disk_darwin.go b/common/disk/disk_darwin.go index ddd162b0296..959875e2f7c 100644 --- a/common/disk/disk_darwin.go +++ b/common/disk/disk_darwin.go @@ -21,7 +21,7 @@ package disk import ( "runtime" - "github.com/erigontech/erigon/common/metrics" + "github.com/erigontech/erigon/diagnostics/metrics" ) var cgoCount = metrics.NewGauge(`go_cgo_calls_count`) diff --git a/common/disk/disk_linux.go b/common/disk/disk_linux.go index 0d3aeefe566..f1444290cab 100644 --- a/common/disk/disk_linux.go +++ b/common/disk/disk_linux.go @@ -24,7 +24,7 @@ import ( "github.com/shirou/gopsutil/v4/process" - "github.com/erigontech/erigon/common/metrics" + "github.com/erigontech/erigon/diagnostics/metrics" ) var ( diff --git a/db/kv/kv_interface.go b/db/kv/kv_interface.go index e91a61672ff..cdb8e9aa6d2 100644 --- a/db/kv/kv_interface.go +++ b/db/kv/kv_interface.go @@ -30,7 +30,7 @@ import ( "github.com/erigontech/erigon/db/kv/order" "github.com/erigontech/erigon/db/kv/stream" "github.com/erigontech/erigon/db/version" - "github.com/erigontech/erigon/common/metrics" + "github.com/erigontech/erigon/diagnostics/metrics" ) /* diff --git a/db/kv/kvcache/cache.go b/db/kv/kvcache/cache.go index 6041faf1dbb..8be45fe6699 100644 --- a/db/kv/kvcache/cache.go +++ b/db/kv/kvcache/cache.go @@ -33,7 +33,7 @@ import ( "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/db/kv" - "github.com/erigontech/erigon/common/metrics" + "github.com/erigontech/erigon/diagnostics/metrics" "github.com/erigontech/erigon/node/gointerfaces" "github.com/erigontech/erigon/node/gointerfaces/remoteproto" ) diff --git a/db/kv/table_sizes.go b/db/kv/table_sizes.go index 76b03f5da4b..f1fb81cf5cc 100644 --- a/db/kv/table_sizes.go +++ b/db/kv/table_sizes.go @@ -9,7 +9,7 @@ import ( "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/common/dbg" "github.com/erigontech/erigon/common/log/v3" - "github.com/erigontech/erigon/common/metrics" + "github.com/erigontech/erigon/diagnostics/metrics" ) var ( diff --git a/db/rawdb/blockio/block_writer.go b/db/rawdb/blockio/block_writer.go index 216387ac308..6631d5a3dae 100644 --- a/db/rawdb/blockio/block_writer.go +++ b/db/rawdb/blockio/block_writer.go @@ -31,7 +31,7 @@ import ( "github.com/erigontech/erigon/db/kv/dbutils" "github.com/erigontech/erigon/db/kv/rawdbv3" "github.com/erigontech/erigon/db/rawdb" - "github.com/erigontech/erigon/common/metrics" + "github.com/erigontech/erigon/diagnostics/metrics" ) //Naming: diff --git a/db/snapshotsync/freezeblocks/block_snapshots.go b/db/snapshotsync/freezeblocks/block_snapshots.go index 448d8b6302a..e8295f1854f 100644 --- a/db/snapshotsync/freezeblocks/block_snapshots.go +++ b/db/snapshotsync/freezeblocks/block_snapshots.go @@ -51,7 +51,7 @@ import ( "github.com/erigontech/erigon/db/snapshotsync" "github.com/erigontech/erigon/db/snaptype" "github.com/erigontech/erigon/db/snaptype2" - "github.com/erigontech/erigon/common/metrics" + "github.com/erigontech/erigon/diagnostics/metrics" "github.com/erigontech/erigon/execution/chain" "github.com/erigontech/erigon/execution/rlp" "github.com/erigontech/erigon/execution/stagedsync/stages" diff --git a/db/state/domain.go b/db/state/domain.go index 8067f6d8c99..42c46ceb837 100644 --- a/db/state/domain.go +++ b/db/state/domain.go @@ -51,7 +51,7 @@ import ( "github.com/erigontech/erigon/db/state/changeset" "github.com/erigontech/erigon/db/state/statecfg" "github.com/erigontech/erigon/db/version" - "github.com/erigontech/erigon/common/metrics" + "github.com/erigontech/erigon/diagnostics/metrics" ) var ( diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index de48f510588..614b7b250ac 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -33,7 +33,7 @@ import ( "github.com/erigontech/erigon/db/kv/order" "github.com/erigontech/erigon/db/state/changeset" "github.com/erigontech/erigon/db/state/statecfg" - "github.com/erigontech/erigon/common/metrics" + "github.com/erigontech/erigon/diagnostics/metrics" "github.com/erigontech/erigon/execution/cache" "github.com/erigontech/erigon/execution/commitment" "github.com/erigontech/erigon/execution/commitment/commitmentdb" diff --git a/db/state/inverted_index.go b/db/state/inverted_index.go index 568db2d42fc..88cdb493a39 100644 --- a/db/state/inverted_index.go +++ b/db/state/inverted_index.go @@ -33,7 +33,7 @@ import ( "time" "github.com/erigontech/erigon/db/kv/prune" - "github.com/erigontech/erigon/common/metrics" + "github.com/erigontech/erigon/diagnostics/metrics" "github.com/spaolacci/murmur3" btree2 "github.com/tidwall/btree" "golang.org/x/sync/errgroup" diff --git a/db/state/metrics.go b/db/state/metrics.go index 2514d9fbce7..1b368d034e2 100644 --- a/db/state/metrics.go +++ b/db/state/metrics.go @@ -18,7 +18,7 @@ package state import ( "github.com/erigontech/erigon/db/kv" - "github.com/erigontech/erigon/common/metrics" + "github.com/erigontech/erigon/diagnostics/metrics" ) var ( diff --git a/diagnostics/mem/mem_linux.go b/diagnostics/mem/mem_linux.go index b2bd5a8f82b..8fcaa04e707 100644 --- a/diagnostics/mem/mem_linux.go +++ b/diagnostics/mem/mem_linux.go @@ -24,7 +24,7 @@ import ( "github.com/shirou/gopsutil/v4/process" - "github.com/erigontech/erigon/common/metrics" + "github.com/erigontech/erigon/diagnostics/metrics" ) var ( diff --git a/common/metrics/block_metrics.go b/diagnostics/metrics/block_metrics.go similarity index 100% rename from common/metrics/block_metrics.go rename to diagnostics/metrics/block_metrics.go diff --git a/common/metrics/config.go b/diagnostics/metrics/config.go similarity index 100% rename from common/metrics/config.go rename to diagnostics/metrics/config.go diff --git a/common/metrics/counter.go b/diagnostics/metrics/counter.go similarity index 100% rename from common/metrics/counter.go rename to diagnostics/metrics/counter.go diff --git a/common/metrics/duration_observer.go b/diagnostics/metrics/duration_observer.go similarity index 100% rename from common/metrics/duration_observer.go rename to diagnostics/metrics/duration_observer.go diff --git a/common/metrics/ema.go b/diagnostics/metrics/ema.go similarity index 100% rename from common/metrics/ema.go rename to diagnostics/metrics/ema.go diff --git a/common/metrics/gauge.go b/diagnostics/metrics/gauge.go similarity index 100% rename from common/metrics/gauge.go rename to diagnostics/metrics/gauge.go diff --git a/common/metrics/gaugevec.go b/diagnostics/metrics/gaugevec.go similarity index 100% rename from common/metrics/gaugevec.go rename to diagnostics/metrics/gaugevec.go diff --git a/common/metrics/histogram.go b/diagnostics/metrics/histogram.go similarity index 100% rename from common/metrics/histogram.go rename to diagnostics/metrics/histogram.go diff --git a/common/metrics/parsing.go b/diagnostics/metrics/parsing.go similarity index 100% rename from common/metrics/parsing.go rename to diagnostics/metrics/parsing.go diff --git a/common/metrics/register.go b/diagnostics/metrics/register.go similarity index 100% rename from common/metrics/register.go rename to diagnostics/metrics/register.go diff --git a/common/metrics/set.go b/diagnostics/metrics/set.go similarity index 100% rename from common/metrics/set.go rename to diagnostics/metrics/set.go diff --git a/common/metrics/setup.go b/diagnostics/metrics/setup.go similarity index 100% rename from common/metrics/setup.go rename to diagnostics/metrics/setup.go diff --git a/common/metrics/summary.go b/diagnostics/metrics/summary.go similarity index 100% rename from common/metrics/summary.go rename to diagnostics/metrics/summary.go diff --git a/common/metrics/timer.go b/diagnostics/metrics/timer.go similarity index 100% rename from common/metrics/timer.go rename to diagnostics/metrics/timer.go diff --git a/common/metrics/value_getter.go b/diagnostics/metrics/value_getter.go similarity index 100% rename from common/metrics/value_getter.go rename to diagnostics/metrics/value_getter.go diff --git a/execution/commitment/commitment.go b/execution/commitment/commitment.go index 6176e997de4..d4c54c144b2 100644 --- a/execution/commitment/commitment.go +++ b/execution/commitment/commitment.go @@ -43,7 +43,7 @@ import ( "github.com/erigontech/erigon/common/maphash" "github.com/erigontech/erigon/db/etl" "github.com/erigontech/erigon/db/kv" - "github.com/erigontech/erigon/common/metrics" + "github.com/erigontech/erigon/diagnostics/metrics" "github.com/erigontech/erigon/execution/types/accounts" ) diff --git a/execution/commitment/commitmentdb/commitment_context.go b/execution/commitment/commitmentdb/commitment_context.go index 9d45a198957..95dc869b41f 100644 --- a/execution/commitment/commitmentdb/commitment_context.go +++ b/execution/commitment/commitmentdb/commitment_context.go @@ -18,7 +18,7 @@ import ( "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/kv/rawdbv3" - "github.com/erigontech/erigon/common/metrics" + "github.com/erigontech/erigon/diagnostics/metrics" "github.com/erigontech/erigon/execution/commitment" "github.com/erigontech/erigon/execution/commitment/trie" witnesstypes "github.com/erigontech/erigon/execution/commitment/witness" diff --git a/execution/exec/state.go b/execution/exec/state.go index e4d7cb2c157..6893a026ce8 100644 --- a/execution/exec/state.go +++ b/execution/exec/state.go @@ -31,7 +31,7 @@ import ( "github.com/erigontech/erigon/db/datadir" "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/services" - "github.com/erigontech/erigon/common/metrics" + "github.com/erigontech/erigon/diagnostics/metrics" "github.com/erigontech/erigon/execution/chain" "github.com/erigontech/erigon/execution/protocol/rules" "github.com/erigontech/erigon/execution/state" diff --git a/execution/execmodule/forkchoice.go b/execution/execmodule/forkchoice.go index 4b1160f7ba7..6708c2d8564 100644 --- a/execution/execmodule/forkchoice.go +++ b/execution/execmodule/forkchoice.go @@ -28,7 +28,7 @@ import ( "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/common/dbg" "github.com/erigontech/erigon/common/log/v3" - "github.com/erigontech/erigon/common/metrics" + "github.com/erigontech/erigon/diagnostics/metrics" "github.com/erigontech/erigon/db/consensuschain" "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/kv/rawdbv3" diff --git a/execution/execmodule/inserters.go b/execution/execmodule/inserters.go index aba5d772730..f6b9b0839ad 100644 --- a/execution/execmodule/inserters.go +++ b/execution/execmodule/inserters.go @@ -21,7 +21,7 @@ import ( "fmt" "math/big" - "github.com/erigontech/erigon/common/metrics" + "github.com/erigontech/erigon/diagnostics/metrics" "github.com/erigontech/erigon/db/rawdb" "github.com/erigontech/erigon/execution/execmodule/moduleutil" "github.com/erigontech/erigon/execution/types" diff --git a/execution/execmodule/metrics.go b/execution/execmodule/metrics.go index ca475ce981b..62cea112f5b 100644 --- a/execution/execmodule/metrics.go +++ b/execution/execmodule/metrics.go @@ -19,7 +19,7 @@ package execmodule import ( "time" - "github.com/erigontech/erigon/common/metrics" + "github.com/erigontech/erigon/diagnostics/metrics" ) var ( diff --git a/execution/protocol/block_exec.go b/execution/protocol/block_exec.go index 30f11b8d677..cdff1ca653a 100644 --- a/execution/protocol/block_exec.go +++ b/execution/protocol/block_exec.go @@ -33,7 +33,7 @@ import ( "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/common/math" "github.com/erigontech/erigon/common/u256" - "github.com/erigontech/erigon/common/metrics" + "github.com/erigontech/erigon/diagnostics/metrics" "github.com/erigontech/erigon/execution/chain" "github.com/erigontech/erigon/execution/protocol/params" "github.com/erigontech/erigon/execution/protocol/rules" diff --git a/execution/stagedsync/exec3_metrics.go b/execution/stagedsync/exec3_metrics.go index a98a6df4821..18f972ea169 100644 --- a/execution/stagedsync/exec3_metrics.go +++ b/execution/stagedsync/exec3_metrics.go @@ -12,7 +12,7 @@ import ( "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/state/changeset" - "github.com/erigontech/erigon/common/metrics" + "github.com/erigontech/erigon/diagnostics/metrics" "github.com/erigontech/erigon/execution/commitment" "github.com/erigontech/erigon/execution/protocol/rules" "github.com/erigontech/erigon/execution/state" diff --git a/execution/stagedsync/exec3_parallel.go b/execution/stagedsync/exec3_parallel.go index bb663453e33..a7c6d3889b9 100644 --- a/execution/stagedsync/exec3_parallel.go +++ b/execution/stagedsync/exec3_parallel.go @@ -24,7 +24,7 @@ import ( "github.com/erigontech/erigon/db/kv/temporal" "github.com/erigontech/erigon/db/rawdb/rawdbhelpers" "github.com/erigontech/erigon/db/state/changeset" - "github.com/erigontech/erigon/common/metrics" + "github.com/erigontech/erigon/diagnostics/metrics" "github.com/erigontech/erigon/execution/chain" "github.com/erigontech/erigon/execution/commitment" "github.com/erigontech/erigon/execution/exec" diff --git a/execution/stagedsync/headerdownload/header_algos.go b/execution/stagedsync/headerdownload/header_algos.go index 335061a577b..1087e90c60d 100644 --- a/execution/stagedsync/headerdownload/header_algos.go +++ b/execution/stagedsync/headerdownload/header_algos.go @@ -35,7 +35,7 @@ import ( "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/common/dbg" "github.com/erigontech/erigon/common/log/v3" - "github.com/erigontech/erigon/common/metrics" + "github.com/erigontech/erigon/diagnostics/metrics" "github.com/erigontech/erigon/db/config3" "github.com/erigontech/erigon/db/etl" "github.com/erigontech/erigon/db/kv" diff --git a/execution/stagedsync/metrics.go b/execution/stagedsync/metrics.go index 4452c7a1b72..ea47c888d6a 100644 --- a/execution/stagedsync/metrics.go +++ b/execution/stagedsync/metrics.go @@ -1,7 +1,7 @@ package stagedsync import ( - "github.com/erigontech/erigon/common/metrics" + "github.com/erigontech/erigon/diagnostics/metrics" "github.com/erigontech/erigon/execution/stagedsync/stages" ) diff --git a/execution/stagedsync/stage_bodies.go b/execution/stagedsync/stage_bodies.go index 1081c106a78..6ed36ce4b0c 100644 --- a/execution/stagedsync/stage_bodies.go +++ b/execution/stagedsync/stage_bodies.go @@ -25,7 +25,7 @@ import ( "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/common/dbg" "github.com/erigontech/erigon/common/log/v3" - "github.com/erigontech/erigon/common/metrics" + "github.com/erigontech/erigon/diagnostics/metrics" "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/rawdb" "github.com/erigontech/erigon/db/rawdb/blockio" diff --git a/execution/stagedsync/stage_execute.go b/execution/stagedsync/stage_execute.go index 19b4c517056..2e55314b533 100644 --- a/execution/stagedsync/stage_execute.go +++ b/execution/stagedsync/stage_execute.go @@ -42,7 +42,7 @@ import ( "github.com/erigontech/erigon/db/state" "github.com/erigontech/erigon/db/state/changeset" "github.com/erigontech/erigon/db/state/execctx" - "github.com/erigontech/erigon/common/metrics" + "github.com/erigontech/erigon/diagnostics/metrics" "github.com/erigontech/erigon/execution/chain" "github.com/erigontech/erigon/execution/protocol/rules" "github.com/erigontech/erigon/execution/stagedsync/stages" diff --git a/execution/stagedsync/stage_mining_exec.go b/execution/stagedsync/stage_mining_exec.go index 0eb31fffed4..d961112bc29 100644 --- a/execution/stagedsync/stage_mining_exec.go +++ b/execution/stagedsync/stage_mining_exec.go @@ -29,7 +29,7 @@ import ( "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/common/dbg" "github.com/erigontech/erigon/common/log/v3" - "github.com/erigontech/erigon/common/metrics" + "github.com/erigontech/erigon/diagnostics/metrics" "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/kv/membatchwithdb" "github.com/erigontech/erigon/db/kv/temporal" diff --git a/execution/stagedsync/stageloop/stageloop.go b/execution/stagedsync/stageloop/stageloop.go index 678d33f6656..29448c28cdd 100644 --- a/execution/stagedsync/stageloop/stageloop.go +++ b/execution/stagedsync/stageloop/stageloop.go @@ -26,7 +26,7 @@ import ( "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/common/dbg" "github.com/erigontech/erigon/common/log/v3" - "github.com/erigontech/erigon/common/metrics" + "github.com/erigontech/erigon/diagnostics/metrics" "github.com/erigontech/erigon/db/datadir" "github.com/erigontech/erigon/db/downloader" "github.com/erigontech/erigon/db/kv" diff --git a/execution/stagedsync/stages/metrics.go b/execution/stagedsync/stages/metrics.go index 51b78f05a39..b0d03fee05e 100644 --- a/execution/stagedsync/stages/metrics.go +++ b/execution/stagedsync/stages/metrics.go @@ -22,7 +22,7 @@ import ( "github.com/huandu/xstrings" "github.com/erigontech/erigon/db/kv" - "github.com/erigontech/erigon/common/metrics" + "github.com/erigontech/erigon/diagnostics/metrics" ) var SyncMetrics = map[SyncStage]metrics.Gauge{} diff --git a/node/debug/flags.go b/node/debug/flags.go index e0ce954d215..2050dde7914 100644 --- a/node/debug/flags.go +++ b/node/debug/flags.go @@ -39,7 +39,7 @@ import ( "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/db/downloader" "github.com/erigontech/erigon/diagnostics/mem" - "github.com/erigontech/erigon/common/metrics" + "github.com/erigontech/erigon/diagnostics/metrics" "github.com/erigontech/erigon/execution/tracing/tracers" "github.com/erigontech/erigon/node/logging" ) diff --git a/node/logging/logging.go b/node/logging/logging.go index 08ed9debdf3..f04491070a0 100644 --- a/node/logging/logging.go +++ b/node/logging/logging.go @@ -27,7 +27,7 @@ import ( "gopkg.in/natefinch/lumberjack.v2" "github.com/erigontech/erigon/common/log/v3" - "github.com/erigontech/erigon/common/metrics" + "github.com/erigontech/erigon/diagnostics/metrics" ) // Determine the log dir path based on the given urfave context diff --git a/node/shards/state_cache.go b/node/shards/state_cache.go index 19ff92bb314..c4a3e772072 100644 --- a/node/shards/state_cache.go +++ b/node/shards/state_cache.go @@ -27,7 +27,7 @@ import ( "github.com/holiman/uint256" "github.com/erigontech/erigon/common" - "github.com/erigontech/erigon/common/metrics" + "github.com/erigontech/erigon/diagnostics/metrics" "github.com/erigontech/erigon/execution/types/accounts" ) diff --git a/p2p/discover/metrics.go b/p2p/discover/metrics.go index 58fa38ed042..1accb017801 100644 --- a/p2p/discover/metrics.go +++ b/p2p/discover/metrics.go @@ -21,7 +21,7 @@ import ( "net" "net/netip" - "github.com/erigontech/erigon/common/metrics" + "github.com/erigontech/erigon/diagnostics/metrics" ) const ( diff --git a/p2p/metrics.go b/p2p/metrics.go index e78d32ebf2f..9ee2519983f 100644 --- a/p2p/metrics.go +++ b/p2p/metrics.go @@ -24,7 +24,7 @@ package p2p import ( "net" - "github.com/erigontech/erigon/common/metrics" + "github.com/erigontech/erigon/diagnostics/metrics" ) const ( diff --git a/p2p/peer.go b/p2p/peer.go index 5b20afce4a2..5825b3e7bb6 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -33,7 +33,7 @@ import ( "github.com/erigontech/erigon/common/dbg" "github.com/erigontech/erigon/common/log/v3" "github.com/ethereum/go-ethereum/common/mclock" - "github.com/erigontech/erigon/common/metrics" + "github.com/erigontech/erigon/diagnostics/metrics" "github.com/erigontech/erigon/execution/rlp" "github.com/erigontech/erigon/p2p/enode" "github.com/erigontech/erigon/p2p/enr" diff --git a/polygon/heimdall/poshttp/http.go b/polygon/heimdall/poshttp/http.go index f50a862bf8a..83c0aae6df6 100644 --- a/polygon/heimdall/poshttp/http.go +++ b/polygon/heimdall/poshttp/http.go @@ -14,7 +14,7 @@ import ( "time" "github.com/erigontech/erigon/common/log/v3" - "github.com/erigontech/erigon/common/metrics" + "github.com/erigontech/erigon/diagnostics/metrics" ) var ( diff --git a/polygon/heimdall/poshttp/metrics.go b/polygon/heimdall/poshttp/metrics.go index 8815de9bd2a..c8eb248da27 100644 --- a/polygon/heimdall/poshttp/metrics.go +++ b/polygon/heimdall/poshttp/metrics.go @@ -20,7 +20,7 @@ import ( "context" "time" - "github.com/erigontech/erigon/common/metrics" + "github.com/erigontech/erigon/diagnostics/metrics" ) type ( diff --git a/polygon/sync/metrics.go b/polygon/sync/metrics.go index fb0200e7da6..22adabf9bab 100644 --- a/polygon/sync/metrics.go +++ b/polygon/sync/metrics.go @@ -19,7 +19,7 @@ package sync import ( "time" - "github.com/erigontech/erigon/common/metrics" + "github.com/erigontech/erigon/diagnostics/metrics" ) var ( diff --git a/rpc/metrics.go b/rpc/metrics.go index 7e318d90f4f..9a43075b750 100644 --- a/rpc/metrics.go +++ b/rpc/metrics.go @@ -24,7 +24,7 @@ import ( "reflect" "strings" - "github.com/erigontech/erigon/common/metrics" + "github.com/erigontech/erigon/diagnostics/metrics" ) var ( diff --git a/rpc/rpchelper/metrics.go b/rpc/rpchelper/metrics.go index 7294dc331bf..a91dd46609a 100644 --- a/rpc/rpchelper/metrics.go +++ b/rpc/rpchelper/metrics.go @@ -17,7 +17,7 @@ package rpchelper import ( - "github.com/erigontech/erigon/common/metrics" + "github.com/erigontech/erigon/diagnostics/metrics" ) const ( diff --git a/txnprovider/shutter/metrics.go b/txnprovider/shutter/metrics.go index d1bb6a11254..3e27491fc76 100644 --- a/txnprovider/shutter/metrics.go +++ b/txnprovider/shutter/metrics.go @@ -1,6 +1,6 @@ package shutter -import "github.com/erigontech/erigon/common/metrics" +import "github.com/erigontech/erigon/diagnostics/metrics" var ( encryptedTxnsPoolAdded = metrics.GetOrCreateCounter("shutter_encrypted_txns_pool_added") diff --git a/txnprovider/txpool/metrics.go b/txnprovider/txpool/metrics.go index 7d54cd0b29e..e6d950321ae 100644 --- a/txnprovider/txpool/metrics.go +++ b/txnprovider/txpool/metrics.go @@ -16,7 +16,7 @@ package txpool -import "github.com/erigontech/erigon/common/metrics" +import "github.com/erigontech/erigon/diagnostics/metrics" var ( processBatchTxnsTimer = metrics.NewSummary(`pool_process_remote_txs`) From db09ff4a7ef4007547dd38296a7d0817db453aef Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Wed, 4 Feb 2026 14:46:41 +1100 Subject: [PATCH 13/16] Move the block metrics under execution --- execution/execmodule/forkchoice.go | 2 +- execution/execmodule/inserters.go | 2 +- .../metrics/block.go | 21 ++++++++++--------- .../stagedsync/headerdownload/header_algos.go | 2 +- execution/stagedsync/stage_bodies.go | 2 +- execution/stagedsync/stage_mining_exec.go | 2 +- execution/stagedsync/stageloop/stageloop.go | 2 +- node/logging/logging.go | 2 +- 8 files changed, 18 insertions(+), 17 deletions(-) rename diagnostics/metrics/block_metrics.go => execution/metrics/block.go (69%) diff --git a/execution/execmodule/forkchoice.go b/execution/execmodule/forkchoice.go index 6708c2d8564..4424e029c18 100644 --- a/execution/execmodule/forkchoice.go +++ b/execution/execmodule/forkchoice.go @@ -28,7 +28,7 @@ import ( "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/common/dbg" "github.com/erigontech/erigon/common/log/v3" - "github.com/erigontech/erigon/diagnostics/metrics" + "github.com/erigontech/erigon/execution/metrics" "github.com/erigontech/erigon/db/consensuschain" "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/kv/rawdbv3" diff --git a/execution/execmodule/inserters.go b/execution/execmodule/inserters.go index f6b9b0839ad..e98cb2dba73 100644 --- a/execution/execmodule/inserters.go +++ b/execution/execmodule/inserters.go @@ -21,7 +21,7 @@ import ( "fmt" "math/big" - "github.com/erigontech/erigon/diagnostics/metrics" + "github.com/erigontech/erigon/execution/metrics" "github.com/erigontech/erigon/db/rawdb" "github.com/erigontech/erigon/execution/execmodule/moduleutil" "github.com/erigontech/erigon/execution/types" diff --git a/diagnostics/metrics/block_metrics.go b/execution/metrics/block.go similarity index 69% rename from diagnostics/metrics/block_metrics.go rename to execution/metrics/block.go index a5d2d564ee8..47ee737c155 100644 --- a/diagnostics/metrics/block_metrics.go +++ b/execution/metrics/block.go @@ -20,6 +20,7 @@ import ( "time" "github.com/erigontech/erigon/common/log/v3" + diagmetrics "github.com/erigontech/erigon/diagnostics/metrics" ) var ( @@ -27,18 +28,18 @@ var ( DelayLoggingEnabled bool - BlockConsumerHeaderDownloadDelay = NewSummary(`block_consumer_delay{type="header_download"}`) - BlockConsumerBodyDownloadDelay = NewSummary(`block_consumer_delay{type="body_download"}`) - BlockConsumerPreExecutionDelay = NewSummary(`block_consumer_delay{type="pre_execution"}`) - BlockConsumerPostExecutionDelay = NewSummary(`block_consumer_delay{type="post_execution"}`) - BlockProducerProductionDelay = NewSummary(`block_producer_delay{type="production"}`) + BlockConsumerHeaderDownloadDelay = diagmetrics.NewSummary(`block_consumer_delay{type="header_download"}`) + BlockConsumerBodyDownloadDelay = diagmetrics.NewSummary(`block_consumer_delay{type="body_download"}`) + BlockConsumerPreExecutionDelay = diagmetrics.NewSummary(`block_consumer_delay{type="pre_execution"}`) + BlockConsumerPostExecutionDelay = diagmetrics.NewSummary(`block_consumer_delay{type="post_execution"}`) + BlockProducerProductionDelay = diagmetrics.NewSummary(`block_producer_delay{type="production"}`) - ChainTipMgasPerSec = NewGauge(`chain_tip_mgas_per_sec`) + ChainTipMgasPerSec = diagmetrics.NewGauge(`chain_tip_mgas_per_sec`) - BlockConsumerHeaderDownloadDelayHistogram = NewHistogram(`block_consumer_delay_hist{type="header_download"}`, delayBuckets) - BlockConsumerBodyDownloadDelayHistogram = NewHistogram(`block_consumer_delay_hist{type="body_download"}`, delayBuckets) - BlockConsumerPreExecutionDelayHistogram = NewHistogram(`block_consumer_delay_hist{type="pre_execution"}`, delayBuckets) - BlockConsumerPostExecutionDelayHistogram = NewHistogram(`block_consumer_delay_hist{type="post_execution"}`, delayBuckets) + BlockConsumerHeaderDownloadDelayHistogram = diagmetrics.NewHistogram(`block_consumer_delay_hist{type="header_download"}`, delayBuckets) + BlockConsumerBodyDownloadDelayHistogram = diagmetrics.NewHistogram(`block_consumer_delay_hist{type="body_download"}`, delayBuckets) + BlockConsumerPreExecutionDelayHistogram = diagmetrics.NewHistogram(`block_consumer_delay_hist{type="pre_execution"}`, delayBuckets) + BlockConsumerPostExecutionDelayHistogram = diagmetrics.NewHistogram(`block_consumer_delay_hist{type="post_execution"}`, delayBuckets) ) func UpdateBlockConsumerHeaderDownloadDelay(blockTime uint64, blockNumber uint64, log log.Logger) { diff --git a/execution/stagedsync/headerdownload/header_algos.go b/execution/stagedsync/headerdownload/header_algos.go index 1087e90c60d..ee1fd4a4751 100644 --- a/execution/stagedsync/headerdownload/header_algos.go +++ b/execution/stagedsync/headerdownload/header_algos.go @@ -35,7 +35,7 @@ import ( "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/common/dbg" "github.com/erigontech/erigon/common/log/v3" - "github.com/erigontech/erigon/diagnostics/metrics" + "github.com/erigontech/erigon/execution/metrics" "github.com/erigontech/erigon/db/config3" "github.com/erigontech/erigon/db/etl" "github.com/erigontech/erigon/db/kv" diff --git a/execution/stagedsync/stage_bodies.go b/execution/stagedsync/stage_bodies.go index 6ed36ce4b0c..2ff8be448a2 100644 --- a/execution/stagedsync/stage_bodies.go +++ b/execution/stagedsync/stage_bodies.go @@ -25,7 +25,7 @@ import ( "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/common/dbg" "github.com/erigontech/erigon/common/log/v3" - "github.com/erigontech/erigon/diagnostics/metrics" + "github.com/erigontech/erigon/execution/metrics" "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/rawdb" "github.com/erigontech/erigon/db/rawdb/blockio" diff --git a/execution/stagedsync/stage_mining_exec.go b/execution/stagedsync/stage_mining_exec.go index d961112bc29..dafae0d556d 100644 --- a/execution/stagedsync/stage_mining_exec.go +++ b/execution/stagedsync/stage_mining_exec.go @@ -29,7 +29,7 @@ import ( "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/common/dbg" "github.com/erigontech/erigon/common/log/v3" - "github.com/erigontech/erigon/diagnostics/metrics" + "github.com/erigontech/erigon/execution/metrics" "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/kv/membatchwithdb" "github.com/erigontech/erigon/db/kv/temporal" diff --git a/execution/stagedsync/stageloop/stageloop.go b/execution/stagedsync/stageloop/stageloop.go index 29448c28cdd..409eb3df8c3 100644 --- a/execution/stagedsync/stageloop/stageloop.go +++ b/execution/stagedsync/stageloop/stageloop.go @@ -26,7 +26,7 @@ import ( "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/common/dbg" "github.com/erigontech/erigon/common/log/v3" - "github.com/erigontech/erigon/diagnostics/metrics" + "github.com/erigontech/erigon/execution/metrics" "github.com/erigontech/erigon/db/datadir" "github.com/erigontech/erigon/db/downloader" "github.com/erigontech/erigon/db/kv" diff --git a/node/logging/logging.go b/node/logging/logging.go index f04491070a0..f241df6b6a9 100644 --- a/node/logging/logging.go +++ b/node/logging/logging.go @@ -27,7 +27,7 @@ import ( "gopkg.in/natefinch/lumberjack.v2" "github.com/erigontech/erigon/common/log/v3" - "github.com/erigontech/erigon/diagnostics/metrics" + "github.com/erigontech/erigon/execution/metrics" ) // Determine the log dir path based on the given urfave context From 2cf55182de76dcdbbbc578ba2f6e1b1d7d1ccc37 Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Mon, 9 Feb 2026 15:14:48 +1100 Subject: [PATCH 14/16] Fix new lints... --- cmd/integration/commands/commitment.go | 2 +- cmd/utils/flags.go | 2 +- db/downloader/downloader.go | 2 +- execution/execmodule/forkchoice.go | 2 +- execution/execmodule/inserters.go | 2 +- execution/rlp/decode.go | 14 +++++++------- .../stagedsync/headerdownload/header_algos.go | 2 +- execution/stagedsync/stage_bodies.go | 2 +- execution/stagedsync/stage_mining_exec.go | 2 +- execution/stagedsync/stageloop/stageloop.go | 2 +- go.mod | 1 - go.sum | 2 -- p2p/config.go | 2 +- p2p/dial.go | 2 +- p2p/dial_test.go | 2 +- p2p/discover/common.go | 2 +- p2p/discover/ntp.go | 2 +- p2p/discover/table.go | 2 +- p2p/discover/table_reval.go | 2 +- p2p/discover/table_reval_test.go | 2 +- p2p/discover/table_test.go | 2 +- p2p/discover/v4wire/v4wire.go | 4 ++-- p2p/discover/v5_udp.go | 2 +- p2p/discover/v5wire/crypto.go | 2 +- p2p/discover/v5wire/crypto_test.go | 2 +- p2p/discover/v5wire/encoding.go | 10 +++++----- p2p/discover/v5wire/encoding_test.go | 4 ++-- p2p/discover/v5wire/msg.go | 4 ++-- p2p/discover/v5wire/session.go | 4 ++-- p2p/dnsdisc/client.go | 2 +- p2p/dnsdisc/client_test.go | 2 +- p2p/dnsdisc/sync.go | 2 +- p2p/netutil/net.go | 2 +- p2p/peer.go | 2 +- p2p/server.go | 2 +- 35 files changed, 47 insertions(+), 50 deletions(-) diff --git a/cmd/integration/commands/commitment.go b/cmd/integration/commands/commitment.go index ef57f2145b0..fe27499fc7f 100644 --- a/cmd/integration/commands/commitment.go +++ b/cmd/integration/commands/commitment.go @@ -673,7 +673,7 @@ func benchHistoryLookup(ctx context.Context, logger log.Logger) error { // benchSnapshotsHistoryLookup benchmarks history lookups across snapshot files func benchSnapshotsHistoryLookup(ctx context.Context, tx kv.TemporalTx, historyFiles []dbstate.VisibleFile, compactKey []byte, samplePct float64, rng *rand.Rand, logger log.Logger) ([]HistoryBenchStats, error) { - var allFileStats []HistoryBenchStats + allFileStats := make([]HistoryBenchStats, 0, len(historyFiles)) for _, f := range historyFiles { fpath := f.Fullpath() diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index fba6c0c70bb..b4b64882cac 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -44,12 +44,12 @@ import ( "github.com/erigontech/erigon/common" libkzg "github.com/erigontech/erigon/common/crypto/kzg" "github.com/erigontech/erigon/common/log/v3" - "github.com/erigontech/erigon/diagnostics/metrics" "github.com/erigontech/erigon/db/datadir" "github.com/erigontech/erigon/db/downloader/downloadercfg" "github.com/erigontech/erigon/db/snapcfg" "github.com/erigontech/erigon/db/state/statecfg" "github.com/erigontech/erigon/db/version" + "github.com/erigontech/erigon/diagnostics/metrics" "github.com/erigontech/erigon/execution/builder/buildercfg" "github.com/erigontech/erigon/execution/chain/networkname" chainspec "github.com/erigontech/erigon/execution/chain/spec" diff --git a/db/downloader/downloader.go b/db/downloader/downloader.go index 77dfffcb0c0..d5f37faf126 100644 --- a/db/downloader/downloader.go +++ b/db/downloader/downloader.go @@ -434,7 +434,7 @@ func (d *Downloader) snapshotDataLooksComplete(info *metainfo.Info) bool { // Log the names of torrents missing metainfo. We can pass a level in to scale the urgency of the // situation. func (d *Downloader) logNoMetadata(lvl log.Lvl, torrents []snapshot) { - var noMetadata []string + noMetadata := make([]string, 0, len(torrents)) for _, ps := range torrents { t, ok := d.torrentClient.Torrent(ps.InfoHash) diff --git a/execution/execmodule/forkchoice.go b/execution/execmodule/forkchoice.go index 342b6c72427..b1615f3a419 100644 --- a/execution/execmodule/forkchoice.go +++ b/execution/execmodule/forkchoice.go @@ -28,7 +28,6 @@ import ( "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/common/dbg" "github.com/erigontech/erigon/common/log/v3" - "github.com/erigontech/erigon/execution/metrics" "github.com/erigontech/erigon/db/consensuschain" "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/kv/rawdbv3" @@ -37,6 +36,7 @@ import ( "github.com/erigontech/erigon/db/state/execctx" "github.com/erigontech/erigon/execution/commitment/commitmentdb" "github.com/erigontech/erigon/execution/engineapi/engine_helpers" + "github.com/erigontech/erigon/execution/metrics" "github.com/erigontech/erigon/execution/protocol/rules" "github.com/erigontech/erigon/execution/stagedsync" "github.com/erigontech/erigon/execution/stagedsync/stages" diff --git a/execution/execmodule/inserters.go b/execution/execmodule/inserters.go index e98cb2dba73..1a9d39513da 100644 --- a/execution/execmodule/inserters.go +++ b/execution/execmodule/inserters.go @@ -21,9 +21,9 @@ import ( "fmt" "math/big" - "github.com/erigontech/erigon/execution/metrics" "github.com/erigontech/erigon/db/rawdb" "github.com/erigontech/erigon/execution/execmodule/moduleutil" + "github.com/erigontech/erigon/execution/metrics" "github.com/erigontech/erigon/execution/types" "github.com/erigontech/erigon/node/gointerfaces/executionproto" ) diff --git a/execution/rlp/decode.go b/execution/rlp/decode.go index dfc345832c5..796f5b49833 100644 --- a/execution/rlp/decode.go +++ b/execution/rlp/decode.go @@ -42,12 +42,12 @@ import ( var EOL = errors.New("rlp: end of list") var ( - ErrExpectedString = errors.New("rlp: expected String or Byte") - ErrExpectedList = errors.New("rlp: expected List") - ErrCanonInt = errors.New("rlp: non-canonical integer format") - ErrCanonSize = errors.New("rlp: non-canonical size information") - ErrElemTooLarge = errors.New("rlp: element is larger than containing list") - ErrValueTooLarge = errors.New("rlp: value size exceeds available input length") + ErrExpectedString = errors.New("rlp: expected String or Byte") + ErrExpectedList = errors.New("rlp: expected List") + ErrCanonInt = errors.New("rlp: non-canonical integer format") + ErrCanonSize = errors.New("rlp: non-canonical size information") + ErrElemTooLarge = errors.New("rlp: element is larger than containing list") + ErrValueTooLarge = errors.New("rlp: value size exceeds available input length") ErrMoreThanOneValue = errors.New("rlp: input contains more than one value") ErrWrongTxTypePrefix = errors.New("rlp: only 1-byte tx type prefix is supported") ErrUnknownTxTypePrefix = errors.New("rlp: unknown tx type prefix") @@ -1224,7 +1224,7 @@ func (s *Stream) readUint(size byte) (uint64, error) { // The error needs to be adjusted to become ErrCanonInt in this case. return 0, ErrCanonSize } - return binary.BigEndian.Uint64(buffer[:]), nil + return binary.BigEndian.Uint64(buffer), nil } } diff --git a/execution/stagedsync/headerdownload/header_algos.go b/execution/stagedsync/headerdownload/header_algos.go index ee1fd4a4751..512e4e09b1c 100644 --- a/execution/stagedsync/headerdownload/header_algos.go +++ b/execution/stagedsync/headerdownload/header_algos.go @@ -35,13 +35,13 @@ import ( "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/common/dbg" "github.com/erigontech/erigon/common/log/v3" - "github.com/erigontech/erigon/execution/metrics" "github.com/erigontech/erigon/db/config3" "github.com/erigontech/erigon/db/etl" "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/kv/dbutils" "github.com/erigontech/erigon/db/rawdb" "github.com/erigontech/erigon/db/services" + "github.com/erigontech/erigon/execution/metrics" "github.com/erigontech/erigon/execution/protocol/rules" "github.com/erigontech/erigon/execution/rlp" "github.com/erigontech/erigon/execution/stagedsync/dataflow" diff --git a/execution/stagedsync/stage_bodies.go b/execution/stagedsync/stage_bodies.go index 2ff8be448a2..20577cce8de 100644 --- a/execution/stagedsync/stage_bodies.go +++ b/execution/stagedsync/stage_bodies.go @@ -25,13 +25,13 @@ import ( "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/common/dbg" "github.com/erigontech/erigon/common/log/v3" - "github.com/erigontech/erigon/execution/metrics" "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/rawdb" "github.com/erigontech/erigon/db/rawdb/blockio" "github.com/erigontech/erigon/db/services" "github.com/erigontech/erigon/diagnostics/diaglib" "github.com/erigontech/erigon/execution/chain" + "github.com/erigontech/erigon/execution/metrics" "github.com/erigontech/erigon/execution/stagedsync/bodydownload" "github.com/erigontech/erigon/execution/stagedsync/dataflow" "github.com/erigontech/erigon/execution/stagedsync/headerdownload" diff --git a/execution/stagedsync/stage_mining_exec.go b/execution/stagedsync/stage_mining_exec.go index f96c9b54939..92c58cd2a2b 100644 --- a/execution/stagedsync/stage_mining_exec.go +++ b/execution/stagedsync/stage_mining_exec.go @@ -29,7 +29,6 @@ import ( "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/common/dbg" "github.com/erigontech/erigon/common/log/v3" - "github.com/erigontech/erigon/execution/metrics" "github.com/erigontech/erigon/db/kv" "github.com/erigontech/erigon/db/kv/membatchwithdb" "github.com/erigontech/erigon/db/kv/temporal" @@ -37,6 +36,7 @@ import ( "github.com/erigontech/erigon/db/services" "github.com/erigontech/erigon/db/state/execctx" "github.com/erigontech/erigon/execution/chain" + "github.com/erigontech/erigon/execution/metrics" "github.com/erigontech/erigon/execution/protocol" "github.com/erigontech/erigon/execution/protocol/aa" "github.com/erigontech/erigon/execution/protocol/params" diff --git a/execution/stagedsync/stageloop/stageloop.go b/execution/stagedsync/stageloop/stageloop.go index 4026cbda277..2eac976639f 100644 --- a/execution/stagedsync/stageloop/stageloop.go +++ b/execution/stagedsync/stageloop/stageloop.go @@ -26,7 +26,6 @@ import ( "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/common/dbg" "github.com/erigontech/erigon/common/log/v3" - "github.com/erigontech/erigon/execution/metrics" "github.com/erigontech/erigon/db/datadir" "github.com/erigontech/erigon/db/downloader" "github.com/erigontech/erigon/db/kv" @@ -38,6 +37,7 @@ import ( "github.com/erigontech/erigon/db/state/execctx" "github.com/erigontech/erigon/execution/chain" "github.com/erigontech/erigon/execution/engineapi/engine_helpers" + "github.com/erigontech/erigon/execution/metrics" execp2p "github.com/erigontech/erigon/execution/p2p" "github.com/erigontech/erigon/execution/protocol/misc" "github.com/erigontech/erigon/execution/protocol/rules" diff --git a/go.mod b/go.mod index 96753ab834a..fedeab7e6fa 100644 --- a/go.mod +++ b/go.mod @@ -100,7 +100,6 @@ require ( github.com/quic-go/quic-go v0.49.1 github.com/rs/cors v1.11.1 github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417 - github.com/shirou/gopsutil v3.21.11+incompatible github.com/shirou/gopsutil/v4 v4.25.10 github.com/spaolacci/murmur3 v1.1.0 github.com/spf13/afero v1.10.0 diff --git a/go.sum b/go.sum index cc051f5012a..b04e3d57038 100644 --- a/go.sum +++ b/go.sum @@ -888,8 +888,6 @@ github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5P github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= -github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= -github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shirou/gopsutil/v4 v4.25.10 h1:at8lk/5T1OgtuCp+AwrDofFRjnvosn0nkN2OLQ6g8tA= github.com/shirou/gopsutil/v4 v4.25.10/go.mod h1:+kSwyC8DRUD9XXEHCAFjK+0nuArFJM0lva+StQAcskM= github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= diff --git a/p2p/config.go b/p2p/config.go index 472e9f70b6f..e89f43605f0 100644 --- a/p2p/config.go +++ b/p2p/config.go @@ -5,10 +5,10 @@ import ( "net" "strconv" - "github.com/ethereum/go-ethereum/common/mclock" "github.com/erigontech/erigon/p2p/enode" "github.com/erigontech/erigon/p2p/nat" "github.com/erigontech/erigon/p2p/netutil" + "github.com/ethereum/go-ethereum/common/mclock" ) // Config holds Server options. diff --git a/p2p/dial.go b/p2p/dial.go index d5fbb316426..5dce04a082d 100644 --- a/p2p/dial.go +++ b/p2p/dial.go @@ -33,9 +33,9 @@ import ( "github.com/erigontech/erigon/common/dbg" "github.com/erigontech/erigon/common/log/v3" - "github.com/ethereum/go-ethereum/common/mclock" "github.com/erigontech/erigon/p2p/enode" "github.com/erigontech/erigon/p2p/netutil" + "github.com/ethereum/go-ethereum/common/mclock" ) const ( diff --git a/p2p/dial_test.go b/p2p/dial_test.go index 7d69e25171e..84d701da839 100644 --- a/p2p/dial_test.go +++ b/p2p/dial_test.go @@ -31,10 +31,10 @@ import ( "time" "github.com/erigontech/erigon/common/log/v3" - "github.com/ethereum/go-ethereum/common/mclock" "github.com/erigontech/erigon/common/testlog" "github.com/erigontech/erigon/p2p/enode" "github.com/erigontech/erigon/p2p/netutil" + "github.com/ethereum/go-ethereum/common/mclock" ) // This test checks that dynamic dials are launched from discovery results. diff --git a/p2p/discover/common.go b/p2p/discover/common.go index f8149dacbeb..48b20a6a09f 100644 --- a/p2p/discover/common.go +++ b/p2p/discover/common.go @@ -29,11 +29,11 @@ import ( "sync" "time" - "github.com/ethereum/go-ethereum/common/mclock" "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/p2p/enode" "github.com/erigontech/erigon/p2p/enr" "github.com/erigontech/erigon/p2p/netutil" + "github.com/ethereum/go-ethereum/common/mclock" ) // UDPConn is a network connection on which discovery can operate. diff --git a/p2p/discover/ntp.go b/p2p/discover/ntp.go index 4e4c85ca992..4f5c5f6582c 100644 --- a/p2p/discover/ntp.go +++ b/p2p/discover/ntp.go @@ -98,7 +98,7 @@ func sntpDrift(measurements int) (time.Duration, error) { nanosec := sec*1e9 + (frac*1e9)>>32 - t := time.Date(1900, 1, 1, 0, 0, 0, 0, time.UTC).Add(time.Duration(nanosec)).Local() + t := time.Date(1900, 1, 1, 0, 0, 0, 0, time.UTC).Add(time.Duration(nanosec)) // Calculate the drift based on an assumed answer time of RRT/2 drifts = append(drifts, sent.Sub(t)+elapsed/2) diff --git a/p2p/discover/table.go b/p2p/discover/table.go index 915c5490bc2..07540b5adea 100644 --- a/p2p/discover/table.go +++ b/p2p/discover/table.go @@ -35,10 +35,10 @@ import ( "github.com/erigontech/erigon/common" "github.com/erigontech/erigon/common/log/v3" - "github.com/ethereum/go-ethereum/common/mclock" "github.com/erigontech/erigon/p2p/enode" "github.com/erigontech/erigon/p2p/event" "github.com/erigontech/erigon/p2p/netutil" + "github.com/ethereum/go-ethereum/common/mclock" ) const ( diff --git a/p2p/discover/table_reval.go b/p2p/discover/table_reval.go index 00b96d74e8a..8ae345ee6f8 100644 --- a/p2p/discover/table_reval.go +++ b/p2p/discover/table_reval.go @@ -22,8 +22,8 @@ import ( "slices" "time" - "github.com/ethereum/go-ethereum/common/mclock" "github.com/erigontech/erigon/p2p/enode" + "github.com/ethereum/go-ethereum/common/mclock" ) const never = mclock.AbsTime(math.MaxInt64) diff --git a/p2p/discover/table_reval_test.go b/p2p/discover/table_reval_test.go index 3831bcaba8e..76f47e20a72 100644 --- a/p2p/discover/table_reval_test.go +++ b/p2p/discover/table_reval_test.go @@ -21,9 +21,9 @@ import ( "testing" "time" - "github.com/ethereum/go-ethereum/common/mclock" "github.com/erigontech/erigon/p2p/enode" "github.com/erigontech/erigon/p2p/enr" + "github.com/ethereum/go-ethereum/common/mclock" ) // This test checks that revalidation can handle a node disappearing while diff --git a/p2p/discover/table_test.go b/p2p/discover/table_test.go index 3419a7ed749..f413d319418 100644 --- a/p2p/discover/table_test.go +++ b/p2p/discover/table_test.go @@ -32,11 +32,11 @@ import ( "github.com/erigontech/erigon/common/crypto" "github.com/erigontech/erigon/common/log/v3" - "github.com/ethereum/go-ethereum/common/mclock" "github.com/erigontech/erigon/common/testlog" "github.com/erigontech/erigon/p2p/enode" "github.com/erigontech/erigon/p2p/enr" "github.com/erigontech/erigon/p2p/netutil" + "github.com/ethereum/go-ethereum/common/mclock" ) func TestTable_pingReplace(t *testing.T) { diff --git a/p2p/discover/v4wire/v4wire.go b/p2p/discover/v4wire/v4wire.go index baad4fae555..d4fce4ce5c2 100644 --- a/p2p/discover/v4wire/v4wire.go +++ b/p2p/discover/v4wire/v4wire.go @@ -31,11 +31,11 @@ import ( "net/netip" "time" - "github.com/erigontech/erigon/common/math" "github.com/erigontech/erigon/common/crypto" + "github.com/erigontech/erigon/common/math" + "github.com/erigontech/erigon/execution/rlp" "github.com/erigontech/erigon/p2p/enode" "github.com/erigontech/erigon/p2p/enr" - "github.com/erigontech/erigon/execution/rlp" ) // RPC packet types diff --git a/p2p/discover/v5_udp.go b/p2p/discover/v5_udp.go index ca38cc538f9..9a5158b5d96 100644 --- a/p2p/discover/v5_udp.go +++ b/p2p/discover/v5_udp.go @@ -33,12 +33,12 @@ import ( "sync" "time" - "github.com/ethereum/go-ethereum/common/mclock" "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/p2p/discover/v5wire" "github.com/erigontech/erigon/p2p/enode" "github.com/erigontech/erigon/p2p/enr" "github.com/erigontech/erigon/p2p/netutil" + "github.com/ethereum/go-ethereum/common/mclock" ) const ( diff --git a/p2p/discover/v5wire/crypto.go b/p2p/discover/v5wire/crypto.go index e651946c46f..568eff8d4ee 100644 --- a/p2p/discover/v5wire/crypto.go +++ b/p2p/discover/v5wire/crypto.go @@ -28,8 +28,8 @@ import ( "fmt" "hash" - "github.com/erigontech/erigon/common/math" "github.com/erigontech/erigon/common/crypto" + "github.com/erigontech/erigon/common/math" "github.com/erigontech/erigon/p2p/enode" "golang.org/x/crypto/hkdf" ) diff --git a/p2p/discover/v5wire/crypto_test.go b/p2p/discover/v5wire/crypto_test.go index 8a673245da3..1e2cbeb7158 100644 --- a/p2p/discover/v5wire/crypto_test.go +++ b/p2p/discover/v5wire/crypto_test.go @@ -28,8 +28,8 @@ import ( "strings" "testing" - "github.com/erigontech/erigon/common/hexutil" "github.com/erigontech/erigon/common/crypto" + "github.com/erigontech/erigon/common/hexutil" "github.com/erigontech/erigon/p2p/enode" ) diff --git a/p2p/discover/v5wire/encoding.go b/p2p/discover/v5wire/encoding.go index 87334a9f1d0..8c26697ae54 100644 --- a/p2p/discover/v5wire/encoding.go +++ b/p2p/discover/v5wire/encoding.go @@ -32,10 +32,10 @@ import ( "hash" "slices" - "github.com/ethereum/go-ethereum/common/mclock" + "github.com/erigontech/erigon/execution/rlp" "github.com/erigontech/erigon/p2p/enode" "github.com/erigontech/erigon/p2p/enr" - "github.com/erigontech/erigon/execution/rlp" + "github.com/ethereum/go-ethereum/common/mclock" ) // TODO concurrent WHOAREYOU tie-breaker @@ -254,7 +254,7 @@ func (c *Codec) EncodeRaw(id enode.ID, head Header, msgdata []byte) ([]byte, err // Apply masking. masked := c.buf.Bytes()[sizeofMaskingIV:] mask := head.mask(id) - mask.XORKeyStream(masked[:], masked[:]) + mask.XORKeyStream(masked, masked) // Write message data. c.buf.Write(msgdata) @@ -395,12 +395,12 @@ func (c *Codec) makeHandshakeAuth(toID enode.ID, addr string, challenge *Whoarey return nil, nil, errors.New("can't generate ephemeral key") } ephpubkey := EncodePubkey(&ephkey.PublicKey) - auth.pubkey = ephpubkey[:] + auth.pubkey = ephpubkey auth.h.PubkeySize = byte(len(auth.pubkey)) // Add ID nonce signature to response. cdata := challenge.ChallengeData - idsig, err := makeIDSignature(c.sha256, c.privkey, cdata, ephpubkey[:], toID) + idsig, err := makeIDSignature(c.sha256, c.privkey, cdata, ephpubkey, toID) if err != nil { return nil, nil, fmt.Errorf("can't sign: %v", err) } diff --git a/p2p/discover/v5wire/encoding_test.go b/p2p/discover/v5wire/encoding_test.go index b58f3f12af6..f742c71d7f6 100644 --- a/p2p/discover/v5wire/encoding_test.go +++ b/p2p/discover/v5wire/encoding_test.go @@ -34,10 +34,10 @@ import ( "github.com/davecgh/go-spew/spew" - "github.com/erigontech/erigon/common/hexutil" - "github.com/ethereum/go-ethereum/common/mclock" "github.com/erigontech/erigon/common/crypto" + "github.com/erigontech/erigon/common/hexutil" "github.com/erigontech/erigon/p2p/enode" + "github.com/ethereum/go-ethereum/common/mclock" ) // To regenerate discv5 test vectors, run diff --git a/p2p/discover/v5wire/msg.go b/p2p/discover/v5wire/msg.go index d7948cff153..0290cbbfc81 100644 --- a/p2p/discover/v5wire/msg.go +++ b/p2p/discover/v5wire/msg.go @@ -24,10 +24,10 @@ import ( "net" "github.com/erigontech/erigon/common/hexutil" - "github.com/ethereum/go-ethereum/common/mclock" + "github.com/erigontech/erigon/execution/rlp" "github.com/erigontech/erigon/p2p/enode" "github.com/erigontech/erigon/p2p/enr" - "github.com/erigontech/erigon/execution/rlp" + "github.com/ethereum/go-ethereum/common/mclock" ) // Packet is implemented by all message types. diff --git a/p2p/discover/v5wire/session.go b/p2p/discover/v5wire/session.go index c01461ddb75..b81d516f928 100644 --- a/p2p/discover/v5wire/session.go +++ b/p2p/discover/v5wire/session.go @@ -25,10 +25,10 @@ import ( "encoding/binary" "time" - "github.com/ethereum/go-ethereum/common/lru" - "github.com/ethereum/go-ethereum/common/mclock" "github.com/erigontech/erigon/common/crypto" "github.com/erigontech/erigon/p2p/enode" + "github.com/ethereum/go-ethereum/common/lru" + "github.com/ethereum/go-ethereum/common/mclock" ) const handshakeTimeout = time.Second diff --git a/p2p/dnsdisc/client.go b/p2p/dnsdisc/client.go index cdb1b1e6f8b..3debb0a47cb 100644 --- a/p2p/dnsdisc/client.go +++ b/p2p/dnsdisc/client.go @@ -36,9 +36,9 @@ import ( "github.com/erigontech/erigon/common/crypto" "github.com/erigontech/erigon/common/log/v3" - "github.com/ethereum/go-ethereum/common/mclock" "github.com/erigontech/erigon/p2p/enode" "github.com/erigontech/erigon/p2p/enr" + "github.com/ethereum/go-ethereum/common/mclock" ) // Client discovers nodes by querying DNS servers. diff --git a/p2p/dnsdisc/client_test.go b/p2p/dnsdisc/client_test.go index a2e2c89c923..0145d79d89b 100644 --- a/p2p/dnsdisc/client_test.go +++ b/p2p/dnsdisc/client_test.go @@ -33,10 +33,10 @@ import ( "github.com/erigontech/erigon/common/crypto" "github.com/erigontech/erigon/common/hexutil" "github.com/erigontech/erigon/common/log/v3" - "github.com/ethereum/go-ethereum/common/mclock" "github.com/erigontech/erigon/common/testlog" "github.com/erigontech/erigon/p2p/enode" "github.com/erigontech/erigon/p2p/enr" + "github.com/ethereum/go-ethereum/common/mclock" ) var signingKeyForTesting, _ = crypto.ToECDSA(hexutil.MustDecode("0xdc599867fc513f8f5e2c2c9c489cde5e71362d1d9ec6e693e0de063236ed1240")) diff --git a/p2p/dnsdisc/sync.go b/p2p/dnsdisc/sync.go index c619f15f394..86f402647e8 100644 --- a/p2p/dnsdisc/sync.go +++ b/p2p/dnsdisc/sync.go @@ -24,8 +24,8 @@ import ( "math/rand" "time" - "github.com/ethereum/go-ethereum/common/mclock" "github.com/erigontech/erigon/p2p/enode" + "github.com/ethereum/go-ethereum/common/mclock" ) // This is the number of consecutive leaf requests that may fail before diff --git a/p2p/netutil/net.go b/p2p/netutil/net.go index fad3505b30c..f6a9cc03646 100644 --- a/p2p/netutil/net.go +++ b/p2p/netutil/net.go @@ -317,7 +317,7 @@ func (s *DistinctNetSet) key(ip netip.Addr) netip.Prefix { if s.members == nil { s.members = make(map[netip.Prefix]uint) } - p, err := ip.Prefix(int(s.Subnet)) + p, err := ip.Prefix(int(s.Subnet)) //nolint:gocritic if err != nil { panic(err) } diff --git a/p2p/peer.go b/p2p/peer.go index fdafcf60736..be36e26891a 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -32,12 +32,12 @@ import ( "github.com/erigontech/erigon/common/dbg" "github.com/erigontech/erigon/common/log/v3" - "github.com/ethereum/go-ethereum/common/mclock" "github.com/erigontech/erigon/diagnostics/metrics" "github.com/erigontech/erigon/execution/rlp" "github.com/erigontech/erigon/p2p/enode" "github.com/erigontech/erigon/p2p/enr" "github.com/erigontech/erigon/p2p/event" + "github.com/ethereum/go-ethereum/common/mclock" ) var ( diff --git a/p2p/server.go b/p2p/server.go index f85d80bcec9..597c7188a1d 100644 --- a/p2p/server.go +++ b/p2p/server.go @@ -41,13 +41,13 @@ import ( "github.com/erigontech/erigon/common/crypto" "github.com/erigontech/erigon/common/dbg" "github.com/erigontech/erigon/common/log/v3" - "github.com/ethereum/go-ethereum/common/mclock" "github.com/erigontech/erigon/p2p/discover" "github.com/erigontech/erigon/p2p/enode" "github.com/erigontech/erigon/p2p/enr" "github.com/erigontech/erigon/p2p/event" "github.com/erigontech/erigon/p2p/nat" "github.com/erigontech/erigon/p2p/netutil" + "github.com/ethereum/go-ethereum/common/mclock" ) const ( From ee40b6df3601925ee1298f77bc23393788e1f429 Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Mon, 9 Feb 2026 20:14:45 +1100 Subject: [PATCH 15/16] Prevent inconsistent golangci-lint versions from being used --- .github/workflows/lint.yml | 6 +- .golangci-lint-version | 0 Makefile | 15 +- go.mod | 177 ++++++++- go.sum | 748 +++++++++++++++++++++---------------- p2p/discover/v5_udp.go | 2 +- tools/golangci_lint.sh | 20 - 7 files changed, 593 insertions(+), 375 deletions(-) create mode 100644 .golangci-lint-version delete mode 100755 tools/golangci_lint.sh diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 9766f7c2ae0..9e29e2d864e 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -38,9 +38,5 @@ jobs: with: go-version: '1.25' - - name: Install golangci-lint - uses: golangci/golangci-lint-action@v9 - with: - version: 'v2.8.0' - + - run: go tool golangci-lint version - run: make lint diff --git a/.golangci-lint-version b/.golangci-lint-version new file mode 100644 index 00000000000..e69de29bb2d diff --git a/Makefile b/Makefile index c7a26b7872a..dadb5d9ae01 100644 --- a/Makefile +++ b/Makefile @@ -60,6 +60,7 @@ ifeq ($(CGO_CXXFLAGS),) CGO_CXXFLAGS += -g CGO_CXXFLAGS += -O2 endif +export CGO_CXXFLAGS BUILD_TAGS = @@ -83,9 +84,9 @@ GO_BUILD_ENV = GOARCH=${GOARCH} ${CPU_ARCH} CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLA GOBUILD = $(GO_BUILD_ENV) $(GO) build $(GO_RELEASE_FLAGS) $(GO_FLAGS) -tags $(BUILD_TAGS) DLV_GO_FLAGS := -gcflags='all="-N -l" -trimpath=false' GO_BUILD_DEBUG = $(GO_BUILD_ENV) CGO_CFLAGS="$(CGO_CFLAGS) -DMDBX_DEBUG=1" $(GO) build $(DLV_GO_FLAGS) $(GO_FLAGS) -tags $(BUILD_TAGS),debug -GOTEST = $(GO_BUILD_ENV) CGO_CXXFLAGS="$(CGO_CXXFLAGS)" GODEBUG=cgocheck=0 GOTRACEBACK=1 GOEXPERIMENT=synctest $(GO) test $(GO_FLAGS) ./... +GOTEST = $(GO_BUILD_ENV) GODEBUG=cgocheck=0 GOTRACEBACK=1 GOEXPERIMENT=synctest $(GO) test $(GO_FLAGS) ./... -GOINSTALL = CGO_CXXFLAGS="$(CGO_CXXFLAGS)" go install -trimpath +GOINSTALL = go install -trimpath OS = $(shell uname -s) ARCH = $(shell uname -m) @@ -335,17 +336,13 @@ kurtosis-cleanup: @echo "-----------------------------------\n" kurtosis enclave rm -f makefile-kurtosis-testnet -## lint-deps: install lint dependencies -lint-deps: - @./tools/golangci_lint.sh --install-deps - ## lintci: run golangci-lint linters lintci: - @CGO_CXXFLAGS="$(CGO_CXXFLAGS)" ./tools/golangci_lint.sh + @GOEXPERIMENT=synctest go tool golangci-lint run ## lint: run all linters -lint: - @./tools/golangci_lint.sh +lint: + @GOEXPERIMENT=synctest go tool golangci-lint run @./tools/mod_tidy_check.sh ## tidy: `go mod tidy` diff --git a/go.mod b/go.mod index fedeab7e6fa..d48cb97e2b7 100644 --- a/go.mod +++ b/go.mod @@ -96,14 +96,14 @@ require ( github.com/prysmaticlabs/go-bitfield v0.0.0-20240618144021-706c95b2dd15 github.com/prysmaticlabs/gohashtree v0.0.4-beta github.com/puzpuzpuz/xsync/v4 v4.4.0 - github.com/quasilyte/go-ruleguard/dsl v0.3.22 + github.com/quasilyte/go-ruleguard/dsl v0.3.23 github.com/quic-go/quic-go v0.49.1 github.com/rs/cors v1.11.1 github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417 - github.com/shirou/gopsutil/v4 v4.25.10 + github.com/shirou/gopsutil/v4 v4.25.12 github.com/spaolacci/murmur3 v1.1.0 - github.com/spf13/afero v1.10.0 - github.com/spf13/cobra v1.10.1 + github.com/spf13/afero v1.15.0 + github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.10 github.com/stretchr/testify v1.11.1 github.com/supranational/blst v0.3.16-0.20250831170142-f48500c1fdbe @@ -133,14 +133,36 @@ require ( ) require ( + 4d63.com/gocheckcompilerdirectives v1.3.0 // indirect + 4d63.com/gochecknoglobals v0.2.2 // indirect cel.dev/expr v0.24.0 // indirect + codeberg.org/chavacava/garif v0.2.0 // indirect + codeberg.org/polyfloyd/go-errorlint v1.9.0 // indirect + dev.gaijin.team/go/exhaustruct/v4 v4.0.0 // indirect + dev.gaijin.team/go/golib v0.6.0 // indirect + github.com/4meepo/tagalign v1.4.3 // indirect + github.com/Abirdcfly/dupword v0.1.7 // indirect + github.com/AdminBenni/iota-mixing v1.0.0 // indirect + github.com/AlwxSin/noinlineerr v1.0.5 // indirect + github.com/Antonboom/errname v1.1.1 // indirect + github.com/Antonboom/nilnil v1.1.1 // indirect + github.com/Antonboom/testifylint v1.6.4 // indirect + github.com/BurntSushi/toml v1.6.0 // indirect + github.com/Djarvur/go-err113 v0.1.1 // indirect github.com/Masterminds/goutils v1.1.1 // indirect - github.com/Masterminds/semver/v3 v3.2.0 // indirect + github.com/Masterminds/semver/v3 v3.4.0 // indirect + github.com/MirrexOne/unqueryvet v1.4.0 // indirect + github.com/OpenPeeDeeP/depguard/v2 v2.2.1 // indirect github.com/RoaringBitmap/roaring v1.9.4 // indirect github.com/agnivade/levenshtein v1.2.1 // indirect - github.com/alecthomas/assert/v2 v2.8.1 // indirect github.com/alecthomas/atomic v0.1.0-alpha2 // indirect - github.com/alecthomas/repr v0.4.0 // indirect + github.com/alecthomas/chroma/v2 v2.21.1 // indirect + github.com/alecthomas/go-check-sumtype v0.3.1 // indirect + github.com/alexkohler/nakedret/v2 v2.0.6 // indirect + github.com/alexkohler/prealloc v1.0.1 // indirect + github.com/alfatraining/structtag v1.0.0 // indirect + github.com/alingse/asasalint v0.0.11 // indirect + github.com/alingse/nilnesserr v0.2.0 // indirect github.com/anacrolix/btree v0.0.0-20251201064447-d86c3fa41bd8 // indirect github.com/anacrolix/chansync v0.7.0 // indirect github.com/anacrolix/dht/v2 v2.23.0 // indirect @@ -152,6 +174,8 @@ require ( github.com/anacrolix/upnp v0.1.4 // indirect github.com/anacrolix/utp v0.1.0 // indirect github.com/antlr4-go/antlr/v4 v4.13.0 // indirect + github.com/ashanbrown/forbidigo/v2 v2.3.0 // indirect + github.com/ashanbrown/makezero/v2 v2.1.0 // indirect github.com/atotto/clipboard v0.1.4 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/bahlo/generic-list-go v0.2.0 // indirect @@ -159,31 +183,55 @@ require ( github.com/benbjohnson/immutable v0.4.1-0.20221220213129-8932b999621d // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.24.2 // indirect + github.com/bkielbasa/cyclop v1.2.3 // indirect + github.com/blizzy78/varnamelen v0.8.0 // indirect + github.com/bombsimon/wsl/v4 v4.7.0 // indirect + github.com/bombsimon/wsl/v5 v5.3.0 // indirect github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 // indirect + github.com/breml/bidichk v0.3.3 // indirect + github.com/breml/errchkjson v0.4.1 // indirect github.com/buger/jsonparser v1.1.1 // indirect + github.com/butuzov/ireturn v0.4.0 // indirect + github.com/butuzov/mirror v1.3.0 // indirect + github.com/catenacyber/perfsprint v0.10.1 // indirect + github.com/ccojocar/zxcvbn-go v1.0.4 // indirect github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/charithe/durationcheck v0.0.11 // indirect github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc // indirect github.com/charmbracelet/x/ansi v0.9.3 // indirect github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd // indirect github.com/charmbracelet/x/term v0.2.1 // indirect github.com/cilium/ebpf v0.16.0 // indirect + github.com/ckaznocha/intrange v0.3.1 // indirect github.com/containerd/cgroups v1.1.0 // indirect github.com/containerd/log v0.1.0 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect + github.com/curioswitch/go-reassign v0.3.0 // indirect + github.com/daixiang0/gci v0.13.7 // indirect + github.com/dave/dst v0.27.3 // indirect github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect - github.com/dlclark/regexp2 v1.7.0 // indirect + github.com/denis-tingaikin/go-header v0.5.0 // indirect + github.com/dlclark/regexp2 v1.11.5 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/ebitengine/purego v0.9.1 // indirect github.com/elastic/gosigar v0.14.3 // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect + github.com/ettle/strcase v0.2.0 // indirect + github.com/fatih/color v1.18.0 // indirect + github.com/fatih/structtag v1.2.0 // indirect + github.com/firefart/nonamedreturns v1.0.6 // indirect github.com/fjl/gencodec v0.1.0 // indirect github.com/flynn/noise v1.1.0 // indirect github.com/francoispqt/gojay v1.2.13 // indirect + github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/fzipp/gocyclo v0.6.0 // indirect github.com/garslo/gogen v0.0.0-20170307003452-d6ebae628c7c // indirect + github.com/ghostiam/protogetter v0.3.18 // indirect + github.com/go-critic/go-critic v0.14.3 // indirect github.com/go-llsqlite/adapter v0.0.0-20230927005056-7f5ce7f0c916 // indirect github.com/go-llsqlite/crawshaw v0.6.0 // indirect github.com/go-logr/logr v1.4.3 // indirect @@ -191,12 +239,42 @@ require ( github.com/go-ole/go-ole v1.3.0 // indirect github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect + github.com/go-toolsmith/astcast v1.1.0 // indirect + github.com/go-toolsmith/astcopy v1.1.0 // indirect + github.com/go-toolsmith/astequal v1.2.0 // indirect + github.com/go-toolsmith/astfmt v1.1.0 // indirect + github.com/go-toolsmith/astp v1.1.0 // indirect + github.com/go-toolsmith/strparse v1.1.0 // indirect + github.com/go-toolsmith/typep v1.1.0 // indirect + github.com/go-xmlfmt/xmlfmt v1.1.3 // indirect + github.com/gobwas/glob v0.2.3 // indirect github.com/godbus/dbus/v5 v5.1.0 // indirect + github.com/godoc-lint/godoc-lint v0.11.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect + github.com/golangci/asciicheck v0.5.0 // indirect + github.com/golangci/dupl v0.0.0-20250308024227-f665c8d69b32 // indirect + github.com/golangci/go-printf-func-name v0.1.1 // indirect + github.com/golangci/gofmt v0.0.0-20250106114630-d62b90e6713d // indirect + github.com/golangci/golangci-lint/v2 v2.8.0 // indirect + github.com/golangci/golines v0.14.0 // indirect + github.com/golangci/misspell v0.7.0 // indirect + github.com/golangci/plugin-module-register v0.1.2 // indirect + github.com/golangci/revgrep v0.8.0 // indirect + github.com/golangci/swaggoswag v0.0.0-20250504205917-77f2aca3143e // indirect + github.com/golangci/unconvert v0.0.0-20250410112200-a129a6e6413e // indirect github.com/google/gopacket v1.1.19 // indirect - github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e // indirect + github.com/google/pprof v0.0.0-20250820193118-f64d9cf942d6 // indirect + github.com/gordonklaus/ineffassign v0.2.0 // indirect + github.com/gostaticanalysis/analysisutil v0.7.1 // indirect + github.com/gostaticanalysis/comment v1.5.0 // indirect + github.com/gostaticanalysis/forcetypeassert v0.2.0 // indirect + github.com/gostaticanalysis/nilerr v0.1.2 // indirect github.com/grafana/pyroscope-go/godeltaprof v0.1.9 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-immutable-radix/v2 v2.1.0 // indirect + github.com/hashicorp/go-version v1.8.0 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/hexops/gotextdiff v1.0.3 // indirect github.com/ianlancetaylor/cgosymbolizer v0.0.0-20241129212102-9c50ad6b591e // indirect github.com/imdario/mergo v0.3.11 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect @@ -204,10 +282,27 @@ require ( github.com/ipfs/go-cid v0.4.1 // indirect github.com/ipfs/go-log/v2 v2.5.1 // indirect github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect + github.com/jgautheron/goconst v1.8.2 // indirect + github.com/jingyugao/rowserrcheck v1.1.1 // indirect + github.com/jjti/go-spancheck v0.6.5 // indirect + github.com/julz/importas v0.2.0 // indirect + github.com/karamaru-alpha/copyloopvar v1.2.2 // indirect + github.com/kisielk/errcheck v1.9.0 // indirect + github.com/kkHAIKE/contextcheck v1.1.6 // indirect github.com/klauspost/cpuid/v2 v2.2.8 // indirect github.com/koron/go-ssdp v0.0.4 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect + github.com/kulti/thelper v0.7.1 // indirect + github.com/kunwardeep/paralleltest v1.0.15 // indirect + github.com/lasiar/canonicalheader v1.1.2 // indirect + github.com/ldez/exptostd v0.4.5 // indirect + github.com/ldez/gomoddirectives v0.8.0 // indirect + github.com/ldez/grignotin v0.10.1 // indirect + github.com/ldez/structtags v0.6.1 // indirect + github.com/ldez/tagliatelle v0.7.2 // indirect + github.com/ldez/usetesting v0.5.0 // indirect + github.com/leonklingele/grouper v1.1.2 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect github.com/libp2p/go-flow-metrics v0.2.0 // indirect github.com/libp2p/go-libp2p-asn-util v0.4.1 // indirect @@ -219,19 +314,30 @@ require ( github.com/libp2p/go-yamux/v4 v4.0.1 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3 // indirect + github.com/macabu/inamedparam v0.2.0 // indirect + github.com/magiconair/properties v1.8.6 // indirect github.com/mailru/easyjson v0.7.7 // indirect + github.com/manuelarte/embeddedstructfieldcheck v0.4.0 // indirect + github.com/manuelarte/funcorder v0.5.0 // indirect + github.com/maratori/testableexamples v1.0.1 // indirect + github.com/maratori/testpackage v1.1.2 // indirect github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect + github.com/matoous/godox v1.1.0 // indirect github.com/mattn/go-localereader v0.0.1 // indirect github.com/mattn/go-runewidth v0.0.16 // indirect + github.com/mgechev/revive v1.13.0 // indirect github.com/miekg/dns v1.1.62 // indirect github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect github.com/minio/sha256-simd v1.0.1 // indirect github.com/mitchellh/copystructure v1.0.0 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mitchellh/reflectwalk v1.0.0 // indirect github.com/moby/sys/userns v0.1.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/moricho/tparallel v0.3.2 // indirect github.com/mr-tron/base58 v1.2.0 // indirect github.com/mschoch/smat v0.2.0 // indirect github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect @@ -247,8 +353,12 @@ require ( github.com/multiformats/go-multistream v0.6.0 // indirect github.com/multiformats/go-varint v0.0.7 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/nakabonne/nestif v0.3.1 // indirect github.com/ncruces/go-strftime v0.1.9 // indirect - github.com/onsi/ginkgo/v2 v2.20.2 // indirect + github.com/nishanths/exhaustive v0.12.0 // indirect + github.com/nishanths/predeclared v0.2.2 // indirect + github.com/nunnatsa/ginkgolinter v0.21.2 // indirect + github.com/onsi/ginkgo/v2 v2.27.2 // indirect github.com/opencontainers/runtime-spec v1.2.0 // indirect github.com/pion/datachannel v1.5.9 // indirect github.com/pion/dtls/v2 v2.2.12 // indirect @@ -279,46 +389,92 @@ require ( github.com/prometheus/common v0.66.1 // indirect github.com/prometheus/procfs v0.16.1 // indirect github.com/protolambda/ctxlock v0.1.0 // indirect + github.com/quasilyte/go-ruleguard v0.4.5 // indirect + github.com/quasilyte/gogrep v0.5.0 // indirect + github.com/quasilyte/regex/syntax v0.0.0-20210819130434-b3f0c404a727 // indirect + github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567 // indirect github.com/quic-go/qpack v0.5.1 // indirect github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66 // indirect + github.com/raeperd/recvcheck v0.2.0 // indirect github.com/raulk/go-watchdog v1.3.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/rogpeppe/go-internal v1.14.1 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/ryancurrah/gomodguard v1.4.1 // indirect + github.com/ryanrolds/sqlclosecheck v0.5.1 // indirect + github.com/sanposhiho/wastedassign/v2 v2.1.0 // indirect + github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect + github.com/sashamelentyev/interfacebloat v1.1.0 // indirect + github.com/sashamelentyev/usestdlibvars v1.29.0 // indirect + github.com/securego/gosec/v2 v2.22.11 // indirect github.com/shopspring/decimal v1.2.0 // indirect github.com/sirupsen/logrus v1.9.3 // indirect + github.com/sivchari/containedctx v1.0.3 // indirect + github.com/sonatard/noctx v0.4.0 // indirect github.com/sosodev/duration v1.3.1 // indirect + github.com/sourcegraph/go-diff v0.7.0 // indirect github.com/spf13/cast v1.7.1 // indirect + github.com/spf13/jwalterweatherman v1.1.0 // indirect + github.com/spf13/viper v1.12.0 // indirect + github.com/ssgreg/nlreturn/v2 v2.2.1 // indirect + github.com/stbenjam/no-sprintf-host-port v0.3.1 // indirect github.com/stoewer/go-strcase v1.3.0 // indirect + github.com/stretchr/objx v0.5.2 // indirect + github.com/subosito/gotenv v1.4.1 // indirect + github.com/tetafro/godot v1.5.4 // indirect + github.com/timakin/bodyclose v0.0.0-20241222091800-1db5c5ca4d67 // indirect + github.com/timonwong/loggercheck v0.11.0 // indirect github.com/tklauser/go-sysconf v0.3.16 // indirect github.com/tklauser/numcpus v0.11.0 // indirect + github.com/tomarrell/wrapcheck/v2 v2.12.0 // indirect + github.com/tommy-muehle/go-mnd/v2 v2.5.1 // indirect + github.com/ultraware/funlen v0.2.0 // indirect + github.com/ultraware/whitespace v0.2.0 // indirect github.com/urfave/cli/v3 v3.6.0 // indirect + github.com/uudashr/gocognit v1.2.0 // indirect + github.com/uudashr/iface v1.4.1 // indirect github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect github.com/wlynxg/anet v0.0.5 // indirect + github.com/xen0n/gosmopolitan v1.3.0 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect + github.com/yagipy/maintidx v1.0.0 // indirect + github.com/yeya24/promlinter v0.3.0 // indirect + github.com/ykadowak/zerologlint v0.1.5 // indirect github.com/yosida95/uritemplate/v3 v3.0.2 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect + gitlab.com/bosi/decorder v0.4.2 // indirect + go-simpler.org/musttag v0.14.0 // indirect + go-simpler.org/sloglint v0.11.1 // indirect + go.augendre.info/arangolint v0.3.1 // indirect + go.augendre.info/fatcontext v0.9.0 // indirect go.etcd.io/bbolt v1.3.6 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/otel v1.38.0 // indirect go.opentelemetry.io/otel/metric v1.38.0 // indirect go.opentelemetry.io/otel/trace v1.38.0 // indirect + go.uber.org/automaxprocs v1.6.0 // indirect go.uber.org/dig v1.18.0 // indirect go.uber.org/fx v1.23.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.yaml.in/yaml/v2 v2.4.2 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/exp/typeparams v0.0.0-20251023183803-a4bb9ffd2546 // indirect golang.org/x/mod v0.31.0 // indirect golang.org/x/telemetry v0.0.0-20251203150158-8fff8a5912fc // indirect google.golang.org/genproto/googleapis/api v0.0.0-20251111163417-95abcf5c77ba // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20251124214823-79d6a2a48846 // indirect google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.6.0 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect + honnef.co/go/tools v0.6.1 // indirect lukechampine.com/blake3 v1.3.0 // indirect modernc.org/libc v1.66.7 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.11.0 // indirect modernc.org/sqlite v1.21.1 // indirect + mvdan.cc/gofumpt v0.9.2 // indirect + mvdan.cc/unparam v0.0.0-20251027182757-5beb8c8f8f15 // indirect zombiezen.com/go/sqlite v0.13.1 // indirect ) @@ -328,6 +484,7 @@ tool ( github.com/erigontech/mdbx-go github.com/erigontech/mdbx-go/libmdbx github.com/fjl/gencodec + github.com/golangci/golangci-lint/v2/cmd/golangci-lint go.uber.org/mock/mockgen go.uber.org/mock/mockgen/model golang.org/x/tools/cmd/stringer diff --git a/go.sum b/go.sum index b04e3d57038..6fa7b676f17 100644 --- a/go.sum +++ b/go.sum @@ -1,48 +1,24 @@ +4d63.com/gocheckcompilerdirectives v1.3.0 h1:Ew5y5CtcAAQeTVKUVFrE7EwHMrTO6BggtEj8BZSjZ3A= +4d63.com/gocheckcompilerdirectives v1.3.0/go.mod h1:ofsJ4zx2QAuIP/NO/NAh1ig6R1Fb18/GI7RVMwz7kAY= +4d63.com/gochecknoglobals v0.2.2 h1:H1vdnwnMaZdQW/N+NrkT1SZMTBmcwHe9Vq8lJcYYTtU= +4d63.com/gochecknoglobals v0.2.2/go.mod h1:lLxwTQjL5eIesRbvnzIP3jZtG140FnTdz+AlMa+ogt0= cel.dev/expr v0.24.0 h1:56OvJKSH3hDGL0ml5uSxZmz3/3Pq4tJ+fb1unVLAFcY= cel.dev/expr v0.24.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.31.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.37.0/go.mod h1:TS1dMSSfndXH133OKGwekG838Om/cQT0BUHV3HcBgoo= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= -cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= -cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= -cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= -cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= -cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= -cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= -cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= -cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= +codeberg.org/chavacava/garif v0.2.0 h1:F0tVjhYbuOCnvNcU3YSpO6b3Waw6Bimy4K0mM8y6MfY= +codeberg.org/chavacava/garif v0.2.0/go.mod h1:P2BPbVbT4QcvLZrORc2T29szK3xEOlnl0GiPTJmEqBQ= +codeberg.org/polyfloyd/go-errorlint v1.9.0 h1:VkdEEmA1VBpH6ecQoMR4LdphVI3fA4RrCh2an7YmodI= +codeberg.org/polyfloyd/go-errorlint v1.9.0/go.mod h1:GPRRu2LzVijNn4YkrZYJfatQIdS+TrcK8rL5Xs24qw8= crawshaw.io/iox v0.0.0-20181124134642-c51c3df30797/go.mod h1:sXBiorCo8c46JlQV3oXPKINnZ8mcqnye1EkVkqsectk= crawshaw.io/sqlite v0.3.2/go.mod h1:igAO5JulrQ1DbdZdtVq48mnZUBAPOeFzer7VhDWNtW4= +dev.gaijin.team/go/exhaustruct/v4 v4.0.0 h1:873r7aNneqoBB3IaFIzhvt2RFYTuHgmMjoKfwODoI1Y= +dev.gaijin.team/go/exhaustruct/v4 v4.0.0/go.mod h1:aZ/k2o4Y05aMJtiux15x8iXaumE88YdiB0Ai4fXOzPI= +dev.gaijin.team/go/golib v0.6.0 h1:v6nnznFTs4bppib/NyU1PQxobwDHwCXXl15P7DV5Zgo= +dev.gaijin.team/go/golib v0.6.0/go.mod h1:uY1mShx8Z/aNHWDyAkZTkX+uCi5PdX7KsG1eDQa2AVE= dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBrvjyP0v+ecvNYvCpyZgu5/xkfAUhi6wJj28eUfSU= dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4= dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU= @@ -51,24 +27,46 @@ filippo.io/edwards25519 v1.0.0-rc.1/go.mod h1:N1IkdkCkiLB6tki+MYJoSx2JTY9NUlxZE7 gfx.cafe/util/go/generic v0.0.0-20230721185457-c559e86c829c h1:alCfDKmPC0EC0KGlZWrNF0hilVWBkzMz+aAYTJ/2hY4= gfx.cafe/util/go/generic v0.0.0-20230721185457-c559e86c829c/go.mod h1:WvSX4JsCRBuIXj0FRBFX9YLg+2SoL3w8Ww19uZO9yNE= git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= +github.com/4meepo/tagalign v1.4.3 h1:Bnu7jGWwbfpAie2vyl63Zup5KuRv21olsPIha53BJr8= +github.com/4meepo/tagalign v1.4.3/go.mod h1:00WwRjiuSbrRJnSVeGWPLp2epS5Q/l4UEy0apLLS37c= github.com/99designs/gqlgen v0.17.83 h1:LZOd4Of2snK5V22/ZWfBAPa3WoAZkBO70dKXM0ODHQk= github.com/99designs/gqlgen v0.17.83/go.mod h1:q6Lb64wknFqNFSbSUGzKRKupklvY/xgNr62g0GGWPB8= +github.com/Abirdcfly/dupword v0.1.7 h1:2j8sInznrje4I0CMisSL6ipEBkeJUJAmK1/lfoNGWrQ= +github.com/Abirdcfly/dupword v0.1.7/go.mod h1:K0DkBeOebJ4VyOICFdppB23Q0YMOgVafM0zYW0n9lF4= +github.com/AdminBenni/iota-mixing v1.0.0 h1:Os6lpjG2dp/AE5fYBPAA1zfa2qMdCAWwPMCgpwKq7wo= +github.com/AdminBenni/iota-mixing v1.0.0/go.mod h1:i4+tpAaB+qMVIV9OK3m4/DAynOd5bQFaOu+2AhtBCNY= +github.com/AlwxSin/noinlineerr v1.0.5 h1:RUjt63wk1AYWTXtVXbSqemlbVTb23JOSRiNsshj7TbY= +github.com/AlwxSin/noinlineerr v1.0.5/go.mod h1:+QgkkoYrMH7RHvcdxdlI7vYYEdgeoFOVjU9sUhw/rQc= +github.com/Antonboom/errname v1.1.1 h1:bllB7mlIbTVzO9jmSWVWLjxTEbGBVQ1Ff/ClQgtPw9Q= +github.com/Antonboom/errname v1.1.1/go.mod h1:gjhe24xoxXp0ScLtHzjiXp0Exi1RFLKJb0bVBtWKCWQ= +github.com/Antonboom/nilnil v1.1.1 h1:9Mdr6BYd8WHCDngQnNVV0b554xyisFioEKi30sksufQ= +github.com/Antonboom/nilnil v1.1.1/go.mod h1:yCyAmSw3doopbOWhJlVci+HuyNRuHJKIv6V2oYQa8II= +github.com/Antonboom/testifylint v1.6.4 h1:gs9fUEy+egzxkEbq9P4cpcMB6/G0DYdMeiFS87UiqmQ= +github.com/Antonboom/testifylint v1.6.4/go.mod h1:YO33FROXX2OoUfwjz8g+gUxQXio5i9qpVy7nXGbxDD4= github.com/AskAlexSharov/bloomfilter/v2 v2.0.9 h1:BuZqNjRlYmcXJIsI7nrIkejYMz9mgFi7ZsNFCbSPpaI= github.com/AskAlexSharov/bloomfilter/v2 v2.0.9/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk= +github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= +github.com/Djarvur/go-err113 v0.1.1 h1:eHfopDqXRwAi+YmCUas75ZE0+hoBHJ2GQNLYRSxao4g= +github.com/Djarvur/go-err113 v0.1.1/go.mod h1:IaWJdYFLg76t2ihfflPZnM1LIQszWOsFDh2hhhAVF6k= github.com/FastFilter/xorfilter v0.2.1 h1:lbdeLG9BdpquK64ZsleBS8B4xO/QW1IM0gMzF7KaBKc= github.com/FastFilter/xorfilter v0.2.1/go.mod h1:aumvdkhscz6YBZF9ZA/6O4fIoNod4YR50kIVGGZ7l9I= github.com/Giulio2002/zero-alloc-go-eth-kzg v0.0.0-20260105034637-43cb6f34f8e0 h1:wWX3lg8PLXojtYeie+Eoug06T1td4fBKeh2/s87AtxY= github.com/Giulio2002/zero-alloc-go-eth-kzg v0.0.0-20260105034637-43cb6f34f8e0/go.mod h1:J9/u5sWfznSObptgfa92Jq8rTswn6ahQWEuiLHOjCUI= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= -github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g= github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= +github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= +github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj9n6YA= github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM= +github.com/MirrexOne/unqueryvet v1.4.0 h1:6KAkqqW2KUnkl9Z0VuTphC3IXRPoFqEkJEtyxxHj5eQ= +github.com/MirrexOne/unqueryvet v1.4.0/go.mod h1:IWwCwMQlSWjAIteW0t+28Q5vouyktfujzYznSIWiuOg= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/OpenPeeDeeP/depguard/v2 v2.2.1 h1:vckeWVESWp6Qog7UZSARNqfu/cZqvki8zsuj3piCMx4= +github.com/OpenPeeDeeP/depguard/v2 v2.2.1/go.mod h1:q4DKzC4UcVaAvcfd41CZh0PWpGgzrVxUYBlgKNGquUo= github.com/PuerkitoBio/goquery v1.10.3 h1:pFYcNSqHxBD06Fpj/KsbStFRsgRATgnf3LeXiUkhzPo= github.com/PuerkitoBio/goquery v1.10.3/go.mod h1:tMUX0zDMHXYlAQk6p35XxQMqMweEKB7iK7iLNd4RH4Y= github.com/RoaringBitmap/roaring v0.4.7/go.mod h1:8khRDP4HmeXns4xIj9oGrKSz7XTQiJx2zgh7AcNke4w= @@ -82,18 +80,32 @@ github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWX github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/agnivade/levenshtein v1.2.1 h1:EHBY3UOn1gwdy/VbFwgo4cxecRznFk7fKWN1KOX7eoM= github.com/agnivade/levenshtein v1.2.1/go.mod h1:QVVI16kDrtSuwcpd0p1+xMC6Z/VfhtCyDIjcwga4/DU= -github.com/alecthomas/assert/v2 v2.8.1 h1:YCxnYR6jjpfnEK5AK5SysALKdUEBPGH4Y7As6tBnDw0= -github.com/alecthomas/assert/v2 v2.8.1/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= +github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0= +github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= github.com/alecthomas/atomic v0.1.0-alpha2 h1:dqwXmax66gXvHhsOS4pGPZKqYOlTkapELkLb3MNdlH8= github.com/alecthomas/atomic v0.1.0-alpha2/go.mod h1:zD6QGEyw49HIq19caJDc2NMXAy8rNi9ROrxtMXATfyI= +github.com/alecthomas/chroma/v2 v2.21.1 h1:FaSDrp6N+3pphkNKU6HPCiYLgm8dbe5UXIXcoBhZSWA= +github.com/alecthomas/chroma/v2 v2.21.1/go.mod h1:NqVhfBR0lte5Ouh3DcthuUCTUpDC9cxBOfyMbMQPs3o= +github.com/alecthomas/go-check-sumtype v0.3.1 h1:u9aUvbGINJxLVXiFvHUlPEaD7VDULsrxJb4Aq31NLkU= +github.com/alecthomas/go-check-sumtype v0.3.1/go.mod h1:A8TSiN3UPRw3laIgWEUOHHLPa6/r9MtoigdlP5h3K/E= github.com/alecthomas/kong v0.8.1 h1:acZdn3m4lLRobeh3Zi2S2EpnXTd1mOL6U7xVml+vfkY= github.com/alecthomas/kong v0.8.1/go.mod h1:n1iCIO2xS46oE8ZfYCNDqdR0b0wZNrXAIAqro/2132U= -github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc= -github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= +github.com/alecthomas/repr v0.5.2 h1:SU73FTI9D1P5UNtvseffFSGmdNci/O6RsqzeXJtP0Qs= +github.com/alecthomas/repr v0.5.2/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alexkohler/nakedret/v2 v2.0.6 h1:ME3Qef1/KIKr3kWX3nti3hhgNxw6aqN5pZmQiFSsuzQ= +github.com/alexkohler/nakedret/v2 v2.0.6/go.mod h1:l3RKju/IzOMQHmsEvXwkqMDzHHvurNQfAgE1eVmT40Q= +github.com/alexkohler/prealloc v1.0.1 h1:A9P1haqowqUxWvU9nk6tQ7YktXIHf+LQM9wPRhuteEE= +github.com/alexkohler/prealloc v1.0.1/go.mod h1:fT39Jge3bQrfA7nPMDngUfvUbQGQeJyGQnR+913SCig= +github.com/alfatraining/structtag v1.0.0 h1:2qmcUqNcCoyVJ0up879K614L9PazjBSFruTB0GOFjCc= +github.com/alfatraining/structtag v1.0.0/go.mod h1:p3Xi5SwzTi+Ryj64DqjLWz7XurHxbGsq6y3ubePJPus= +github.com/alingse/asasalint v0.0.11 h1:SFwnQXJ49Kx/1GghOFz1XGqHYKp21Kq1nHad/0WQRnw= +github.com/alingse/asasalint v0.0.11/go.mod h1:nCaoMhw7a9kSJObvQyVzNTPBDbNpdocqrSP7t/cW5+I= +github.com/alingse/nilnesserr v0.2.0 h1:raLem5KG7EFVb4UIDAXgrv3N2JIaffeKNtcEXkEWd/w= +github.com/alingse/nilnesserr v0.2.0/go.mod h1:1xJPrXonEtX7wyTq8Dytns5P2hNzoWymVUIaKm4HNFg= github.com/anacrolix/btree v0.0.0-20251201064447-d86c3fa41bd8 h1:c02PsmoaChabVqAFm7pqPI1UIkDdDAjUaWa6ZmfxybQ= github.com/anacrolix/btree v0.0.0-20251201064447-d86c3fa41bd8/go.mod h1:7stWJ39LeusmMI8mjJuhFNRqep//vx0AsaySRoK9or0= github.com/anacrolix/chansync v0.7.0 h1:wgwxbsJRmOqNjil4INpxHrDp4rlqQhECxR8/WBP4Et0= @@ -158,6 +170,10 @@ github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9 github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Ig+0+Ap1h4unLjW6YQJpKZVmUzxsD4E/Q= github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE= +github.com/ashanbrown/forbidigo/v2 v2.3.0 h1:OZZDOchCgsX5gvToVtEBoV2UWbFfI6RKQTir2UZzSxo= +github.com/ashanbrown/forbidigo/v2 v2.3.0/go.mod h1:5p6VmsG5/1xx3E785W9fouMxIOkvY2rRV9nMdWadd6c= +github.com/ashanbrown/makezero/v2 v2.1.0 h1:snuKYMbqosNokUKm+R6/+vOPs8yVAi46La7Ck6QYSaE= +github.com/ashanbrown/makezero/v2 v2.1.0/go.mod h1:aEGT/9q3S8DHeE57C88z2a6xydvgx8J5hgXIGWgo0MY= github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4= github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= @@ -182,24 +198,45 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bits-and-blooms/bitset v1.12.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/bits-and-blooms/bitset v1.24.2 h1:M7/NzVbsytmtfHbumG+K2bremQPMJuqv1JD3vOaFxp0= github.com/bits-and-blooms/bitset v1.24.2/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= +github.com/bkielbasa/cyclop v1.2.3 h1:faIVMIGDIANuGPWH031CZJTi2ymOQBULs9H21HSMa5w= +github.com/bkielbasa/cyclop v1.2.3/go.mod h1:kHTwA9Q0uZqOADdupvcFJQtp/ksSnytRMe8ztxG8Fuo= +github.com/blizzy78/varnamelen v0.8.0 h1:oqSblyuQvFsW1hbBHh1zfwrKe3kcSj0rnXkKzsQ089M= +github.com/blizzy78/varnamelen v0.8.0/go.mod h1:V9TzQZ4fLJ1DSrjVDfl89H7aMnTvKkApdHeyESmyR7k= +github.com/bombsimon/wsl/v4 v4.7.0 h1:1Ilm9JBPRczjyUs6hvOPKvd7VL1Q++PL8M0SXBDf+jQ= +github.com/bombsimon/wsl/v4 v4.7.0/go.mod h1:uV/+6BkffuzSAVYD+yGyld1AChO7/EuLrCF/8xTiapg= +github.com/bombsimon/wsl/v5 v5.3.0 h1:nZWREJFL6U3vgW/B1lfDOigl+tEF6qgs6dGGbFeR0UM= +github.com/bombsimon/wsl/v5 v5.3.0/go.mod h1:Gp8lD04z27wm3FANIUPZycXp+8huVsn0oxc+n4qfV9I= github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= github.com/bradfitz/iter v0.0.0-20140124041915-454541ec3da2/go.mod h1:PyRFw1Lt2wKX4ZVSQ2mk+PeDa1rxyObEDlApuIsUKuo= github.com/bradfitz/iter v0.0.0-20190303215204-33e6a9893b0c/go.mod h1:PyRFw1Lt2wKX4ZVSQ2mk+PeDa1rxyObEDlApuIsUKuo= github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 h1:GKTyiRCL6zVf5wWaqKnf+7Qs6GbEPfd4iMOitWzXJx8= github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8/go.mod h1:spo1JLcs67NmW1aVLEgtA8Yy1elc+X8y5SRW1sFW4Og= +github.com/breml/bidichk v0.3.3 h1:WSM67ztRusf1sMoqH6/c4OBCUlRVTKq+CbSeo0R17sE= +github.com/breml/bidichk v0.3.3/go.mod h1:ISbsut8OnjB367j5NseXEGGgO/th206dVa427kR8YTE= +github.com/breml/errchkjson v0.4.1 h1:keFSS8D7A2T0haP9kzZTi7o26r7kE3vymjZNeNDRDwg= +github.com/breml/errchkjson v0.4.1/go.mod h1:a23OvR6Qvcl7DG/Z4o0el6BRAjKnaReoPQFciAl9U3s= github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= +github.com/butuzov/ireturn v0.4.0 h1:+s76bF/PfeKEdbG8b54aCocxXmi0wvYdOVsWxVO7n8E= +github.com/butuzov/ireturn v0.4.0/go.mod h1:ghI0FrCmap8pDWZwfPisFD1vEc56VKH4NpQUxDHta70= +github.com/butuzov/mirror v1.3.0 h1:HdWCXzmwlQHdVhwvsfBb2Au0r3HyINry3bDWLYXiKoc= +github.com/butuzov/mirror v1.3.0/go.mod h1:AEij0Z8YMALaq4yQj9CPPVYOyJQyiexpQEQgihajRfI= github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500 h1:6lhrsTEnloDPXyeZBvSYvQf8u86jbKehZPVDDlkgDl4= github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500/go.mod h1:S/7n9copUssQ56c7aAgHqftWO4LTf4xY6CGWt8Bc+3M= +github.com/catenacyber/perfsprint v0.10.1 h1:u7Riei30bk46XsG8nknMhKLXG9BcXz3+3tl/WpKm0PQ= +github.com/catenacyber/perfsprint v0.10.1/go.mod h1:DJTGsi/Zufpuus6XPGJyKOTMELe347o6akPvWG9Zcsc= +github.com/ccojocar/zxcvbn-go v1.0.4 h1:FWnCIRMXPj43ukfX000kvBZvV6raSxakYr1nzyNrUcc= +github.com/ccojocar/zxcvbn-go v1.0.4/go.mod h1:3GxGX+rHmueTUMvm5ium7irpyjmm7ikxYFOSJB21Das= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/charithe/durationcheck v0.0.11 h1:g1/EX1eIiKS57NTWsYtHDZ/APfeXKhye1DidBcABctk= +github.com/charithe/durationcheck v0.0.11/go.mod h1:x5iZaixRNl8ctbM+3B2RrPG5t856TxRyVQEnbIEM2X4= github.com/charmbracelet/bubbles v0.21.0 h1:9TdC97SdRVg/1aaXNVWfFH3nnLAwOXr8Fn6u6mfQdFs= github.com/charmbracelet/bubbles v0.21.0/go.mod h1:HF+v6QUR4HkEpz62dx7ym2xc71/KBHg+zKwJtMw+qtg= github.com/charmbracelet/bubbletea v1.3.6 h1:VkHIxPJQeDt0aFJIsVxw8BQdh/F/L2KKZGsK6et5taU= @@ -219,22 +256,18 @@ github.com/charmbracelet/x/term v0.2.1/go.mod h1:oQ4enTYFV7QN4m0i9mzHrViD7TQKvNE github.com/chromedp/cdproto v0.0.0-20230802225258-3cf4e6d46a89/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs= github.com/chromedp/chromedp v0.9.2/go.mod h1:LkSXJKONWTCHAfQasKFUZI+mxqS4tZqhmtGzzhLsnLs= github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moAV0xufSww= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/logex v1.2.0/go.mod h1:9+9sk7u7pGNWYMkh0hdiL++6OeibzJccyQU4p4MedaY= github.com/chzyer/logex v1.2.1/go.mod h1:JLbx6lG2kDbNRFnfkgvh4eRJRPX1QCoOIWomwysCBrQ= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/readline v1.5.0/go.mod h1:x22KAscuvRqlLoK9CsoYsmxoXZMMFVyOl86cAH8qUic= github.com/chzyer/readline v1.5.1/go.mod h1:Eh+b79XXUwfKfcPLepksvw2tcLE/Ct21YObkaSkeBlk= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/chzyer/test v0.0.0-20210722231415-061457976a23/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/chzyer/test v1.0.0/go.mod h1:2JlltgoNkt4TW/z9V/IzDdFaMTM2JPIi26O1pF38GC8= github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs= github.com/cilium/ebpf v0.16.0 h1:+BiEnHL6Z7lXnlGUsXQPPAE7+kenAd4ES8MQ5min0Ok= github.com/cilium/ebpf v0.16.0/go.mod h1:L7u2Blt2jMM/vLAVgjxluxtBKlz3/GWjB0dMOEngfwE= +github.com/ckaznocha/intrange v0.3.1 h1:j1onQyXvHUsPWujDH6WIjhyH26gkRt/txNlV7LspvJs= +github.com/ckaznocha/intrange v0.3.1/go.mod h1:QVepyz1AkUoFQkpEqksSYpNpUo3c5W7nWh/s6SHIJJk= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/consensys/gnark-crypto v0.19.1 h1:FWO1JDs7A2OajswzwMG7f8l2Zrxc/yOkxSTByKTc3O0= github.com/consensys/gnark-crypto v0.19.1/go.mod h1:rT23F0XSZqE0mUA0+pRtnL56IbPxs6gp4CeRsBk4XS0= github.com/containerd/cgroups v0.0.0-20201119153540-4cbc285b3327/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE= @@ -254,6 +287,14 @@ github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6N github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3sHPnBo= github.com/cpuguy83/go-md2man/v2 v2.0.7/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/curioswitch/go-reassign v0.3.0 h1:dh3kpQHuADL3cobV/sSGETA8DOv457dwl+fbBAhrQPs= +github.com/curioswitch/go-reassign v0.3.0/go.mod h1:nApPCCTtqLJN/s8HfItCcKV0jIPwluBOvZP+dsJGA88= +github.com/daixiang0/gci v0.13.7 h1:+0bG5eK9vlI08J+J/NWGbWPTNiXPG4WhNLJOkSxWITQ= +github.com/daixiang0/gci v0.13.7/go.mod h1:812WVN6JLFY9S6Tv76twqmNqevN0pa3SX3nih0brVzQ= +github.com/dave/dst v0.27.3 h1:P1HPoMza3cMEquVf9kKy8yXsFirry4zEnWOdYPOoIzY= +github.com/dave/dst v0.27.3/go.mod h1:jHh6EOibnHgcUW3WjKHisiooEkYwqpHLBSX1iOBhEyc= +github.com/dave/jennifer v1.7.1 h1:B4jJJDHelWcDhlRQxWeo0Npa/pYKBLrirAQoTN45txo= +github.com/dave/jennifer v1.7.1/go.mod h1:nXbxhEmQfOZhWml3D1cDK5M1FLnMSozpbFN/m3RmGZc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -266,11 +307,14 @@ github.com/decred/dcrd/crypto/blake256 v1.0.1 h1:7PltbUIQB7u/FfZ39+DGa/ShuMyJ5il github.com/decred/dcrd/crypto/blake256 v1.0.1/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 h1:rpfIENRNNilwHwZeG5+P150SMrnNEcHYvcCuK6dPZSg= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= +github.com/denis-tingaikin/go-header v0.5.0 h1:SRdnP5ZKvcO9KKRP1KJrhFR3RrlGuD+42t4429eC9k8= +github.com/denis-tingaikin/go-header v0.5.0/go.mod h1:mMenU5bWrok6Wl2UsZjy+1okegmwQ3UgWl4V1D8gjlY= github.com/dgryski/trifles v0.0.0-20230903005119-f50d829f2e54 h1:SG7nF6SRlWhcT7cNTs5R6Hk4V2lcmLz2NsG2VnInyNo= github.com/dgryski/trifles v0.0.0-20230903005119-f50d829f2e54/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA= github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= -github.com/dlclark/regexp2 v1.7.0 h1:7lJfhqlPssTb1WQx4yvTHN0uElPEv52sbaECrAQxjAo= github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ= +github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= @@ -298,12 +342,6 @@ github.com/elastic/gosigar v0.14.3 h1:xwkKwPia+hSfg9GqrCUKYdId102m9qTJIIr7egmK/u github.com/elastic/gosigar v0.14.3/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= -github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/erigontech/erigon-snapshot v1.3.1-0.20260105114333-2f59a10db72b h1:xXkx2s/uCiBujVSBih1esDtwH2NKfP/R7F7+Om13iv8= github.com/erigontech/erigon-snapshot v1.3.1-0.20260105114333-2f59a10db72b/go.mod h1:ooHlCl+eEYzebiPu+FP6Q6SpPUeMADn8Jxabv3IKb9M= github.com/erigontech/erigonwatch v0.0.0-20240718131902-b6576bde1116 h1:KCFa2uXEfZoBjV4buzjWmCmoqVLXiGCq0ZmQ2OjeRvQ= @@ -320,10 +358,16 @@ github.com/ethereum/go-bigmodexpfix v0.0.0-20250911101455-f9e208c548ab h1:rvv6MJ github.com/ethereum/go-bigmodexpfix v0.0.0-20250911101455-f9e208c548ab/go.mod h1:IuLm4IsPipXKF7CW5Lzf68PIbZ5yl7FFd74l/E0o9A8= github.com/ethereum/go-ethereum v1.16.8 h1:LLLfkZWijhR5m6yrAXbdlTeXoqontH+Ga2f9igY7law= github.com/ethereum/go-ethereum v1.16.8/go.mod h1:Fs6QebQbavneQTYcA39PEKv2+zIjX7rPUZ14DER46wk= -github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= -github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= +github.com/ettle/strcase v0.2.0 h1:fGNiVF21fHXpX1niBgk0aROov1LagYsOwV/xqKDKR/Q= +github.com/ettle/strcase v0.2.0/go.mod h1:DajmHElDSaX76ITe3/VHVyMin4LWSJN5Z909Wp+ED1A= +github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= +github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= +github.com/fatih/structtag v1.2.0 h1:/OdNE99OxoI/PqaW/SuSK9uxxT3f/tcSZgon/ssNSx4= +github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= github.com/felixge/fgprof v0.9.5 h1:8+vR6yu2vvSKn08urWyEuxx75NWPEvybbkBirEpsbVY= github.com/felixge/fgprof v0.9.5/go.mod h1:yKl+ERSa++RYOs32d8K6WEXCB4uXdLls4ZaZPpayhMM= +github.com/firefart/nonamedreturns v1.0.6 h1:vmiBcKV/3EqKY3ZiPxCINmpS431OcE1S47AQUwhrg8E= +github.com/firefart/nonamedreturns v1.0.6/go.mod h1:R8NisJnSIpvPWheCq0mNRXJok6D8h7fagJTF8EMEwCo= github.com/fjl/gencodec v0.1.0 h1:B3K0xPfc52cw52BBgUbSPxYo+HlLfAgWMVKRWXUXBcs= github.com/fjl/gencodec v0.1.0/go.mod h1:Um1dFHPONZGTHog1qD1NaWjXJW/SPB38wPv0O8uZ2fI= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= @@ -337,9 +381,19 @@ github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7z github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/fzipp/gocyclo v0.6.0 h1:lsblElZG7d3ALtGMx9fmxeTKZaLLpU8mET09yN4BBLo= +github.com/fzipp/gocyclo v0.6.0/go.mod h1:rXPyn8fnlpa0R2csP/31uerbiVBugk5whMdlyaLkLoA= github.com/garslo/gogen v0.0.0-20170307003452-d6ebae628c7c h1:uYNKzPntb8c6DKvP9EfrBjkLkU7pM4lM+uuHSIa8UtU= github.com/garslo/gogen v0.0.0-20170307003452-d6ebae628c7c/go.mod h1:Q0X6pkwTILDlzrGEckF6HKjXe48EgsY/l7K7vhY4MW8= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/ghostiam/protogetter v0.3.18 h1:yEpghRGtP9PjKvVXtEzGpYfQj1Wl/ZehAfU6fr62Lfo= +github.com/ghostiam/protogetter v0.3.18/go.mod h1:FjIu5Yfs6FT391m+Fjp3fbAYJ6rkL/J6ySpZBfnODuI= +github.com/gkampitakis/ciinfo v0.3.2 h1:JcuOPk8ZU7nZQjdUhctuhQofk7BGHuIy0c9Ez8BNhXs= +github.com/gkampitakis/ciinfo v0.3.2/go.mod h1:1NIwaOcFChN4fa/B0hEBdAb6npDlFL8Bwx4dfRLRqAo= +github.com/gkampitakis/go-diff v1.3.2 h1:Qyn0J9XJSDTgnsgHRdz9Zp24RaJeKMUHg2+PDZZdC4M= +github.com/gkampitakis/go-diff v1.3.2/go.mod h1:LLgOrpqleQe26cte8s36HTWcTmMEur6OPYerdAAS9tk= +github.com/gkampitakis/go-snaps v0.5.15 h1:amyJrvM1D33cPHwVrjo9jQxX8g/7E2wYdZ+01KS3zGE= +github.com/gkampitakis/go-snaps v0.5.15/go.mod h1:HNpx/9GoKisdhw9AFOBT1N7DBs9DiHo/hGheFGBZ+mc= github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= @@ -351,12 +405,11 @@ github.com/go-chi/chi/v5 v5.2.3 h1:WQIt9uxdsAbgIYgid+BpYc+liqQZGMHRaUwp0JUcvdE= github.com/go-chi/chi/v5 v5.2.3/go.mod h1:L2yAIGWB3H+phAw1NxKwWM+7eUH/lU8pOMm5hHcoops= github.com/go-chi/cors v1.2.2 h1:Jmey33TE+b+rB7fT8MUy1u0I4L+NARQlK6LhzKPSyQE= github.com/go-chi/cors v1.2.2/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58= +github.com/go-critic/go-critic v0.14.3 h1:5R1qH2iFeo4I/RJU8vTezdqs08Egi4u5p6vOESA0pog= +github.com/go-critic/go-critic v0.14.3/go.mod h1:xwntfW6SYAd7h1OqDzmN6hBX/JxsEKl5up/Y2bsxgVQ= github.com/go-echarts/go-echarts/v2 v2.6.7 h1:J9Y6/vVn06BBSGeoowPbdUWsxzHktwqF1uwOuSEUyTY= github.com/go-echarts/go-echarts/v2 v2.6.7/go.mod h1:Z+spPygZRIEyqod69r0WMnkN5RV3MwhYDtw601w3G8w= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-llsqlite/adapter v0.0.0-20230927005056-7f5ce7f0c916 h1:OyQmpAN302wAopDgwVjgs2HkFawP9ahIEqkUYz7V7CA= @@ -385,15 +438,42 @@ github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZ github.com/go-test/deep v1.1.0/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= github.com/go-test/deep v1.1.1 h1:0r/53hagsehfO4bzD2Pgr/+RgHqhmf+k1Bpse2cTu1U= github.com/go-test/deep v1.1.1/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= +github.com/go-toolsmith/astcast v1.1.0 h1:+JN9xZV1A+Re+95pgnMgDboWNVnIMMQXwfBwLRPgSC8= +github.com/go-toolsmith/astcast v1.1.0/go.mod h1:qdcuFWeGGS2xX5bLM/c3U9lewg7+Zu4mr+xPwZIB4ZU= +github.com/go-toolsmith/astcopy v1.1.0 h1:YGwBN0WM+ekI/6SS6+52zLDEf8Yvp3n2seZITCUBt5s= +github.com/go-toolsmith/astcopy v1.1.0/go.mod h1:hXM6gan18VA1T/daUEHCFcYiW8Ai1tIwIzHY6srfEAw= +github.com/go-toolsmith/astequal v1.0.3/go.mod h1:9Ai4UglvtR+4up+bAD4+hCj7iTo4m/OXVTSLnCyTAx4= +github.com/go-toolsmith/astequal v1.1.0/go.mod h1:sedf7VIdCL22LD8qIvv7Nn9MuWJruQA/ysswh64lffQ= +github.com/go-toolsmith/astequal v1.2.0 h1:3Fs3CYZ1k9Vo4FzFhwwewC3CHISHDnVUPC4x0bI2+Cw= +github.com/go-toolsmith/astequal v1.2.0/go.mod h1:c8NZ3+kSFtFY/8lPso4v8LuJjdJiUFVnSuU3s0qrrDY= +github.com/go-toolsmith/astfmt v1.1.0 h1:iJVPDPp6/7AaeLJEruMsBUlOYCmvg0MoCfJprsOmcco= +github.com/go-toolsmith/astfmt v1.1.0/go.mod h1:OrcLlRwu0CuiIBp/8b5PYF9ktGVZUjlNMV634mhwuQ4= +github.com/go-toolsmith/astp v1.1.0 h1:dXPuCl6u2llURjdPLLDxJeZInAeZ0/eZwFJmqZMnpQA= +github.com/go-toolsmith/astp v1.1.0/go.mod h1:0T1xFGz9hicKs8Z5MfAqSUitoUYS30pDMsRVIDHs8CA= +github.com/go-toolsmith/pkgload v1.2.2 h1:0CtmHq/02QhxcF7E9N5LIFcYFsMR5rdovfqTtRKkgIk= +github.com/go-toolsmith/pkgload v1.2.2/go.mod h1:R2hxLNRKuAsiXCo2i5J6ZQPhnPMOVtU+f0arbFPWCus= +github.com/go-toolsmith/strparse v1.0.0/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8= +github.com/go-toolsmith/strparse v1.1.0 h1:GAioeZUK9TGxnLS+qfdqNbA4z0SSm5zVNtCQiyP2Bvw= +github.com/go-toolsmith/strparse v1.1.0/go.mod h1:7ksGy58fsaQkGQlY8WVoBFNyEPMGuJin1rfoPS4lBSQ= +github.com/go-toolsmith/typep v1.1.0 h1:fIRYDyF+JywLfqzyhdiHzRop/GQDxxNhLGQ6gFUNHus= +github.com/go-toolsmith/typep v1.1.0/go.mod h1:fVIw+7zjdsMxDA3ITWnH1yOiw1rnTQKCsF/sk2H/qig= github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/go-xmlfmt/xmlfmt v1.1.3 h1:t8Ey3Uy7jDSEisW2K3somuMKIpzktkWptA0iFCnRUWY= +github.com/go-xmlfmt/xmlfmt v1.1.3/go.mod h1:aUCEOzzezBEjDBbFBoSiya/gduyIiWYRP6CnSFIV8AM= +github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= +github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= github.com/gobwas/ws v1.2.1/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/KY= +github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw= +github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/godoc-lint/godoc-lint v0.11.1 h1:z9as8Qjiy6miRIa3VRymTa+Gt2RLnGICVikcvlUVOaA= +github.com/godoc-lint/godoc-lint v0.11.1/go.mod h1:BAqayheFSuZrEAqCRxgw9MyvsM+S/hZwJbU1s/ejRj8= github.com/gofrs/flock v0.13.0 h1:95JolYOvGMqeH31+FC7D2+uULf6mG61mEZ/A8dRYMzw= github.com/gofrs/flock v0.13.0/go.mod h1:jxeyy9R1auM5S6JYDBhDt+E2TCo7DkratH4Pgi8P+Z0= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -405,36 +485,46 @@ github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXe github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golangci/asciicheck v0.5.0 h1:jczN/BorERZwK8oiFBOGvlGPknhvq0bjnysTj4nUfo0= +github.com/golangci/asciicheck v0.5.0/go.mod h1:5RMNAInbNFw2krqN6ibBxN/zfRFa9S6tA1nPdM0l8qQ= +github.com/golangci/dupl v0.0.0-20250308024227-f665c8d69b32 h1:WUvBfQL6EW/40l6OmeSBYQJNSif4O11+bmWEz+C7FYw= +github.com/golangci/dupl v0.0.0-20250308024227-f665c8d69b32/go.mod h1:NUw9Zr2Sy7+HxzdjIULge71wI6yEg1lWQr7Evcu8K0E= +github.com/golangci/go-printf-func-name v0.1.1 h1:hIYTFJqAGp1iwoIfsNTpoq1xZAarogrvjO9AfiW3B4U= +github.com/golangci/go-printf-func-name v0.1.1/go.mod h1:Es64MpWEZbh0UBtTAICOZiB+miW53w/K9Or/4QogJss= +github.com/golangci/gofmt v0.0.0-20250106114630-d62b90e6713d h1:viFft9sS/dxoYY0aiOTsLKO2aZQAPT4nlQCsimGcSGE= +github.com/golangci/gofmt v0.0.0-20250106114630-d62b90e6713d/go.mod h1:ivJ9QDg0XucIkmwhzCDsqcnxxlDStoTl89jDMIoNxKY= +github.com/golangci/golangci-lint/v2 v2.8.0 h1:wJnr3hJWY3eVzOUcfwbDc2qbi2RDEpvLmQeNFaPSNYA= +github.com/golangci/golangci-lint/v2 v2.8.0/go.mod h1:xl+HafQ9xoP8rzw0z5AwnO5kynxtb80e8u02Ej/47RI= +github.com/golangci/golines v0.14.0 h1:xt9d3RKBjhasA3qpoXs99J2xN2t6eBlpLHt0TrgyyXc= +github.com/golangci/golines v0.14.0/go.mod h1:gf555vPG2Ia7mmy2mzmhVQbVjuK8Orw0maR1G4vVAAQ= +github.com/golangci/misspell v0.7.0 h1:4GOHr/T1lTW0hhR4tgaaV1WS/lJ+ncvYCoFKmqJsj0c= +github.com/golangci/misspell v0.7.0/go.mod h1:WZyyI2P3hxPY2UVHs3cS8YcllAeyfquQcKfdeE9AFVg= +github.com/golangci/plugin-module-register v0.1.2 h1:e5WM6PO6NIAEcij3B053CohVp3HIYbzSuP53UAYgOpg= +github.com/golangci/plugin-module-register v0.1.2/go.mod h1:1+QGTsKBvAIvPvoY/os+G5eoqxWn70HYDm2uvUyGuVw= +github.com/golangci/revgrep v0.8.0 h1:EZBctwbVd0aMeRnNUsFogoyayvKHyxlV3CdUA46FX2s= +github.com/golangci/revgrep v0.8.0/go.mod h1:U4R/s9dlXZsg8uJmaR1GrloUr14D7qDl8gi2iPXJH8k= +github.com/golangci/swaggoswag v0.0.0-20250504205917-77f2aca3143e h1:ai0EfmVYE2bRA5htgAG9r7s3tHsfjIhN98WshBTJ9jM= +github.com/golangci/swaggoswag v0.0.0-20250504205917-77f2aca3143e/go.mod h1:Vrn4B5oR9qRwM+f54koyeH3yzphlecwERs0el27Fr/s= +github.com/golangci/unconvert v0.0.0-20250410112200-a129a6e6413e h1:gD6P7NEo7Eqtt0ssnqSJNNndxe69DOQ24A5h7+i3KpM= +github.com/golangci/unconvert v0.0.0-20250410112200-a129a6e6413e/go.mod h1:h+wZwLjUTJnm/P2rwlbJdRPZXOzaT36/FwnPnY2inzc= github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -446,11 +536,9 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= @@ -462,42 +550,40 @@ github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8= github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20230207041349-798e818bf904/go.mod h1:uglQLonpP8qtYCYyzA+8c/9qtqgA3qsXGYqCPKARAFg= github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= -github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs= -github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/pprof v0.0.0-20250820193118-f64d9cf942d6 h1:EEHtgt9IwisQ2AZ4pIsMjahcegHh6rmhqxzIRQIyepY= +github.com/google/pprof v0.0.0-20250820193118-f64d9cf942d6/go.mod h1:I6V7YzU0XDpsHqbsyrghnFZLO1gwK6NPTNvmetQIk9U= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20190309154008-847fc94819f9/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gordonklaus/ineffassign v0.2.0 h1:Uths4KnmwxNJNzq87fwQQDDnbNb7De00VOk9Nu0TySs= +github.com/gordonklaus/ineffassign v0.2.0/go.mod h1:TIpymnagPSexySzs7F9FnO1XFTy8IT3a59vmZp5Y9Lw= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gostaticanalysis/analysisutil v0.7.1 h1:ZMCjoue3DtDWQ5WyU16YbjbQEQ3VuzwxALrpYd+HeKk= +github.com/gostaticanalysis/analysisutil v0.7.1/go.mod h1:v21E3hY37WKMGSnbsw2S/ojApNWb6C1//mXO48CXbVc= +github.com/gostaticanalysis/comment v1.4.2/go.mod h1:KLUTGDv6HOCotCH8h2erHKmpci2ZoR8VPu34YA2uzdM= +github.com/gostaticanalysis/comment v1.5.0 h1:X82FLl+TswsUMpMh17srGRuKaaXprTaytmEpgnKIDu8= +github.com/gostaticanalysis/comment v1.5.0/go.mod h1:V6eb3gpCv9GNVqb6amXzEUX3jXLVK/AdA+IrAMSqvEc= +github.com/gostaticanalysis/forcetypeassert v0.2.0 h1:uSnWrrUEYDr86OCxWa4/Tp2jeYDlogZiZHzGkWFefTk= +github.com/gostaticanalysis/forcetypeassert v0.2.0/go.mod h1:M5iPavzE9pPqWyeiVXSFghQjljW1+l/Uke3PXHS6ILY= +github.com/gostaticanalysis/nilerr v0.1.2 h1:S6nk8a9N8g062nsx63kUkF6AzbHGw7zzyHMcpu52xQU= +github.com/gostaticanalysis/nilerr v0.1.2/go.mod h1:A19UHhoY3y8ahoL7YKz6sdjDtduwTSI4CsymaC2htPA= +github.com/gostaticanalysis/testutil v0.3.1-0.20210208050101-bfb5c8eec0e4/go.mod h1:D+FIZ+7OahH3ePw/izIEeH5I06eKs1IKI4Xr64/Am3M= +github.com/gostaticanalysis/testutil v0.5.0 h1:Dq4wT1DdTwTGCQQv3rl3IvD5Ld0E6HiY+3Zh0sUGqw8= +github.com/gostaticanalysis/testutil v0.5.0/go.mod h1:OLQSbuM6zw2EvCcXTz1lVq5unyoNft372msDY0nY5Hs= github.com/grafana/pyroscope-go v1.2.7 h1:VWBBlqxjyR0Cwk2W6UrE8CdcdD80GOFNutj0Kb1T8ac= github.com/grafana/pyroscope-go v1.2.7/go.mod h1:o/bpSLiJYYP6HQtvcoVKiE9s5RiNgjYTj1DhiddP2Pc= github.com/grafana/pyroscope-go/godeltaprof v0.1.9 h1:c1Us8i6eSmkW+Ez05d3co8kasnuOY813tbMN8i/a3Og= @@ -510,14 +596,22 @@ github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9n github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= +github.com/hashicorp/go-immutable-radix/v2 v2.1.0 h1:CUW5RYIcysz+D3B+l1mDeXrQ7fUvGGCwJfdASSzbrfo= +github.com/hashicorp/go-immutable-radix/v2 v2.1.0/go.mod h1:hgdqLXA4f6NIjRVisM1TJ9aOJVNRqKZj+xDGF6m7PBw= github.com/hashicorp/go-retryablehttp v0.7.8 h1:ylXZWnqa7Lhqpk0L1P1LzDtGcCR0rPVUrx/c8Unxc48= github.com/hashicorp/go-retryablehttp v0.7.8/go.mod h1:rjiScheydd+CxvumBsIrFKlx3iS0jrZ7LvzFGFmuKbw= +github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= +github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/go-version v1.8.0 h1:KAkNb1HAiZd1ukkxDFGmokVZe1Xy9HG6NUp+bPle2i4= +github.com/hashicorp/go-version v1.8.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru/arc/v2 v2.0.7 h1:QxkVTxwColcduO+LP7eJO56r2hFiG8zEbfAAzRv52KQ= github.com/hashicorp/golang-lru/arc/v2 v2.0.7/go.mod h1:Pe7gBlGdc8clY5LJ0LpJXMt5AmgmWNH1g+oFFVUHOEc= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/heimdalr/dag v1.5.0 h1:hqVtijvY776P5OKP3QbdVBRt3Xxq6BYopz3XgklsGvo= github.com/heimdalr/dag v1.5.0/go.mod h1:lthekrHl01dddmzqyBQ1YZbi7XcVGGzjFo0jIky5knc= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= @@ -536,8 +630,6 @@ github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc= github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= github.com/ianlancetaylor/cgosymbolizer v0.0.0-20241129212102-9c50ad6b591e h1:8AnObPi8WmIgjwcidUxaREhXMSpyUJeeSrIkZTXdabw= github.com/ianlancetaylor/cgosymbolizer v0.0.0-20241129212102-9c50ad6b591e/go.mod h1:DvXTE/K/RtHehxU8/GtDs4vFtfw64jJ3PaCnFri8CRg= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20220319035150-800ac71e25c2/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= github.com/ianlancetaylor/demangle v0.0.0-20230524184225-eabc099b10ab/go.mod h1:gx7rwoVhcfuVKG5uya9Hs3Sxj7EIvldVofAWIUtGouw= github.com/imdario/mergo v0.3.11 h1:3tnifQM4i+fbajXKBHXWEH+KvNHqojZ778UH75j3bGA= @@ -559,11 +651,19 @@ github.com/jedib0t/go-pretty/v6 v6.6.8/go.mod h1:YwC5CE4fJ1HFUDeivSV1r//AmANFHyq github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU= github.com/jellydator/ttlcache/v3 v3.4.0 h1:YS4P125qQS0tNhtL6aeYkheEaB/m8HCqdMMP4mnWdTY= github.com/jellydator/ttlcache/v3 v3.4.0/go.mod h1:Hw9EgjymziQD3yGsQdf1FqFdpp7YjFMd4Srg5EJlgD4= +github.com/jgautheron/goconst v1.8.2 h1:y0XF7X8CikZ93fSNT6WBTb/NElBu9IjaY7CCYQrCMX4= +github.com/jgautheron/goconst v1.8.2/go.mod h1:A0oxgBCHy55NQn6sYpO7UdnA9p+h7cPtoOZUmvNIako= +github.com/jingyugao/rowserrcheck v1.1.1 h1:zibz55j/MJtLsjP1OF4bSdgXxwL1b+Vn7Tjzq7gFzUs= +github.com/jingyugao/rowserrcheck v1.1.1/go.mod h1:4yvlZSDb3IyDTUZJUmpZfm2Hwok+Dtp+nu2qOq+er9c= github.com/jinzhu/copier v0.4.0 h1:w3ciUoD19shMCRargcpm0cm91ytaBhDvuRpz1ODO/U8= github.com/jinzhu/copier v0.4.0/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg= +github.com/jjti/go-spancheck v0.6.5 h1:lmi7pKxa37oKYIMScialXUK6hP3iY5F1gu+mLBPgYB8= +github.com/jjti/go-spancheck v0.6.5/go.mod h1:aEogkeatBrbYsyW6y5TgDfihCulDYciL1B7rG2vSsrU= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/josharian/native v1.1.0 h1:uuaP0hAbW7Y4l0ZRQ6C9zfb7Mg1mbFKry/xzDAfmtLA= github.com/josharian/native v1.1.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= +github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE= +github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung= github.com/jsimonetti/rtnetlink/v2 v2.0.1 h1:xda7qaHDSVOsADNouv7ukSuicKZO7GgVUCXxpaIEIlM= github.com/jsimonetti/rtnetlink/v2 v2.0.1/go.mod h1:7MoNYNbb3UaDHtF8udiJo/RH6VsTKP1pqKLUTVCvToE= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -571,13 +671,20 @@ github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/u github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julz/importas v0.2.0 h1:y+MJN/UdL63QbFJHws9BVC5RpA2iq0kpjrFajTGivjQ= +github.com/julz/importas v0.2.0/go.mod h1:pThlt589EnCYtMnmhmRYY/qn9lCf/frPOK+WMx3xiJY= +github.com/karamaru-alpha/copyloopvar v1.2.2 h1:yfNQvP9YaGQR7VaWLYcfZUlRP2eo2vhExWKxD/fP6q0= +github.com/karamaru-alpha/copyloopvar v1.2.2/go.mod h1:oY4rGZqZ879JkJMtX3RRkcXRkmUvH0x35ykgaKgsgJY= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/errcheck v1.9.0 h1:9xt1zI9EBfcYBvdU1nVrzMzzUPUtPKs9bVSIM3TAb3M= +github.com/kisielk/errcheck v1.9.0/go.mod h1:kQxWMMVZgIkDq7U8xtG/n2juOjbLgZtedi0D+/VL/i8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kkHAIKE/contextcheck v1.1.6 h1:7HIyRcnyzxL9Lz06NGhiKvenXq7Zw6Q0UQu/ttjfJCE= +github.com/kkHAIKE/contextcheck v1.1.6/go.mod h1:3dDbMRNBFaq8HFXWC1JyvDSPm43CmE6IuHam8Wr0rkg= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= @@ -585,7 +692,6 @@ github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZY github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/koron/go-ssdp v0.0.4 h1:1IDwrghSKYM7yLf7XCzbByg2sJ/JcNOZRXS2jczTwz0= github.com/koron/go-ssdp v0.0.4/go.mod h1:oDXq+E5IL5q0U8uSBcoAXzTzInwy5lEgC91HoKtbmZk= -github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -597,11 +703,31 @@ github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kulti/thelper v0.7.1 h1:fI8QITAoFVLx+y+vSyuLBP+rcVIB8jKooNSCT2EiI98= +github.com/kulti/thelper v0.7.1/go.mod h1:NsMjfQEy6sd+9Kfw8kCP61W1I0nerGSYSFnGaxQkcbs= +github.com/kunwardeep/paralleltest v1.0.15 h1:ZMk4Qt306tHIgKISHWFJAO1IDQJLc6uDyJMLyncOb6w= +github.com/kunwardeep/paralleltest v1.0.15/go.mod h1:di4moFqtfz3ToSKxhNjhOZL+696QtJGCFe132CbBLGk= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/lasiar/canonicalheader v1.1.2 h1:vZ5uqwvDbyJCnMhmFYimgMZnJMjwljN5VGY0VKbMXb4= +github.com/lasiar/canonicalheader v1.1.2/go.mod h1:qJCeLFS0G/QlLQ506T+Fk/fWMa2VmBUiEI2cuMK4djI= +github.com/ldez/exptostd v0.4.5 h1:kv2ZGUVI6VwRfp/+bcQ6Nbx0ghFWcGIKInkG/oFn1aQ= +github.com/ldez/exptostd v0.4.5/go.mod h1:QRjHRMXJrCTIm9WxVNH6VW7oN7KrGSht69bIRwvdFsM= +github.com/ldez/gomoddirectives v0.8.0 h1:JqIuTtgvFC2RdH1s357vrE23WJF2cpDCPFgA/TWDGpk= +github.com/ldez/gomoddirectives v0.8.0/go.mod h1:jutzamvZR4XYJLr0d5Honycp4Gy6GEg2mS9+2YX3F1Q= +github.com/ldez/grignotin v0.10.1 h1:keYi9rYsgbvqAZGI1liek5c+jv9UUjbvdj3Tbn5fn4o= +github.com/ldez/grignotin v0.10.1/go.mod h1:UlDbXFCARrXbWGNGP3S5vsysNXAPhnSuBufpTEbwOas= +github.com/ldez/structtags v0.6.1 h1:bUooFLbXx41tW8SvkfwfFkkjPYvFFs59AAMgVg6DUBk= +github.com/ldez/structtags v0.6.1/go.mod h1:YDxVSgDy/MON6ariaxLF2X09bh19qL7MtGBN5MrvbdY= +github.com/ldez/tagliatelle v0.7.2 h1:KuOlL70/fu9paxuxbeqlicJnCspCRjH0x8FW+NfgYUk= +github.com/ldez/tagliatelle v0.7.2/go.mod h1:PtGgm163ZplJfZMZ2sf5nhUT170rSuPgBimoyYtdaSI= +github.com/ldez/usetesting v0.5.0 h1:3/QtzZObBKLy1F4F8jLuKJiKBjjVFi1IavpoWbmqLwc= +github.com/ldez/usetesting v0.5.0/go.mod h1:Spnb4Qppf8JTuRgblLrEWb7IE6rDmUpGvxY3iRrzvDQ= github.com/leanovate/gopter v0.2.11 h1:vRjThO1EKPb/1NsDXuDrzldR28RLkBflWYcU9CvzWu4= github.com/leanovate/gopter v0.2.11/go.mod h1:aK3tzZP/C+p1m3SPRE4SYZFGP7jjkuSI4f7Xvpt0S9c= github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs= +github.com/leonklingele/grouper v1.1.2 h1:o1ARBDLOmmasUaNDesWqWCIFH3u7hoFlM84YrjT3mIY= +github.com/leonklingele/grouper v1.1.2/go.mod h1:6D0M/HVkhs2yRKRFZUoGjeDy7EZTfFBE9gl4kjmIGkA= github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8= github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= github.com/libp2p/go-flow-metrics v0.2.0 h1:EIZzjmeOE6c8Dav0sNv35vhZxATIXWZg6j/C08XmmDw= @@ -633,13 +759,31 @@ github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3 h1:PwQumkgq4/acIiZhtifTV5OUqqiP82UAl0h87xj/l9k= github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg= github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= +github.com/macabu/inamedparam v0.2.0 h1:VyPYpOc10nkhI2qeNUdh3Zket4fcZjEWe35poddBCpE= +github.com/macabu/inamedparam v0.2.0/go.mod h1:+Pee9/YfGe5LJ62pYXqB89lJ+0k5bsR8Wgz/C0Zlq3U= +github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo= +github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/manuelarte/embeddedstructfieldcheck v0.4.0 h1:3mAIyaGRtjK6EO9E73JlXLtiy7ha80b2ZVGyacxgfww= +github.com/manuelarte/embeddedstructfieldcheck v0.4.0/go.mod h1:z8dFSyXqp+fC6NLDSljRJeNQJJDWnY7RoWFzV3PC6UM= +github.com/manuelarte/funcorder v0.5.0 h1:llMuHXXbg7tD0i/LNw8vGnkDTHFpTnWqKPI85Rknc+8= +github.com/manuelarte/funcorder v0.5.0/go.mod h1:Yt3CiUQthSBMBxjShjdXMexmzpP8YGvGLjrxJNkO2hA= +github.com/maratori/testableexamples v1.0.1 h1:HfOQXs+XgfeRBJ+Wz0XfH+FHnoY9TVqL6Fcevpzy4q8= +github.com/maratori/testableexamples v1.0.1/go.mod h1:XE2F/nQs7B9N08JgyRmdGjYVGqxWwClLPCGSQhXQSrQ= +github.com/maratori/testpackage v1.1.2 h1:ffDSh+AgqluCLMXhM19f/cpvQAKygKAJXFl9aUjmbqs= +github.com/maratori/testpackage v1.1.2/go.mod h1:8F24GdVDFW5Ew43Et02jamrVMNXLUNaOynhDssITGfc= github.com/mark3labs/mcp-go v0.43.2 h1:21PUSlWWiSbUPQwXIJ5WKlETixpFpq+WBpbMGDSVy/I= github.com/mark3labs/mcp-go v0.43.2/go.mod h1:YnJfOL382MIWDx1kMY+2zsRHU/q78dBg9aFb8W6Thdw= github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd h1:br0buuQ854V8u83wA0rVZ8ttrq5CpaPZdvrK0LP2lOk= github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd/go.mod h1:QuCEs1Nt24+FYQEqAAncTDPJIuGs+LxK1MCiFL25pMU= +github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= +github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= +github.com/matoous/godox v1.1.0 h1:W5mqwbyWrwZv6OQ5Z1a/DHGMOvXYCBP3+Ht7KMoJhq4= +github.com/matoous/godox v1.1.0/go.mod h1:jgE/3fUXiTurkdHOLT5WEkThTSuE7yxHv5iWPa80afs= +github.com/matryer/is v1.4.0 h1:sosSmIWwkYITGrxZ25ULNDeKiMNzFSr4V/eqBQP0PeE= +github.com/matryer/is v1.4.0/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwMRU= github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= @@ -654,6 +798,10 @@ github.com/mdlayher/netlink v1.7.2 h1:/UtM3ofJap7Vl4QWCPDGXY8d3GIY2UGSDbK+QWmY8/ github.com/mdlayher/netlink v1.7.2/go.mod h1:xraEF7uJbxLhc5fpHL4cPe221LI2bdttWlU+ZGLfQSw= github.com/mdlayher/socket v0.4.1 h1:eM9y2/jlbs1M615oshPQOHZzj6R6wMT7bX5NPiQvn2U= github.com/mdlayher/socket v0.4.1/go.mod h1:cAqeGjoufqdxWkD7DkpyS+wcefOtmu5OQ8KuoJGIReA= +github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE= +github.com/mfridman/tparse v0.18.0/go.mod h1:gEvqZTuCgEhPbYk/2lS3Kcxg1GmTxxU7kTC8DvP0i/A= +github.com/mgechev/revive v1.13.0 h1:yFbEVliCVKRXY8UgwEO7EOYNopvjb1BFbmYqm9hZjBM= +github.com/mgechev/revive v1.13.0/go.mod h1:efJfeBVCX2JUumNQ7dtOLDja+QKj9mYGgEZA7rt5u+0= github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= github.com/miekg/dns v1.1.62 h1:cN8OuEF1/x5Rq6Np+h1epln8OiyPWV+lROx9LxcGgIQ= github.com/miekg/dns v1.1.62/go.mod h1:mvDlcItzm+br7MToIKqkglaGhlFMHJ9DTNNWONWXbNQ= @@ -669,6 +817,10 @@ github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dz github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/mitchellh/copystructure v1.0.0 h1:Laisrj+bAB6b/yJwB5Bt3ITZhGJdqmxquMKeZ+mmkFQ= github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/reflectwalk v1.0.0 h1:9D+8oIskB4VJBN5SFlmc27fSlIBZaov1Wpk/IfikLNY= github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= @@ -680,6 +832,8 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/moricho/tparallel v0.3.2 h1:odr8aZVFA3NZrNybggMkYO3rgPRcqjeQUlBBFVxKHTI= +github.com/moricho/tparallel v0.3.2/go.mod h1:OQ+K3b4Ln3l2TZveGCywybl68glfLEwFGqvnjok8b+U= github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= @@ -717,10 +871,18 @@ github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOEL github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/nakabonne/nestif v0.3.1 h1:wm28nZjhQY5HyYPx+weN3Q65k6ilSBxDb8v5S81B81U= +github.com/nakabonne/nestif v0.3.1/go.mod h1:9EtoZochLn5iUprVDmDjqGKPofoUEBL8U4Ngq6aY7OE= github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= +github.com/nishanths/exhaustive v0.12.0 h1:vIY9sALmw6T/yxiASewa4TQcFsVYZQQRUQJhKRf3Swg= +github.com/nishanths/exhaustive v0.12.0/go.mod h1:mEZ95wPIZW+x8kC4TgC+9YCUgiST7ecevsVDTgc2obs= +github.com/nishanths/predeclared v0.2.2 h1:V2EPdZPliZymNAn79T8RkNApBjMmVKh5XRpLm/w98Vk= +github.com/nishanths/predeclared v0.2.2/go.mod h1:RROzoN6TnGQupbC+lqggsOlcgysk3LMK/HI84Mp280c= +github.com/nunnatsa/ginkgolinter v0.21.2 h1:khzWfm2/Br8ZemX8QM1pl72LwM+rMeW6VUbQ4rzh0Po= +github.com/nunnatsa/ginkgolinter v0.21.2/go.mod h1:GItSI5fw7mCGLPmkvGYrr1kEetZe7B593jcyOpyabsY= github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY= github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc= github.com/nyaosorg/go-windows-shortcut v0.0.0-20220529122037-8b0c89bca4c4 h1:+3bXHpIl3RiBuPKlqeCZZeShGHC9RFhR/P2OJfOLRyA= @@ -729,17 +891,24 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/ginkgo/v2 v2.20.2 h1:7NVCeyIWROIAheY21RLS+3j2bb52W0W82tkberYytp4= -github.com/onsi/ginkgo/v2 v2.20.2/go.mod h1:K9gyxPIlb+aIvnZ8bd9Ak+YP18w3APlR+5coaZoE2ag= +github.com/onsi/ginkgo/v2 v2.27.2 h1:LzwLj0b89qtIy6SSASkzlNvX6WktqurSHwkk2ipF/Ns= +github.com/onsi/ginkgo/v2 v2.27.2/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= -github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= +github.com/onsi/gomega v1.38.2 h1:eZCjf2xjZAqe+LeWvKb5weQ+NcPwX84kqJ0cZNxok2A= +github.com/onsi/gomega v1.38.2/go.mod h1:W2MJcYxRGV63b418Ai34Ud0hEdTVXq9NW9+Sx6uXf3k= github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.2.0 h1:z97+pHb3uELt/yiAWD691HNHQIF07bE7dzrbT927iTk= github.com/opencontainers/runtime-spec v1.2.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0= +github.com/otiai10/copy v1.2.0/go.mod h1:rrF5dJ5F0t/EWSYODDu4j9/vEeYHMkc8jt0zJChqQWw= +github.com/otiai10/copy v1.14.0 h1:dCI/t1iTdYGtkvCuBG2BgR6KZa83PTclw4U5n2wAllU= +github.com/otiai10/copy v1.14.0/go.mod h1:ECfuL02W+/FkTWZWgQqXPWZgW9oeKCSQ5qVfSc4qc4w= +github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE= +github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs= +github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo= +github.com/otiai10/mint v1.3.1/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= @@ -810,12 +979,13 @@ github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= +github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= @@ -826,7 +996,6 @@ github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UH github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= @@ -854,14 +1023,24 @@ github.com/prysmaticlabs/gohashtree v0.0.4-beta h1:H/EbCuXPeTV3lpKeXGPpEV9gsUpkq github.com/prysmaticlabs/gohashtree v0.0.4-beta/go.mod h1:BFdtALS+Ffhg3lGQIHv9HDWuHS8cTvHZzrHWxwOtGOs= github.com/puzpuzpuz/xsync/v4 v4.4.0 h1:vlSN6/CkEY0pY8KaB0yqo/pCLZvp9nhdbBdjipT4gWo= github.com/puzpuzpuz/xsync/v4 v4.4.0/go.mod h1:VJDmTCJMBt8igNxnkQd86r+8KUeN1quSfNKu5bLYFQo= -github.com/quasilyte/go-ruleguard/dsl v0.3.22 h1:wd8zkOhSNr+I+8Qeciml08ivDt1pSXe60+5DqOpCjPE= -github.com/quasilyte/go-ruleguard/dsl v0.3.22/go.mod h1:KeCP03KrjuSO0H1kTuZQCWlQPulDV6YMIXmpQss17rU= +github.com/quasilyte/go-ruleguard v0.4.5 h1:AGY0tiOT5hJX9BTdx/xBdoCubQUAE2grkqY2lSwvZcA= +github.com/quasilyte/go-ruleguard v0.4.5/go.mod h1:Vl05zJ538vcEEwu16V/Hdu7IYZWyKSwIy4c88Ro1kRE= +github.com/quasilyte/go-ruleguard/dsl v0.3.23 h1:lxjt5B6ZCiBeeNO8/oQsegE6fLeCzuMRoVWSkXC4uvY= +github.com/quasilyte/go-ruleguard/dsl v0.3.23/go.mod h1:KeCP03KrjuSO0H1kTuZQCWlQPulDV6YMIXmpQss17rU= +github.com/quasilyte/gogrep v0.5.0 h1:eTKODPXbI8ffJMN+W2aE0+oL0z/nh8/5eNdiO34SOAo= +github.com/quasilyte/gogrep v0.5.0/go.mod h1:Cm9lpz9NZjEoL1tgZ2OgeUKPIxL1meE7eo60Z6Sk+Ng= +github.com/quasilyte/regex/syntax v0.0.0-20210819130434-b3f0c404a727 h1:TCg2WBOl980XxGFEZSS6KlBGIV0diGdySzxATTWoqaU= +github.com/quasilyte/regex/syntax v0.0.0-20210819130434-b3f0c404a727/go.mod h1:rlzQ04UMyJXu/aOvhd8qT+hvDrFpiwqp8MRXDY9szc0= +github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567 h1:M8mH9eK4OUR4lu7Gd+PU1fV2/qnDNfzT635KRSObncs= +github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567/go.mod h1:DWNGW8A4Y+GyBgPuaQJuWiy0XYftx4Xm/y5Jqk9I6VQ= github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI= github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg= github.com/quic-go/quic-go v0.49.1 h1:e5JXpUyF0f2uFjckQzD8jTghZrOUK1xxDqqZhlwixo0= github.com/quic-go/quic-go v0.49.1/go.mod h1:s2wDnmCdooUQBmQfpUSTCYBl1/D4FcqbULMMkASvR6s= github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66 h1:4WFk6u3sOT6pLa1kQ50ZVdm8BQFgJNA117cepZxtLIg= github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66/go.mod h1:Vp72IJajgeOL6ddqrAhmp7IM9zbTcgkQxD/YdxrVwMw= +github.com/raeperd/recvcheck v0.2.0 h1:GnU+NsbiCqdC2XX5+vMZzP+jAJC5fht7rcVTAhX74UI= +github.com/raeperd/recvcheck v0.2.0/go.mod h1:n04eYkwIR0JbgD73wT8wL4JjPC3wm0nFtzBnWNocnYU= github.com/raulk/go-watchdog v1.3.0 h1:oUmdlHxdkXRJlwfG0O9omj8ukerm8MEQavSiDTEtBsk= github.com/raulk/go-watchdog v1.3.0/go.mod h1:fIvOnLbF0b0ZwkB9YU4mOW9Did//4vPZtDqv66NfsMU= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= @@ -870,7 +1049,6 @@ github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qq github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= @@ -883,13 +1061,27 @@ github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryancurrah/gomodguard v1.4.1 h1:eWC8eUMNZ/wM/PWuZBv7JxxqT5fiIKSIyTvjb7Elr+g= +github.com/ryancurrah/gomodguard v1.4.1/go.mod h1:qnMJwV1hX9m+YJseXEBhd2s90+1Xn6x9dLz11ualI1I= +github.com/ryanrolds/sqlclosecheck v0.5.1 h1:dibWW826u0P8jNLsLN+En7+RqWWTYrjCB9fJfSfdyCU= +github.com/ryanrolds/sqlclosecheck v0.5.1/go.mod h1:2g3dUjoS6AL4huFdv6wn55WpLIDjY7ZgUR4J8HOO/XQ= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= +github.com/sanposhiho/wastedassign/v2 v2.1.0 h1:crurBF7fJKIORrV85u9UUpePDYGWnwvv3+A96WvwXT0= +github.com/sanposhiho/wastedassign/v2 v2.1.0/go.mod h1:+oSmSC+9bQ+VUAxA66nBb0Z7N8CK7mscKTDYC6aIek4= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= +github.com/sashamelentyev/interfacebloat v1.1.0 h1:xdRdJp0irL086OyW1H/RTZTr1h/tMEOsumirXcOJqAw= +github.com/sashamelentyev/interfacebloat v1.1.0/go.mod h1:+Y9yU5YdTkrNvoX0xHc84dxiN1iBi9+G8zZIhPVoNjQ= +github.com/sashamelentyev/usestdlibvars v1.29.0 h1:8J0MoRrw4/NAXtjQqTHrbW9NN+3iMf7Knkq057v4XOQ= +github.com/sashamelentyev/usestdlibvars v1.29.0/go.mod h1:8PpnjHMk5VdeWlVb4wCdrB8PNbLqZ3wBZTZWkrpZZL8= +github.com/securego/gosec/v2 v2.22.11 h1:tW+weM/hCM/GX3iaCV91d5I6hqaRT2TPsFM1+USPXwg= +github.com/securego/gosec/v2 v2.22.11/go.mod h1:KE4MW/eH0GLWztkbt4/7XpyH0zJBBnu7sYB4l6Wn7Mw= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= -github.com/shirou/gopsutil/v4 v4.25.10 h1:at8lk/5T1OgtuCp+AwrDofFRjnvosn0nkN2OLQ6g8tA= -github.com/shirou/gopsutil/v4 v4.25.10/go.mod h1:+kSwyC8DRUD9XXEHCAFjK+0nuArFJM0lva+StQAcskM= +github.com/shirou/gopsutil/v4 v4.25.12 h1:e7PvW/0RmJ8p8vPGJH4jvNkOyLmbkXgXW4m6ZPic6CY= +github.com/shirou/gopsutil/v4 v4.25.12/go.mod h1:EivAfP5x2EhLp2ovdpKSozecVXn1TmuG7SMzs/Wh4PU= github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY= @@ -920,27 +1112,42 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sivchari/containedctx v1.0.3 h1:x+etemjbsh2fB5ewm5FeLNi5bUjK0V8n0RB+Wwfd0XE= +github.com/sivchari/containedctx v1.0.3/go.mod h1:c1RDvCbnJLtH4lLcYD/GqwiBSSf4F5Qk0xld2rBqzJ4= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v0.0.0-20190215210624-980c5ac6f3ac/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s= github.com/smartystreets/goconvey v0.0.0-20190306220146-200a235640ff/go.mod h1:KSQcGKpxUMHk3nbYzs/tIBAM2iDooCn0BmttHOJEbLs= +github.com/sonatard/noctx v0.4.0 h1:7MC/5Gg4SQ4lhLYR6mvOP6mQVSxCrdyiExo7atBs27o= +github.com/sonatard/noctx v0.4.0/go.mod h1:64XdbzFb18XL4LporKXp8poqZtPKbCrqQ402CV+kJas= github.com/sosodev/duration v1.3.1 h1:qtHBDMQ6lvMQsL15g4aopM4HEfOaYuhWBw3NPTtlqq4= github.com/sosodev/duration v1.3.1/go.mod h1:RQIBBX0+fMLc/D9+Jb/fwvVmo0eZvDDEERAikUR6SDg= github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE= +github.com/sourcegraph/go-diff v0.7.0 h1:9uLlrd5T46OXs5qpp8L/MTltk0zikUGi0sNNyCpA8G0= +github.com/sourcegraph/go-diff v0.7.0/go.mod h1:iBszgVvyxdc8SFZ7gm69go2KDdt3ag071iBaWPF6cjs= github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.10.0 h1:EaGW2JJh15aKOejeuJ+wpFSHnbd7GE6Wvp3TsNhb6LY= -github.com/spf13/afero v1.10.0/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ= +github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= +github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= -github.com/spf13/cobra v1.10.1 h1:lJeBwCfmrnXthfAupyUTzJ/J4Nc1RsHC/mSRU2dll/s= -github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4XaB0= +github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= +github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= +github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= +github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.12.0 h1:CZ7eSOd3kZoaYDLbXnmzgQI5RlciuXBMA+18HwHRfZQ= +github.com/spf13/viper v1.12.0/go.mod h1:b6COn30jlNxbm/V2IqWiNWkJ+vZNiMNksliPCiuKtSI= +github.com/ssgreg/nlreturn/v2 v2.2.1 h1:X4XDI7jstt3ySqGU86YGAURbxw3oTDPK9sPEi6YEwQ0= +github.com/ssgreg/nlreturn/v2 v2.2.1/go.mod h1:E/iiPB78hV7Szg2YfRgyIrk1AD6JVMTRkkxBiELzh2I= +github.com/stbenjam/no-sprintf-host-port v0.3.1 h1:AyX7+dxI4IdLBPtDbsGAyqiTSLpCP9hWRrXQDU4Cm/g= +github.com/stbenjam/no-sprintf-host-port v0.3.1/go.mod h1:ODbZesTCHMVKthBHskvUUexdcNHAQRXk9NpSsL8p/HQ= github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs= github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -963,13 +1170,33 @@ github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/subosito/gotenv v1.4.1 h1:jyEFiXpy21Wm81FBN71l9VoMMV8H8jG+qIK3GCpY6Qs= +github.com/subosito/gotenv v1.4.1/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= github.com/supranational/blst v0.3.16-0.20250831170142-f48500c1fdbe h1:nbdqkIGOGfUAD54q1s2YBcBz/WcsxCO9HUQ4aGV5hUw= github.com/supranational/blst v0.3.16-0.20250831170142-f48500c1fdbe/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= +github.com/tenntenn/modver v1.0.1 h1:2klLppGhDgzJrScMpkj9Ujy3rXPUspSjAcev9tSEBgA= +github.com/tenntenn/modver v1.0.1/go.mod h1:bePIyQPb7UeioSRkw3Q0XeMhYZSMx9B8ePqg6SAMGH0= +github.com/tenntenn/text/transform v0.0.0-20200319021203-7eef512accb3 h1:f+jULpRQGxTSkNYKJ51yaw6ChIqO+Je8UqsTKN/cDag= +github.com/tenntenn/text/transform v0.0.0-20200319021203-7eef512accb3/go.mod h1:ON8b8w4BN/kE1EOhwT0o+d62W65a6aPw1nouo9LMgyY= +github.com/tetafro/godot v1.5.4 h1:u1ww+gqpRLiIA16yF2PV1CV1n/X3zhyezbNXC3E14Sg= +github.com/tetafro/godot v1.5.4/go.mod h1:eOkMrVQurDui411nBY2FA05EYH01r14LuWY/NrVDVcU= github.com/thomaso-mirodin/intmath v0.0.0-20160323211736-5dc6d854e46e h1:cR8/SYRgyQCt5cNCMniB/ZScMkhI9nk8U5C7SbISXjo= github.com/thomaso-mirodin/intmath v0.0.0-20160323211736-5dc6d854e46e/go.mod h1:Tu4lItkATkonrYuvtVjG0/rhy15qrNGNTjPdaphtZ/8= github.com/tidwall/btree v1.8.1 h1:27ehoXvm5AG/g+1VxLS1SD3vRhp/H7LuEfwNvddEdmA= github.com/tidwall/btree v1.8.1/go.mod h1:jBbTdUWhSZClZWoDg54VnvV7/54modSOzDN7VXftj1A= +github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= +github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= +github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= +github.com/timakin/bodyclose v0.0.0-20241222091800-1db5c5ca4d67 h1:9LPGD+jzxMlnk5r6+hJnar67cgpDIz/iyD+rfl5r2Vk= +github.com/timakin/bodyclose v0.0.0-20241222091800-1db5c5ca4d67/go.mod h1:mkjARE7Yr8qU23YcGMSALbIxTQ9r9QBVahQOBRfU460= +github.com/timonwong/loggercheck v0.11.0 h1:jdaMpYBl+Uq9mWPXv1r8jc5fC3gyXx4/WGwTnnNKn4M= +github.com/timonwong/loggercheck v0.11.0/go.mod h1:HEAWU8djynujaAVX7QI65Myb8qgfcZ1uKbdpg3ZzKl8= github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tinylib/msgp v1.1.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= @@ -977,13 +1204,25 @@ github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYI github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI= github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw= github.com/tklauser/numcpus v0.11.0/go.mod h1:z+LwcLq54uWZTX0u/bGobaV34u6V7KNlTZejzM6/3MQ= +github.com/tomarrell/wrapcheck/v2 v2.12.0 h1:H/qQ1aNWz/eeIhxKAFvkfIA+N7YDvq6TWVFL27Of9is= +github.com/tomarrell/wrapcheck/v2 v2.12.0/go.mod h1:AQhQuZd0p7b6rfW+vUwHm5OMCGgp63moQ9Qr/0BpIWo= +github.com/tommy-muehle/go-mnd/v2 v2.5.1 h1:NowYhSdyE/1zwK9QCLeRb6USWdoif80Ie+v+yU8u1Zw= +github.com/tommy-muehle/go-mnd/v2 v2.5.1/go.mod h1:WsUAkMJMYww6l/ufffCD3m+P7LEvr8TnZn9lwVDlgzw= github.com/ugorji/go/codec v1.2.13 h1:6nvAfJXxwEVFG0UdQwvobVN44a+xQAFiQajSG1Z6bU8= github.com/ugorji/go/codec v1.2.13/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/ultraware/funlen v0.2.0 h1:gCHmCn+d2/1SemTdYMiKLAHFYxTYz7z9VIDRaTGyLkI= +github.com/ultraware/funlen v0.2.0/go.mod h1:ZE0q4TsJ8T1SQcjmkhN/w+MceuatI6pBFSxxyteHIJA= +github.com/ultraware/whitespace v0.2.0 h1:TYowo2m9Nfj1baEQBjuHzvMRbp19i+RCcRYrSWoFa+g= +github.com/ultraware/whitespace v0.2.0/go.mod h1:XcP1RLD81eV4BW8UhQlpaR+SDc2givTvyI8a586WjW8= github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli/v2 v2.27.7 h1:bH59vdhbjLv3LAvIu6gd0usJHgoTTPhCFib8qqOwXYU= github.com/urfave/cli/v2 v2.27.7/go.mod h1:CyNAG/xg+iAOg0N4MPGZqVmv2rCoP267496AOXUZjA4= github.com/urfave/cli/v3 v3.6.0 h1:oIdArVjkdIXHWg3iqxgmqwQGC8NM0JtdgwQAj2sRwFo= github.com/urfave/cli/v3 v3.6.0/go.mod h1:ysVLtOEmg2tOy6PknnYVhDoouyC/6N42TMeoMzskhso= +github.com/uudashr/gocognit v1.2.0 h1:3BU9aMr1xbhPlvJLSydKwdLN3tEUUrzPSSM8S4hDYRA= +github.com/uudashr/gocognit v1.2.0/go.mod h1:k/DdKPI6XBZO1q7HgoV2juESI2/Ofj9AcHPZhBBdrTU= +github.com/uudashr/iface v1.4.1 h1:J16Xl1wyNX9ofhpHmQ9h9gk5rnv2A6lX/2+APLTo0zU= +github.com/uudashr/iface v1.4.1/go.mod h1:pbeBPlbuU2qkNDn0mmfrxP2X+wjPMIQAy+r1MBXSXtg= github.com/valyala/fastjson v1.6.4 h1:uAUNq9Z6ymTgGhcm0UynUAB6tlbakBrz6CQFax3BXVQ= github.com/valyala/fastjson v1.6.4/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY= github.com/vektah/gqlparser/v2 v2.5.31 h1:YhWGA1mfTjID7qJhd1+Vxhpk5HTgydrGU9IgkWBTJ7k= @@ -997,12 +1236,20 @@ github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+x github.com/wlynxg/anet v0.0.3/go.mod h1:eay5PRQr7fIVAMbTbchTnO9gG65Hg/uYGdc7mguHxoA= github.com/wlynxg/anet v0.0.5 h1:J3VJGi1gvo0JwZ/P1/Yc/8p63SoW98B5dHkYDmpgvvU= github.com/wlynxg/anet v0.0.5/go.mod h1:eay5PRQr7fIVAMbTbchTnO9gG65Hg/uYGdc7mguHxoA= +github.com/xen0n/gosmopolitan v1.3.0 h1:zAZI1zefvo7gcpbCOrPSHJZJYA9ZgLfJqtKzZ5pHqQM= +github.com/xen0n/gosmopolitan v1.3.0/go.mod h1:rckfr5T6o4lBtM1ga7mLGKZmLxswUoH1zxHgNXOsEt4= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= github.com/xsleonard/go-merkle v1.1.0 h1:fHe1fuhJjGH22ZzVTAH0jqHLhTGhOq3wQjJN+8P0jQg= github.com/xsleonard/go-merkle v1.1.0/go.mod h1:cW4z+UZ/4f2n9IJgIiyDCdYguchoDyDAPmpuOWGxdGg= +github.com/yagipy/maintidx v1.0.0 h1:h5NvIsCz+nRDapQ0exNv4aJ0yXSI0420omVANTv3GJM= +github.com/yagipy/maintidx v1.0.0/go.mod h1:0qNf/I/CCZXSMhsRsrEPDZ+DkekpKLXAJfsTACwgXLk= +github.com/yeya24/promlinter v0.3.0 h1:JVDbMp08lVCP7Y6NP3qHroGAO6z2yGKQtS5JsjqtoFs= +github.com/yeya24/promlinter v0.3.0/go.mod h1:cDfJQQYv9uYciW60QT0eeHlFodotkYZlL+YcPQN+mW4= +github.com/ykadowak/zerologlint v0.1.5 h1:Gy/fMz1dFQN9JZTPjv1hxEk+sRWm05row04Yoolgdiw= +github.com/ykadowak/zerologlint v0.1.5/go.mod h1:KaUskqF3e/v59oPmdq1U1DnKcuHokl2/K1U4pmIELKg= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -1014,17 +1261,24 @@ github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +gitlab.com/bosi/decorder v0.4.2 h1:qbQaV3zgwnBZ4zPMhGLW4KZe7A7NwxEhJx39R3shffo= +gitlab.com/bosi/decorder v0.4.2/go.mod h1:muuhHoaJkA9QLcYHq4Mj8FJUwDZ+EirSHRiaTcTf6T8= +go-simpler.org/assert v0.9.0 h1:PfpmcSvL7yAnWyChSjOz6Sp6m9j5lyK8Ok9pEL31YkQ= +go-simpler.org/assert v0.9.0/go.mod h1:74Eqh5eI6vCK6Y5l3PI8ZYFXG4Sa+tkr70OIPJAUr28= +go-simpler.org/musttag v0.14.0 h1:XGySZATqQYSEV3/YTy+iX+aofbZZllJaqwFWs+RTtSo= +go-simpler.org/musttag v0.14.0/go.mod h1:uP8EymctQjJ4Z1kUnjX0u2l60WfUdQxCwSNKzE1JEOE= +go-simpler.org/sloglint v0.11.1 h1:xRbPepLT/MHPTCA6TS/wNfZrDzkGvCCqUv4Bdwc3H7s= +go-simpler.org/sloglint v0.11.1/go.mod h1:2PowwiCOK8mjiF+0KGifVOT8ZsCNiFzvfyJeJOIt8MQ= +go.augendre.info/arangolint v0.3.1 h1:n2E6p8f+zfXSFLa2e2WqFPp4bfvcuRdd50y6cT65pSo= +go.augendre.info/arangolint v0.3.1/go.mod h1:6ZKzEzIZuBQwoSvlKT+qpUfIbBfFCE5gbAoTg0/117g= +go.augendre.info/fatcontext v0.9.0 h1:Gt5jGD4Zcj8CDMVzjOJITlSb9cEch54hjRRlN3qDojE= +go.augendre.info/fatcontext v0.9.0/go.mod h1:L94brOAT1OOUNue6ph/2HnwxoNlds9aXDF2FcUntbNw= go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= @@ -1038,6 +1292,8 @@ go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689Cbtr go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs= +go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8= go.uber.org/dig v1.18.0 h1:imUL1UiY0Mg4bqbFfsRQO5G4CGRBec/ZujWTvSVp3pw= go.uber.org/dig v1.18.0/go.mod h1:Us0rSJiThwCv2GteUN0Q7OKvU7n5J4dxZ9JKUXozFdE= go.uber.org/fx v1.23.0 h1:lIr/gYWQGfTwGcSXWXu4vP5Ws6iqnNEIY+F/aFzCKTg= @@ -1055,72 +1311,55 @@ go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= -go.yaml.in/yaml/v3 v3.0.3 h1:bXOww4E/J3f66rav3pX3m8w6jDE4knZjGOw8b5Y6iNE= -go.yaml.in/yaml/v3 v3.0.3/go.mod h1:tBHosrYAkRZjRAOREWbDnBXUf08JOwYq++0QNwQiWzI= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200602180216-279210d13fed/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= -golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20220428152302-39d4317da171/go.mod h1:lgLbSvA5ygNOMpwM/9anMpWVlVJ7Z+cHWq/eFuinpGE= golang.org/x/exp v0.0.0-20251113190631-e25ba8c21ef6 h1:zfMcR1Cs4KNuomFFgGefv5N0czO2XZpUbxGUy8i8ug0= golang.org/x/exp v0.0.0-20251113190631-e25ba8c21ef6/go.mod h1:46edojNIoXTNOhySWIWdix628clX9ODXwPsQuG6hsK0= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/exp/typeparams v0.0.0-20220428152302-39d4317da171/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= +golang.org/x/exp/typeparams v0.0.0-20230203172020-98cc5a0785f9/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= +golang.org/x/exp/typeparams v0.0.0-20251023183803-a4bb9ffd2546 h1:HDjDiATsGqvuqvkDvgJjD1IgPrVekcSXVVE21JwvzGE= +golang.org/x/exp/typeparams v0.0.0-20251023183803-a4bb9ffd2546/go.mod h1:4Mzdyp/6jzw9auFDJ3OMF5qksa7UvPnzKqTVGcb04ms= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= +golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.31.0 h1:HaW9xtz0+kOcWKwli0ZXy79Ix+UW/vOfmWI5QVd2tgI= golang.org/x/mod v0.31.0/go.mod h1:43JraMp9cGx1Rx3AqioxrbrhNsLl2l/iNAvuBkrezpg= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1135,42 +1374,23 @@ golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= @@ -1178,13 +1398,6 @@ golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAG golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1192,13 +1405,13 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1209,58 +1422,36 @@ golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190316082340-a2f829d7f35f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200413165638-669c56c373c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211105183446-c75c47738b0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1269,6 +1460,8 @@ golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= @@ -1281,13 +1474,13 @@ golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= @@ -1295,13 +1488,12 @@ golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1311,57 +1503,29 @@ golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200329025819-fd4102a86c65/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= -golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20200724022722-7017fd6b1305/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.1-0.20210205202024-ef80cdb6ec6d/go.mod h1:9bzcO0MWcOuT0tm1iBGzDVPshzfwoVvREIui8C+MHqU= +golang.org/x/tools v0.1.1-0.20210302220138-2ac05c832e1a/go.mod h1:9bzcO0MWcOuT0tm1iBGzDVPshzfwoVvREIui8C+MHqU= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= +golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= golang.org/x/tools v0.40.0 h1:yLkxfA+Qnul4cs9QA3KnlFu0lVmd8JJfoq+E41uSutA= golang.org/x/tools v0.40.0/go.mod h1:Ik/tzLRlbscWpqqMRjyWYDisX8bG13FrdXp3o4Sr9lc= +golang.org/x/tools/go/expect v0.1.1-deprecated h1:jpBZDwmgPhXsKZC6WhL20P4b/wmnpsEAGHaNy0n/rJM= +golang.org/x/tools/go/expect v0.1.1-deprecated/go.mod h1:eihoPOH+FgIqa3FpoTwguz/bVUSGBlGQU67vpBeOrBY= +golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated h1:1h2MnaIAIXISqTFKdENegdpAgUXz6NrPEsbIeWaBRvM= +golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated/go.mod h1:RVAQXBGNv1ib0J382/DPCRS/BPnsGebyM1Gj5VSDpG8= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1372,74 +1536,17 @@ google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+ google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y= google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= -google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= -google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= -google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg= google.golang.org/genproto v0.0.0-20190306203927-b5d61aea6440/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto/googleapis/api v0.0.0-20251111163417-95abcf5c77ba h1:B14OtaXuMaCQsl2deSvNkyPKIzq3BjfxQp8d00QyWx4= google.golang.org/genproto/googleapis/api v0.0.0-20251111163417-95abcf5c77ba/go.mod h1:G5IanEx8/PgI9w6CFcYQf7jMtHQhZruvfM1i3qOqk5U= google.golang.org/genproto/googleapis/rpc v0.0.0-20251124214823-79d6a2a48846 h1:Wgl1rcDNThT+Zn47YyCXOXyX/COgMTIdhJ717F0l4xk= @@ -1449,20 +1556,6 @@ google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9M google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= -google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= -google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.77.0 h1:wVVY6/8cGA6vvffn+wWK5ToddbgdU3d8MNENr4evgXM= google.golang.org/grpc v1.77.0/go.mod h1:z0BY1iVj0q8E1uSQCjL9cppRj+gnZjzDnzV0dHhrNig= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.6.0 h1:6Al3kEFFP9VJhRz3DID6quisgPnTeZVr4lep9kkxdPA= @@ -1472,11 +1565,6 @@ google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= @@ -1488,6 +1576,8 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EV gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= @@ -1508,11 +1598,8 @@ grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJd honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.6.1 h1:R094WgE8K4JirYjBaOpz/AvTyUu/3wbmAoskKN/pxTI= +honnef.co/go/tools v0.6.1/go.mod h1:3puzxxljPCe8RGJX7BIy1plGbxEOZni5mR2aXe3/uk4= lukechampine.com/blake3 v1.3.0 h1:sJ3XhFINmHSrYCgl958hscfIa3bw8x4DqMP3u1YvoYE= lukechampine.com/blake3 v1.3.0/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1LM6k= modernc.org/cc/v4 v4.26.3 h1:yEN8dzrkRFnn4PUUKXLYIqVf2PJYAEjMTFjO3BDGc3I= @@ -1541,9 +1628,10 @@ modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +mvdan.cc/gofumpt v0.9.2 h1:zsEMWL8SVKGHNztrx6uZrXdp7AX8r421Vvp23sz7ik4= +mvdan.cc/gofumpt v0.9.2/go.mod h1:iB7Hn+ai8lPvofHd9ZFGVg2GOr8sBUw1QUWjNbmIL/s= +mvdan.cc/unparam v0.0.0-20251027182757-5beb8c8f8f15 h1:ssMzja7PDPJV8FStj7hq9IKiuiKhgz9ErWw+m68e7DI= +mvdan.cc/unparam v0.0.0-20251027182757-5beb8c8f8f15/go.mod h1:4M5MMXl2kW6fivUT6yRGpLLPNfuGtU2Z0cPvFquGDYU= sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck= diff --git a/p2p/discover/v5_udp.go b/p2p/discover/v5_udp.go index 9a5158b5d96..8702afcbcec 100644 --- a/p2p/discover/v5_udp.go +++ b/p2p/discover/v5_udp.go @@ -269,7 +269,7 @@ func (t *UDPv5) ResolveNodeId(id enode.ID) *enode.Node { func (t *UDPv5) AllNodes() []*enode.Node { t.tab.mutex.Lock() defer t.tab.mutex.Unlock() - nodes := make([]*enode.Node, 0) + nodes := make([]*enode.Node, 0, len(t.tab.buckets)*bucketSize) for _, b := range &t.tab.buckets { for _, n := range b.entries { diff --git a/tools/golangci_lint.sh b/tools/golangci_lint.sh deleted file mode 100755 index 05576b0e22f..00000000000 --- a/tools/golangci_lint.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -version="v2.8.0" - -if [[ "$1" == "--install-deps" ]] -then - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b "$(go env GOPATH)/bin" "$version" - exit -fi - -if ! which golangci-lint > /dev/null -then - echo "golangci-lint tool is not found, install it with:" - echo " make lint-deps" - echo "or follow https://golangci-lint.run/usage/install/" - exit 2 -fi - -GOEXPERIMENT=synctest golangci-lint run --config ./.golangci.yml --fast-only -GOEXPERIMENT=synctest golangci-lint run --config ./.golangci.yml From 02b257a3fb235a076dc611ba3b061837f72cc994 Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Mon, 9 Feb 2026 21:18:13 +1100 Subject: [PATCH 16/16] go mod tidy Go mod turds are so shit. --- go.mod | 1 - go.sum | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 4166ac87993..320849bedd3 100644 --- a/go.mod +++ b/go.mod @@ -163,7 +163,6 @@ require ( github.com/alfatraining/structtag v1.0.0 // indirect github.com/alingse/asasalint v0.0.11 // indirect github.com/alingse/nilnesserr v0.2.0 // indirect - github.com/alecthomas/repr v0.4.0 // indirect github.com/anacrolix/btree v0.1.1 // indirect github.com/anacrolix/chansync v0.7.0 // indirect github.com/anacrolix/dht/v2 v2.23.0 // indirect diff --git a/go.sum b/go.sum index 329244c6b59..374eda3b3bb 100644 --- a/go.sum +++ b/go.sum @@ -1451,6 +1451,7 @@ golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=