diff --git a/.gitignore b/.gitignore index fa632b8..9f016d4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ build .envrc - -*.txt \ No newline at end of file +.env +*.txt +.Ds_Store +.gocache diff --git a/go.mod b/go.mod index 4e1174a..9a2528f 100644 --- a/go.mod +++ b/go.mod @@ -22,6 +22,7 @@ require ( require ( dario.cat/mergo v1.0.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect + github.com/DataDog/zstd v1.5.5 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/Microsoft/hcsshim v0.11.4 // indirect github.com/benbjohnson/clock v1.3.5 // indirect @@ -29,6 +30,7 @@ require ( github.com/bits-and-blooms/bitset v1.20.0 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cockroachdb/fifo v0.0.0-20240616162244-4768e80dfb9a // indirect github.com/consensys/bavard v0.1.27 // indirect github.com/consensys/gnark-crypto v0.16.0 // indirect github.com/containerd/cgroups v1.1.0 // indirect @@ -38,7 +40,7 @@ require ( github.com/cpuguy83/dockercfg v0.3.1 // indirect github.com/crate-crypto/go-eth-kzg v1.3.0 // indirect github.com/crate-crypto/go-ipa v0.0.0-20240724233137-53bbb0ceb27a // indirect - github.com/davecgh/go-spew v1.1.1 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect github.com/deckarep/golang-set/v2 v2.6.0 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect @@ -53,6 +55,7 @@ require ( github.com/flynn/noise v1.1.0 // indirect github.com/francoispqt/gojay v1.2.13 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/getsentry/sentry-go v0.28.1 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.3.0 // indirect @@ -102,6 +105,7 @@ require ( github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-runewidth v0.0.16 // indirect github.com/mfridman/interpolate v0.0.2 // indirect github.com/miekg/dns v1.1.66 // indirect github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect @@ -151,7 +155,7 @@ require ( github.com/pion/transport/v3 v3.0.7 // indirect github.com/pion/turn/v4 v4.0.0 // indirect github.com/pion/webrtc/v4 v4.0.10 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/polydawn/refmt v0.89.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/prometheus/client_model v0.6.2 // indirect @@ -161,6 +165,7 @@ require ( github.com/quic-go/quic-go v0.50.1 // indirect github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66 // indirect github.com/raulk/go-watchdog v1.3.0 // indirect + github.com/rivo/uniseg v0.4.7 // indirect github.com/robfig/cron/v3 v3.0.1 // indirect github.com/sethvargo/go-retry v0.2.4 // indirect github.com/shirou/gopsutil v3.21.11+incompatible // indirect @@ -172,8 +177,8 @@ require ( github.com/spf13/cast v1.5.1 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/subosito/gotenv v1.4.2 // indirect - github.com/tklauser/go-sysconf v0.3.12 // indirect - github.com/tklauser/numcpus v0.6.1 // indirect + github.com/tklauser/go-sysconf v0.3.13 // indirect + github.com/tklauser/numcpus v0.7.0 // indirect github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect github.com/wlynxg/anet v0.0.5 // indirect github.com/yusufpapurcu/wmi v1.2.3 // indirect @@ -218,6 +223,6 @@ require ( github.com/pressly/goose/v3 v3.20.0 github.com/rs/zerolog v1.33.0 github.com/shutter-network/gnosh-contracts v0.4.0 - github.com/shutter-network/rolling-shutter/rolling-shutter v0.0.7-0.20241107145313-a3a0ae8e79e6 + github.com/shutter-network/rolling-shutter/rolling-shutter v0.0.7-0.20251208133535-1ceca5ea26d9 github.com/spf13/pflag v1.0.5 // indirect ) diff --git a/go.sum b/go.sum index cf92d34..2cdc710 100644 --- a/go.sum +++ b/go.sum @@ -51,8 +51,8 @@ github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOEl github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/DataDog/zstd v1.5.2 h1:vUG4lAyuPCXO0TLbXvPv7EB7cNK1QV/luu55UHLrrn8= -github.com/DataDog/zstd v1.5.2/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= +github.com/DataDog/zstd v1.5.5 h1:oWf5W7GtOLgp6bciQYDmhHHjdhYkALu6S/5Ni9ZgSvQ= +github.com/DataDog/zstd v1.5.5/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/Microsoft/hcsshim v0.11.4 h1:68vKo2VN8DE9AdN4tnkWnmdhqdbpUFM8OF3Airm7fz8= @@ -87,8 +87,8 @@ github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnht github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cockroachdb/errors v1.11.3 h1:5bA+k2Y6r+oz/6Z/RFlNeVCesGARKuC6YymtcDrbC/I= github.com/cockroachdb/errors v1.11.3/go.mod h1:m4UIW4CDjx+R5cybPsNrRbreomiFqt8o1h1wUVazSd8= -github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce h1:giXvy4KSc/6g/esnpM7Geqxka4WSqI1SZc7sMJFd3y4= -github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce/go.mod h1:9/y3cnZ5GKakj/H4y9r9GTjCvAFta7KLgSHPJJYc52M= +github.com/cockroachdb/fifo v0.0.0-20240616162244-4768e80dfb9a h1:f52TdbU4D5nozMAhO9TvTJ2ZMCXtN4VIAmfrrZ0JXQ4= +github.com/cockroachdb/fifo v0.0.0-20240616162244-4768e80dfb9a/go.mod h1:9/y3cnZ5GKakj/H4y9r9GTjCvAFta7KLgSHPJJYc52M= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= github.com/cockroachdb/pebble v1.1.2 h1:CUh2IPtR4swHlEj48Rhfzw6l/d0qA31fItcIszQVIsA= @@ -128,8 +128,9 @@ github.com/crate-crypto/go-kzg-4844 v1.1.0/go.mod h1:JolLjpSff1tCCJKaJx4psrlEdlX github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c h1:pFUpOrbxDR6AkioZ1ySsx5yxlDQZ8stG2b88gTPxgJU= github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c/go.mod h1:6UhI8N9EjYm1c2odKpFpAYeR8dsBeM7PtzQhRgxRr9U= github.com/deckarep/golang-set/v2 v2.6.0 h1:XfcQbWM1LlMB8BsJ8N9vW5ehnnPVIw0je80NsVHagjM= @@ -183,8 +184,8 @@ github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nos github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/gballet/go-libpcsclite v0.0.0-20191108122812-4678299bea08 h1:f6D9Hr8xV8uYKlyuj8XIruxlh9WjVjdh1gIicAS7ays= github.com/gballet/go-libpcsclite v0.0.0-20191108122812-4678299bea08/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= -github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps= -github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= +github.com/getsentry/sentry-go v0.28.1 h1:zzaSm/vHmGllRM6Tpx1492r0YDzauArdBfkJRtY6P5k= +github.com/getsentry/sentry-go v0.28.1/go.mod h1:1fQZ+7l7eeJ3wYi82q5Hg8GqAPgefRq+FP/QhafYVgg= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/go-co-op/gocron/v2 v2.11.0 h1:IOowNA6SzwdRFnD4/Ol3Kj6G2xKfsoiiGq2Jhhm9bvE= @@ -441,8 +442,8 @@ github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/ github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= -github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= +github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6BbAxPY= github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg= @@ -581,8 +582,9 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/polydawn/refmt v0.89.0 h1:ADJTApkvkeBZsN0tBTx8QjpD9JkmxbKp0cxfr9qszm4= github.com/polydawn/refmt v0.89.0/go.mod h1:/zvteZs/GwLtCgZ4BL6CBsk9IKIlexP43ObX9AxTqTw= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= @@ -612,8 +614,9 @@ github.com/raulk/go-watchdog v1.3.0 h1:oUmdlHxdkXRJlwfG0O9omj8ukerm8MEQavSiDTEtB github.com/raulk/go-watchdog v1.3.0/go.mod h1:fIvOnLbF0b0ZwkB9YU4mOW9Did//4vPZtDqv66NfsMU= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -github.com/rivo/uniseg v0.4.3 h1:utMvzDsuh3suAEnhH0RdHmoPbU648o6CvXxTx4SBMOw= -github.com/rivo/uniseg v0.4.3/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -665,8 +668,8 @@ github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYED github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw= github.com/shutter-network/gnosh-contracts v0.4.0 h1:2GJcHK9w4lJZMsccklmxDhNnrkRLJDRwsL1acBnAeak= github.com/shutter-network/gnosh-contracts v0.4.0/go.mod h1:QB0d64ybbVFKMrLjrc1tldri87KNjTmKQjhk9jaso2E= -github.com/shutter-network/rolling-shutter/rolling-shutter v0.0.7-0.20241107145313-a3a0ae8e79e6 h1:wi1BDjwFi61xgD6q36/Obm7El5YoCTYfFcyRHQ27TgU= -github.com/shutter-network/rolling-shutter/rolling-shutter v0.0.7-0.20241107145313-a3a0ae8e79e6/go.mod h1:0ghSBsSWZ2CkUSgBEfNkV9h2J8Z1rIYZt1TD1+NznQA= +github.com/shutter-network/rolling-shutter/rolling-shutter v0.0.7-0.20251208133535-1ceca5ea26d9 h1:AxV/M1mTOBaf2XlfRg3aCroBtEwnd4IF26ehuBwY5ZE= +github.com/shutter-network/rolling-shutter/rolling-shutter v0.0.7-0.20251208133535-1ceca5ea26d9/go.mod h1:U01tVxIpZ1okPKkmjY9XqTiNzVQxC47u1waIPIKtx60= github.com/shutter-network/shutter/shlib v0.1.19 h1:Fm/PnxAapl5UK4JuJyF2wBh3dYB5ESztnNFcbmzNNOI= github.com/shutter-network/shutter/shlib v0.1.19/go.mod h1:miY10OJ0FKjLZTwvmG0AJNijsjsNDAk6l1tROTj8uzU= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= @@ -715,10 +718,12 @@ github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a/go.mod h1:RRCYJ github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= github.com/testcontainers/testcontainers-go v0.31.0 h1:W0VwIhcEVhRflwL9as3dhY6jXjVCA27AkmbnZ+UTh3U= github.com/testcontainers/testcontainers-go v0.31.0/go.mod h1:D2lAoA0zUFiSY+eAflqK5mcUx/A5hrrORaEQrd0SefI= -github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= -github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= +github.com/tklauser/go-sysconf v0.3.13 h1:GBUpcahXSpR2xN01jhkNAbTLRk2Yzgggk8IM08lq3r4= +github.com/tklauser/go-sysconf v0.3.13/go.mod h1:zwleP4Q4OehZHGn4CYZDipCgg9usW5IJePewFCGVEa0= github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= +github.com/tklauser/numcpus v0.7.0 h1:yjuerZP127QG9m5Zh/mSO4wqurYil27tHrqwRoRjpr4= +github.com/tklauser/numcpus v0.7.0/go.mod h1:bb6dMVcj8A42tSE7i32fsIUCbQNllK5iDguyOZRUzAY= github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.10/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.12 h1:igJgVw1JdKH+trcLWLeLwZjU9fEfPesQ+9/e4MQ44S8= diff --git a/internal/metrics/tx_mapper_db.go b/internal/metrics/tx_mapper_db.go index d77c4df..bc0c112 100644 --- a/internal/metrics/tx_mapper_db.go +++ b/internal/metrics/tx_mapper_db.go @@ -289,6 +289,38 @@ func (tm *TxMapperDB) UpdateValidatorStatus(ctx context.Context) error { break } + // Collect all validator indices from the batch + validatorIndices := make([]int64, 0, len(validatorStatus)) + for _, vs := range validatorStatus { + validatorIndex := uint64(vs.ValidatorIndex.Int64) + validatorIndices = append(validatorIndices, int64(validatorIndex)) + } + + // Fetch all validators, splitting into chunks of 64 to respect API limit + const maxIndicesPerRequest = 64 + validatorDataMap := make(map[uint64]*beaconapiclient.ValidatorData) + + for i := 0; i < len(validatorIndices); i += maxIndicesPerRequest { + end := i + maxIndicesPerRequest + if end > len(validatorIndices) { + end = len(validatorIndices) + } + + chunk := validatorIndices[i:end] + validators, err := tm.beaconAPIClient.GetValidatorByIndices(ctx, "head", chunk) + if err != nil { + log.Err(err).Msg("failed to get validators from beacon chain for chunk, skipping this chunk") + continue + } + + if validators != nil { + for j := range validators.Data { + v := &validators.Data[j] + validatorDataMap[uint64(v.Index)] = v + } + } + } + // Launch goroutines to process each status concurrently for _, vs := range validatorStatus { sem <- struct{}{} @@ -300,18 +332,15 @@ func (tm *TxMapperDB) UpdateValidatorStatus(ctx context.Context) error { validatorIndex := uint64(vs.ValidatorIndex.Int64) //TODO: should we keep this log or remove it? log.Debug().Uint64("validatorIndex", validatorIndex).Msg("validator status being updated") - validator, err := tm.beaconAPIClient.GetValidatorByIndex(ctx, "head", validatorIndex) - if err != nil { - log.Err(err).Uint64("validatorIndex", validatorIndex).Msg("failed to get validator from beacon chain") - return - } - if validator == nil { + validatorData, ok := validatorDataMap[validatorIndex] + if !ok { + log.Debug().Uint64("validatorIndex", validatorIndex).Msg("failed to get validator from beacon chain") return } - err = tm.dbQuery.CreateValidatorStatus(ctx, data.CreateValidatorStatusParams{ + err := tm.dbQuery.CreateValidatorStatus(ctx, data.CreateValidatorStatusParams{ ValidatorIndex: dbTypes.Uint64ToPgTypeInt8(validatorIndex), - Status: validator.Data.Status, + Status: validatorData.Status, }) if err != nil { log.Err(err).Uint64("validatorIndex", validatorIndex).Msg("failed to create validator status") @@ -574,9 +603,12 @@ func (tm *TxMapperDB) validateValidatorRegistryEvent( staticRegistrationMessageValidity := validateValidatorRegistryMessageContents(regMessage, chainID, validatorRegistryContractAddress) var publicKeys []*blst.P1Affine - var validators []*beaconapiclient.GetValidatorByIndexResponse + var validatorsResponse *beaconapiclient.GetValidatorByIndexResponse validatorIDtoValidity := make(map[int64]*validatorData) + // collect all validator indices that pass nonce validation + validValidatorIndices := make([]int64, 0) + for _, validatorIndex := range regMessage.ValidatorIndices() { validatorIDtoValidity[validatorIndex] = &validatorData{validatorValidity: staticRegistrationMessageValidity} nonceBefore, err := tm.dbQuery.QueryValidatorRegistrationMessageNonceBefore(ctx, data.QueryValidatorRegistrationMessageNonceBeforeParams{ @@ -604,23 +636,65 @@ func (tm *TxMapperDB) validateValidatorRegistryEvent( validatorIDtoValidity[validatorIndex].validatorValidity = data.ValidatorRegistrationValidityInvalidmessage continue } - validator, err := tm.beaconAPIClient.GetValidatorByIndex(ctx, "head", uint64(validatorIndex)) - if err != nil { - return nil, errors.Wrapf(err, "failed to get validator %d", validatorIndex) - } - if validator == nil { - // validator not found - log.Warn().Msg("registration message for unknown validator") - validatorIDtoValidity[validatorIndex].validatorValidity = data.ValidatorRegistrationValidityInvalidmessage - continue + validValidatorIndices = append(validValidatorIndices, int64(validatorIndex)) + } + + // Fetch all validators, splitting into chunks of 64 to respect API limit + if len(validValidatorIndices) > 0 { + const maxIndicesPerRequest = 64 + var allValidatorData []beaconapiclient.ValidatorData + + for i := 0; i < len(validValidatorIndices); i += maxIndicesPerRequest { + end := i + maxIndicesPerRequest + if end > len(validValidatorIndices) { + end = len(validValidatorIndices) + } + + chunk := validValidatorIndices[i:end] + chunkResponse, err := tm.beaconAPIClient.GetValidatorByIndices(ctx, "head", chunk) + if err != nil { + return nil, errors.Wrapf(err, "failed to get validators for chunk starting at index %d", i) + } + + if chunkResponse != nil && len(chunkResponse.Data) > 0 { + allValidatorData = append(allValidatorData, chunkResponse.Data...) + } } - validatorIDtoValidity[validatorIndex].validatorStatus = validator.Data.Status - publicKey, err := validator.Data.Validator.GetPubkey() - if err != nil { - return nil, errors.Wrapf(err, "failed to get public key of validator %d", validatorIndex) + + if len(allValidatorData) == 0 { + for _, validatorIndex := range validValidatorIndices { + validatorIDtoValidity[validatorIndex].validatorValidity = data.ValidatorRegistrationValidityInvalidmessage + } + } else { + // Create a combined response object + validatorsResponse = &beaconapiclient.GetValidatorByIndexResponse{ + Data: allValidatorData, + } + + // Create a map from validator index to validator data for quick lookup + validatorDataMap := make(map[int64]*beaconapiclient.ValidatorData) + for i := range validatorsResponse.Data { + v := &validatorsResponse.Data[i] + validatorDataMap[int64(v.Index)] = v + } + + // Process each validator from the response + for _, validatorIndex := range validValidatorIndices { + validatorData, ok := validatorDataMap[validatorIndex] + if !ok { + // validator not found + log.Warn().Int64("validatorIndex", validatorIndex).Msg("registration message for unknown validator") + validatorIDtoValidity[validatorIndex].validatorValidity = data.ValidatorRegistrationValidityInvalidmessage + continue + } + validatorIDtoValidity[validatorIndex].validatorStatus = validatorData.Status + publicKey, err := validatorData.Validator.GetPubkey() + if err != nil { + return nil, errors.Wrapf(err, "failed to get public key of validator %d", validatorIndex) + } + publicKeys = append(publicKeys, publicKey) + } } - publicKeys = append(publicKeys, publicKey) - validators = append(validators, validator) } if len(publicKeys) > 0 { // now we need to check for signature verification depending on the message version @@ -636,13 +710,17 @@ func (tm *TxMapperDB) validateValidatorRegistryEvent( return nil, errors.Wrapf(err, "failed to unmarshal legacy registration message") } if valid := validatorregistry.VerifySignature(sig, publicKeys[0], regMessage); !valid { - validatorIDtoValidity[int64(validators[0].Data.Index)].validatorValidity = data.ValidatorRegistrationValidityInvalidsignature + if validatorsResponse != nil { + validatorIDtoValidity[int64(validatorsResponse.Data[0].Index)].validatorValidity = data.ValidatorRegistrationValidityInvalidsignature + } log.Warn().Msg("invalid legacy registration message with invalid signature") } } else { if valid := validatorregistry.VerifyAggregateSignature(sig, publicKeys, regMessage); !valid { - for _, validator := range validators { - validatorIDtoValidity[int64(validator.Data.Index)].validatorValidity = data.ValidatorRegistrationValidityInvalidsignature + if validatorsResponse != nil { + for _, validatorData := range validatorsResponse.Data { + validatorIDtoValidity[int64(validatorData.Index)].validatorValidity = data.ValidatorRegistrationValidityInvalidsignature + } } log.Warn().Msg("invalid aggregate registration message with invalid signature") } diff --git a/tests/validator_test.go b/tests/validator_test.go index 66409df..ec668a6 100644 --- a/tests/validator_test.go +++ b/tests/validator_test.go @@ -137,11 +137,13 @@ func mockBeaconClient(t *testing.T, pubKeyHex string, status string) string { return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { x := beaconapiclient.GetValidatorByIndexResponse{ Finalized: true, - Data: beaconapiclient.ValidatorData{ - Validator: beaconapiclient.Validator{ - PubkeyHex: pubKeyHex, + Data: []beaconapiclient.ValidatorData{ + { + Validator: beaconapiclient.Validator{ + PubkeyHex: pubKeyHex, + }, + Status: status, }, - Status: status, }, } res, err := json.Marshal(x)