Add EC / BN client logging and better request timeouts

beacon/client/std-client.go

@@ -586,3 +586,8 @@ func (c *StandardClient) getValidatorsByOpts(ctx context.Context, pubkeysOrIndic
 
 	return ValidatorsResponse{Data: trueData}, nil
 }
+
+// Get an eth2 epoch number by time
+func epochAt(config beacon.Eth2Config, time uint64) uint64 {
+	return config.GenesisEpoch + (time-config.GenesisTime)/config.SecondsPerEpoch
+}

beacon/client/std-http-client.go

@@ -2,14 +2,37 @@ package client
 
 import "time"
 
+// Options for the standard HTTP client
+type StandardHttpClientOpts struct {
+	// The time to wait for a request that is expected to return quickly
+	FastTimeout time.Duration
+
+	// The time to wait for a request that is expected to take a lot of processing on the BN and return slowly
+	SlowTimeout time.Duration
+}
+
+// Standard high-level client for interacting with a Beacon Node over HTTP
 type StandardHttpClient struct {
 	*StandardClient
 }
 
-// Create a new client instance
-func NewStandardHttpClient(providerAddress string, timeout time.Duration) *StandardHttpClient {
-	provider := NewBeaconHttpProvider(providerAddress, timeout)
+// Create a new client instance.
+func NewStandardHttpClient(providerAddress string, opts *StandardHttpClientOpts) (*StandardHttpClient, error) {
+	var provider *BeaconHttpProvider
+	var err error
+	if opts != nil {
+		provider, err = NewBeaconHttpProvider(providerAddress, &BeaconHttpProviderOpts{
+			DefaultFastTimeout: opts.FastTimeout,
+			DefaultSlowTimeout: opts.SlowTimeout,
+		})
+	} else {
+		provider, err = NewBeaconHttpProvider(providerAddress, nil)
+	}
+	if err != nil {
+		return nil, err
+	}
+
 	return &StandardHttpClient{
 		StandardClient: NewStandardClient(provider),
-	}
+	}, nil
 }

config/external-beacon-config.go

@@ -14,6 +14,12 @@ type ExternalBeaconConfig struct {
 
 	// The URL of the Prysm gRPC endpoint (only needed if using Prysm VCs)
 	PrysmRpcUrl Parameter[string]
+
+	// Number of milliseconds to wait for a fast request to complete
+	FastTimeoutMs Parameter[uint64]
+
+	// Number of milliseconds to wait for a slow request to complete
+	SlowTimeoutMs Parameter[uint64]
 }
 
 // Generates a new ExternalBeaconConfig configuration
@@ -92,6 +98,34 @@ func NewExternalBeaconConfig() *ExternalBeaconConfig {
 				Network_All: "",
 			},
 		},
+
+		FastTimeoutMs: Parameter[uint64]{
+			ParameterCommon: &ParameterCommon{
+				ID:                 ids.FastTimeoutID,
+				Name:               "Fast Timeout",
+				Description:        "Number of milliseconds to wait for a request to complete that is expected to be fast and light before timing out the request.",
+				AffectsContainers:  []ContainerID{ContainerID_Daemon},
+				CanBeBlank:         false,
+				OverwriteOnUpgrade: false,
+			},
+			Default: map[Network]uint64{
+				Network_All: 5000,
+			},
+		},
+
+		SlowTimeoutMs: Parameter[uint64]{
+			ParameterCommon: &ParameterCommon{
+				ID:                 ids.SlowTimeoutID,
+				Name:               "Slow Timeout",
+				Description:        "Number of milliseconds to wait for a request to complete that is expected to be slow and heavy, either taking a long time to process or returning a large amount of data, before timing out the request. Examples include querying the Beacon Node for the state of a large number of validators.",
+				AffectsContainers:  []ContainerID{ContainerID_Daemon},
+				CanBeBlank:         false,
+				OverwriteOnUpgrade: false,
+			},
+			Default: map[Network]uint64{
+				Network_All: 30000,
+			},
+		},
 	}
 }
 
@@ -106,6 +140,8 @@ func (cfg *ExternalBeaconConfig) GetParameters() []IParameter {
 		&cfg.BeaconNode,
 		&cfg.HttpUrl,
 		&cfg.PrysmRpcUrl,
+		&cfg.FastTimeoutMs,
+		&cfg.SlowTimeoutMs,
 	}
 }
 

config/external-execution-config.go

@@ -14,6 +14,12 @@ type ExternalExecutionConfig struct {
 
 	// The URL of the Websocket endpoint
 	WebsocketUrl Parameter[string]
+
+	// Number of milliseconds to wait for a fast request to complete
+	FastTimeoutMs Parameter[uint64]
+
+	// Number of milliseconds to wait for a slow request to complete
+	SlowTimeoutMs Parameter[uint64]
 }
 
 // Generates a new ExternalExecutionConfig configuration
@@ -85,6 +91,34 @@ func NewExternalExecutionConfig() *ExternalExecutionConfig {
 				Network_All: "",
 			},
 		},
+
+		FastTimeoutMs: Parameter[uint64]{
+			ParameterCommon: &ParameterCommon{
+				ID:                 ids.FastTimeoutID,
+				Name:               "Fast Timeout",
+				Description:        "Number of milliseconds to wait for a request to complete that is expected to be fast and light before timing out the request.",
+				AffectsContainers:  []ContainerID{ContainerID_Daemon},
+				CanBeBlank:         false,
+				OverwriteOnUpgrade: false,
+			},
+			Default: map[Network]uint64{
+				Network_All: 5000,
+			},
+		},
+
+		SlowTimeoutMs: Parameter[uint64]{
+			ParameterCommon: &ParameterCommon{
+				ID:                 ids.SlowTimeoutID,
+				Name:               "Slow Timeout",
+				Description:        "Number of milliseconds to wait for a request to complete that is expected to be slow and heavy, either taking a long time to process or returning a large amount of data, before timing out the request. Examples include filtering through Ethereum event logs.",
+				AffectsContainers:  []ContainerID{ContainerID_Daemon},
+				CanBeBlank:         false,
+				OverwriteOnUpgrade: false,
+			},
+			Default: map[Network]uint64{
+				Network_All: 30000,
+			},
+		},
 	}
 }
 
@@ -99,6 +133,8 @@ func (cfg *ExternalExecutionConfig) GetParameters() []IParameter {
 		&cfg.ExecutionClient,
 		&cfg.HttpUrl,
 		&cfg.WebsocketUrl,
+		&cfg.FastTimeoutMs,
+		&cfg.SlowTimeoutMs,
 	}
 }
 

config/fallback-config.go

@@ -15,6 +15,9 @@ type FallbackConfig struct {
 
 	// The URL of the Prysm gRPC endpoint (only needed if using Prysm VCs)
 	PrysmRpcUrl Parameter[string]
+
+	// The delay in milliseconds when checking a client again after it disconnects during a request
+	ReconnectDelayMs Parameter[uint64]
 }
 
 // Generates a new FallbackConfig configuration
@@ -75,6 +78,20 @@ func NewFallbackConfig() *FallbackConfig {
 				Network_All: "",
 			},
 		},
+
+		ReconnectDelayMs: Parameter[uint64]{
+			ParameterCommon: &ParameterCommon{
+				ID:                 ids.FallbackReconnectDelayID,
+				Name:               "Reconnect Delay",
+				Description:        "The delay, in milliseconds, to wait after the primary Execution Client or primary Beacon Node disconnects during a request before trying it again.",
+				AffectsContainers:  []ContainerID{ContainerID_Daemon},
+				CanBeBlank:         false,
+				OverwriteOnUpgrade: false,
+			},
+			Default: map[Network]uint64{
+				Network_All: 60000,
+			},
+		},
 	}
 }
 
@@ -90,6 +107,7 @@ func (cfg *FallbackConfig) GetParameters() []IParameter {
 		&cfg.EcHttpUrl,
 		&cfg.BnHttpUrl,
 		&cfg.PrysmRpcUrl,
+		&cfg.ReconnectDelayMs,
 	}
 }
 

config/iconfig.go

@@ -1,6 +1,22 @@
 package config
 
-import "github.com/rocket-pool/node-manager-core/log"
+import (
+	"time"
+
+	"github.com/rocket-pool/node-manager-core/log"
+)
+
+// Timeout settings for the client
+type ClientTimeouts struct {
+	// The timeout for requests that are expected to be fast
+	FastTimeout time.Duration
+
+	// The timeout for requests that are expected to be slow and require either significant processing or a large return size from the server
+	SlowTimeout time.Duration
+
+	// The delay before rechecking the primary client, if fallbacks support is enabled
+	RecheckDelay time.Duration
+}
 
 // NMC servers typically provide some kind of persistent configuration; it must implement this interface.
 type IConfig interface {
@@ -27,9 +43,15 @@ type IConfig interface {
 	// The URLs for the Execution clients to use
 	GetExecutionClientUrls() (string, string)
 
+	// The timeouts for the Execution clients and manager to use
+	GetExecutionClientTimeouts() ClientTimeouts
+
 	// The URLs for the Beacon nodes to use
 	GetBeaconNodeUrls() (string, string)
 
+	// The timeouts for the Beacon nodes and manager to use
+	GetBeaconNodeTimeouts() ClientTimeouts
+
 	// The configuration for the daemon loggers
 	GetLoggerOptions() log.LoggerOptions
 }

config/ids/ids.go

@@ -11,6 +11,8 @@ const (
 	PortID                  string = "port"
 	OpenPortID              string = "openPort"
 	HttpUrlID               string = "httpUrl"
+	FastTimeoutID           string = "fastTimeout"
+	SlowTimeoutID           string = "slowTimeout"
 	EcID                    string = "executionClient"
 	BnID                    string = "beaconNode"
 	GraffitiID              string = "graffiti"
@@ -19,14 +21,15 @@ const (
 	CacheSizeID             string = "cacheSize"
 
 	// Logger
-	LoggerLevelID      string = "level"
-	LoggerFormatID     string = "format"
-	LoggerAddSourceID  string = "addSource"
-	LoggerMaxSizeID    string = "maxSize"
-	LoggerMaxBackupsID string = "maxBackups"
-	LoggerMaxAgeID     string = "maxAge"
-	LoggerLocalTimeID  string = "localTime"
-	LoggerCompressID   string = "compress"
+	LoggerLevelID             string = "level"
+	LoggerFormatID            string = "format"
+	LoggerAddSourceID         string = "addSource"
+	LoggerMaxSizeID           string = "maxSize"
+	LoggerMaxBackupsID        string = "maxBackups"
+	LoggerMaxAgeID            string = "maxAge"
+	LoggerLocalTimeID         string = "localTime"
+	LoggerCompressID          string = "compress"
+	LoggerEnableHttpTracingID string = "enableHttpTracing"
 
 	// Besu
 	BesuJvmHeapSizeID   string = "jvmHeapSize"
@@ -48,6 +51,7 @@ const (
 	FallbackUseFallbackClientsID string = "useFallbackClients"
 	FallbackEcHttpUrlID          string = "ecHttpUrl"
 	FallbackBnHttpUrlID          string = "bnHttpUrl"
+	FallbackReconnectDelayID     string = "reconnectDelay"
 
 	// Geth
 	GethEvmTimeoutID  string = "evmTimeout"

config/local-beacon-config.go

@@ -23,6 +23,12 @@ type LocalBeaconConfig struct {
 	// Toggle for forwarding the HTTP API port outside of Docker
 	OpenHttpPort Parameter[RpcPortMode]
 
+	// Number of milliseconds to wait for a fast request to complete
+	FastTimeoutMs Parameter[uint64]
+
+	// Number of milliseconds to wait for a slow request to complete
+	SlowTimeoutMs Parameter[uint64]
+
 	// Subconfigs
 	Lighthouse *LighthouseBnConfig
 	Lodestar   *LodestarBnConfig
@@ -138,6 +144,34 @@ func NewLocalBeaconConfig() *LocalBeaconConfig {
 				Network_All: RpcPortMode_Closed,
 			},
 		},
+
+		FastTimeoutMs: Parameter[uint64]{
+			ParameterCommon: &ParameterCommon{
+				ID:                 ids.FastTimeoutID,
+				Name:               "Fast Timeout",
+				Description:        "Number of milliseconds to wait for a request to complete that is expected to be fast and light before timing out the request.",
+				AffectsContainers:  []ContainerID{ContainerID_Daemon},
+				CanBeBlank:         false,
+				OverwriteOnUpgrade: false,
+			},
+			Default: map[Network]uint64{
+				Network_All: 5000,
+			},
+		},
+
+		SlowTimeoutMs: Parameter[uint64]{
+			ParameterCommon: &ParameterCommon{
+				ID:                 ids.SlowTimeoutID,
+				Name:               "Slow Timeout",
+				Description:        "Number of milliseconds to wait for a request to complete that is expected to be slow and heavy, either taking a long time to process or returning a large amount of data, before timing out the request. Examples include querying the Beacon Node for the state of a large number of validators.",
+				AffectsContainers:  []ContainerID{ContainerID_Daemon},
+				CanBeBlank:         false,
+				OverwriteOnUpgrade: false,
+			},
+			Default: map[Network]uint64{
+				Network_All: 30000,
+			},
+		},
 	}
 
 	cfg.Lighthouse = NewLighthouseBnConfig()
@@ -162,6 +196,8 @@ func (cfg *LocalBeaconConfig) GetParameters() []IParameter {
 		&cfg.P2pPort,
 		&cfg.HttpPort,
 		&cfg.OpenHttpPort,
+		&cfg.FastTimeoutMs,
+		&cfg.SlowTimeoutMs,
 	}
 }
 

config/local-execution-config.go

@@ -26,6 +26,12 @@ type LocalExecutionConfig struct {
 	// P2P traffic port
 	P2pPort Parameter[uint16]
 
+	// Number of milliseconds to wait for a fast request to complete
+	FastTimeoutMs Parameter[uint64]
+
+	// Number of milliseconds to wait for a slow request to complete
+	SlowTimeoutMs Parameter[uint64]
+
 	// Subconfigs
 	Geth       *GethConfig
 	Nethermind *NethermindConfig
@@ -146,6 +152,34 @@ func NewLocalExecutionConfig() *LocalExecutionConfig {
 				Network_All: 30303,
 			},
 		},
+
+		FastTimeoutMs: Parameter[uint64]{
+			ParameterCommon: &ParameterCommon{
+				ID:                 ids.FastTimeoutID,
+				Name:               "Fast Timeout",
+				Description:        "Number of milliseconds to wait for a request to complete that is expected to be fast and light before timing out the request.",
+				AffectsContainers:  []ContainerID{ContainerID_Daemon},
+				CanBeBlank:         false,
+				OverwriteOnUpgrade: false,
+			},
+			Default: map[Network]uint64{
+				Network_All: 5000,
+			},
+		},
+
+		SlowTimeoutMs: Parameter[uint64]{
+			ParameterCommon: &ParameterCommon{
+				ID:                 ids.SlowTimeoutID,
+				Name:               "Slow Timeout",
+				Description:        "Number of milliseconds to wait for a request to complete that is expected to be slow and heavy, either taking a long time to process or returning a large amount of data, before timing out the request. Examples include filtering through Ethereum event logs.",
+				AffectsContainers:  []ContainerID{ContainerID_Daemon},
+				CanBeBlank:         false,
+				OverwriteOnUpgrade: false,
+			},
+			Default: map[Network]uint64{
+				Network_All: 30000,
+			},
+		},
 	}
 
 	// Create the subconfigs
@@ -171,6 +205,8 @@ func (cfg *LocalExecutionConfig) GetParameters() []IParameter {
 		&cfg.EnginePort,
 		&cfg.OpenApiPorts,
 		&cfg.P2pPort,
+		&cfg.FastTimeoutMs,
+		&cfg.SlowTimeoutMs,
 	}
 }