feat(ethstats): add ethstats CLI subcommand with per-node credential forwarding

.gitignore

@@ -18,3 +18,4 @@ sage.yaml
 relay-monitor.yaml
 cities1000.txt
 countries.txt
+ethstats.yaml

README.md

@@ -11,37 +11,36 @@ Ethereum network monitoring with collection clients and a centralized server for
 Xatu can run in multiple modes. Each mode can be run independently. The following diagram shows the different modes and how they interact with each other and other services.
 
 ```
-
              ┌───────────┐
              │ CONSENSUS │
              │P2P NETWORK│
              └─────▲─────┘
                    │
       ┌────────────┘─────────────┐
       │                          │
-┌─────▲─────┐              ┌─────▲─────┐       ┌───────────┐
-│ CONSENSUS │              │ ARMIARMA  │       │ EXECUTION │
-│   CLIENT  ◄─────┐        │           │       │P2P NETWORK│
-└─────▲─────┘     │        └─────▲─────┘       └─────▲─────┘
-      │           │              │             ┌─────┘───────┐
-      │           │              │             │             │
-  ┌───▼────┐ ┌────▼─────┐  ┌─────▼─────┐ ┌─────▼─────┐ ┌─────▼─────┐
-  │  XATU  │ │   XATU   │  │   XATU    │ │   XATU    │ │   XATU    │
-  │ SENTRY │ │  CANNON  │  │   SAGE    │ │  MIMICRY  │ │ DISCOVERY │
-  └───┬────┘ └─────┬────┘  └─────┬─────┘ └─────┬─────┘ └─────┬─────┘
-      │            │             │             │             │
-      │            │             │             │             │
-      │       ┌────▼─────┐       │             │             │
-      └───────►          ◄───────┘─────────────┘─────────────┘
-              │   XATU   │
+┌─────▲─────┐              ┌─────▲─────┐       ┌───────────┐        ┌───────────┐
+│ CONSENSUS │              │ ARMIARMA  │       │ EXECUTION ◄────────| EXECUTION |
+│   CLIENT  ◄─────┐        │           │       │P2P NETWORK│        |   CLIENT  |
+└─────▲─────┘     │        └─────▲─────┘       └─────▲─────┘        └─────┬─────┘
+      │           │              │             ┌─────┘───────┐            │
+      │           │              │             │             │            │
+  ┌───▼────┐ ┌────▼─────┐  ┌─────▼─────┐ ┌─────▼─────┐ ┌─────▼─────┐┌─────▼────┐
+  │  XATU  │ │   XATU   │  │   XATU    │ │   XATU    │ │   XATU    ││   XATU   │
+  │ SENTRY │ │  CANNON  │  │   SAGE    │ │  MIMICRY  │ │ DISCOVERY ││ ETHSTATS │
+  └───┬────┘ └─────┬────┘  └─────┬─────┘ └─────┬─────┘ └─────┬─────┘└─────┬────┘
+      │            │             │             │             │            │
+      │            │             │             │             │            │
+      │       ┌────▼─────┐       │             │             │            │
+      └───────►          ◄───────┘─────────────┘─────────────┘────────────┘
+              │   XATU   │                   
               │  SERVER  │    ┌─────────────┐
               │          ◄────► PERSISTENCE │
               │          │    └─────────────┘
-              └─────┬────┘
-                    │
-                    │
-                    ▼
-              DATA PIPELINE
+              └─────┬────┘                   
+                    │                        
+                    │                        
+                    ▼                        
+              DATA PIPELINE                  
 ```
 
 ### Modes
@@ -53,6 +52,7 @@ Follow the links for more information on each mode.
 - [**Discovery**](./docs/discovery.md) - Client that uses the [Node Discovery Protocol v5](https://github.com/ethereum/devp2p/blob/master/discv5/discv5.md) and [Node Discovery Protocol v4](https://github.com/ethereum/devp2p/blob/master/discv4.md) to discovery nodes on the network. Also attempts to connect to execution layer nodes and collect meta data from them.
 - [**Mimicry**](./docs/mimicry.md) - Client that collects data from the execution layer P2P network.
 - [**Cannon**](./docs/cannon.md) - Client that runs along side a [Ethereum consensus client](https://ethereum.org/en/developers/docs/nodes-and-clients/#consensus-clients) and collects canonical finalized data via the consensus client's [Beacon API](https://ethereum.github.io/beacon-APIs/). _You must run your own consensus client_ and this projects cannon client will connect to it via the consensus client's http server.
+- [**Ethstats**](./docs/ethstats.md) - Server that receives data from Ethereum execution clients via the ethstats protocol and forwards events to configured output sinks. Supports per-node credential forwarding where clients connect with `nodename:base64(user:pass)@server:port`.
 
 ## Getting Started
 
@@ -61,7 +61,7 @@ Follow the links for more information on each mode.
 Download the latest release from the [Releases page](https://github.com/ethpandaops/xatu/releases). Extract and run with:
 
 ```
-./xatu <server|sentry|discovery|mimicry> --config your-config.yaml
+./xatu <server|sentry|discovery|mimicry|ethstats> --config your-config.yaml
 ```
 
 ### Install via bash script

cmd/ethstats.go

@@ -0,0 +1,152 @@
+//nolint:dupl // disable duplicate code warning for cmds
+package cmd
+
+import (
+	"os"
+
+	"github.com/creasty/defaults"
+	"github.com/ethpandaops/xatu/pkg/ethstats"
+	"github.com/spf13/cobra"
+	yaml "gopkg.in/yaml.v3"
+)
+
+var (
+	ethstatsCfgFile string
+)
+
+type EthstatsOverride struct {
+	FlagHelper func(cmd *cobra.Command)
+	Setter     func(cmd *cobra.Command, overrides *ethstats.Override) error
+}
+
+type EthstatsOverrideConfig struct {
+	FlagName     string
+	EnvName      string
+	Description  string
+	OverrideFunc func(val string, overrides *ethstats.Override)
+}
+
+func createEthstatsOverride(config EthstatsOverrideConfig) EthstatsOverride {
+	return EthstatsOverride{
+		FlagHelper: func(cmd *cobra.Command) {
+			cmd.Flags().String(config.FlagName, "", config.Description+` (env: `+config.EnvName+`)`)
+		},
+		Setter: func(cmd *cobra.Command, overrides *ethstats.Override) error {
+			val := ""
+
+			if cmd.Flags().Changed(config.FlagName) {
+				val = cmd.Flags().Lookup(config.FlagName).Value.String()
+			}
+
+			if os.Getenv(config.EnvName) != "" {
+				val = os.Getenv(config.EnvName)
+			}
+
+			if val == "" {
+				return nil
+			}
+
+			config.OverrideFunc(val, overrides)
+
+			return nil
+		},
+	}
+}
+
+var EthstatsOverrides = []EthstatsOverride{
+	createEthstatsOverride(EthstatsOverrideConfig{
+		FlagName:    "ethstats-network-name",
+		EnvName:     "ETHSTATS_NETWORK_NAME",
+		Description: "sets the network name override",
+		OverrideFunc: func(val string, overrides *ethstats.Override) {
+			overrides.NetworkName.Enabled = true
+			overrides.NetworkName.Value = val
+		},
+	}),
+	createEthstatsOverride(EthstatsOverrideConfig{
+		FlagName:    "metrics-addr",
+		EnvName:     "METRICS_ADDR",
+		Description: "sets the metrics address",
+		OverrideFunc: func(val string, overrides *ethstats.Override) {
+			overrides.MetricsAddr.Enabled = true
+			overrides.MetricsAddr.Value = val
+		},
+	}),
+}
+
+// ethstatsCmd represents the ethstats command
+var ethstatsCmd = &cobra.Command{
+	Use:   "ethstats",
+	Short: "Runs Xatu in ethstats mode.",
+	Long: `Runs Xatu in ethstats mode, which means it will accept WebSocket connections
+	from ethstats-compatible clients (e.g., geth) and forward the data to configured sinks.`,
+	Run: func(cmd *cobra.Command, args []string) {
+		initCommon()
+
+		config, err := loadEthstatsConfigFromFile(ethstatsCfgFile)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		log = getLogger(config.LoggingLevel, "")
+
+		log.WithField("location", ethstatsCfgFile).Info("Loaded config")
+
+		overrides := &ethstats.Override{}
+		for _, override := range EthstatsOverrides {
+			if errr := override.Setter(cmd, overrides); errr != nil {
+				log.Fatal(errr)
+			}
+		}
+
+		if errr := config.ApplyOverrides(overrides, log); errr != nil {
+			log.Fatal(errr)
+		}
+
+		e, err := ethstats.New(cmd.Context(), log, config)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		if err := e.Start(cmd.Context()); err != nil {
+			log.Fatal(err)
+		}
+
+		log.Info("Xatu ethstats exited - cya!")
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(ethstatsCmd)
+
+	ethstatsCmd.Flags().StringVar(&ethstatsCfgFile, "config", "ethstats.yaml", "config file (default is ethstats.yaml)")
+
+	for _, override := range EthstatsOverrides {
+		override.FlagHelper(ethstatsCmd)
+	}
+}
+
+func loadEthstatsConfigFromFile(file string) (*ethstats.Config, error) {
+	if file == "" {
+		file = "ethstats.yaml"
+	}
+
+	config := &ethstats.Config{}
+
+	if err := defaults.Set(config); err != nil {
+		return nil, err
+	}
+
+	yamlFile, err := os.ReadFile(file)
+	if err != nil {
+		return nil, err
+	}
+
+	type plain ethstats.Config
+
+	if err := yaml.Unmarshal(yamlFile, (*plain)(config)); err != nil {
+		return nil, err
+	}
+
+	return config, nil
+}