diff --git a/.vitepress/config.mts b/.vitepress/config.mts index 904cbad..a870473 100644 --- a/.vitepress/config.mts +++ b/.vitepress/config.mts @@ -228,9 +228,74 @@ function sidebarHome() { ], }, { - text: "How-to Guides (Under Construction 🚧)", - collapsed: true, - items: [], + text: "Hackathon Docs", + collapsed: false, + items: [ + { + text: "Overview", + link: "/docs/hackathon/overview", + }, + { + text: "Quick Start", + collapsed: true, + items: [ + { + text: "Installation & Setup", + link: "/docs/hackathon/quick-start/installation", + }, + { + text: "First Message Example", + link: "/docs/hackathon/quick-start/first-message", + }, + ], + }, + { + text: "Deployment Options", + collapsed: true, + items: [ + { + text: "P2P Network Only", + link: "/docs/hackathon/deployment/p2p-only", + }, + { + text: "P2P Network with Proxy", + link: "/docs/hackathon/deployment/p2p-with-proxy", + }, + ], + }, + { + text: "Client Development", + collapsed: true, + items: [ + { + text: "mump2p-cli", + link: "/docs/hackathon/clients/mump2p-cli", + }, + { + text: "gRPC Client Examples", + link: "/docs/hackathon/clients/grpc-examples", + }, + ], + }, + { + text: "Configuration", + collapsed: true, + items: [ + { + text: "Protocol Selection", + link: "/docs/hackathon/configuration/protocol-selection", + }, + { + text: "OptimumP2P Configuration", + link: "/docs/hackathon/configuration/optimump2p", + }, + { + text: "GossipSub Configuration", + link: "/docs/hackathon/configuration/gossipsub", + }, + ], + }, + ], }, { text: "Research", diff --git a/docs/hackathon/clients/grpc-examples.md b/docs/hackathon/clients/grpc-examples.md new file mode 100644 index 0000000..6151df1 --- /dev/null +++ b/docs/hackathon/clients/grpc-examples.md @@ -0,0 +1,717 @@ +# gRPC Client Examples + +This guide demonstrates how to build and use gRPC clients for direct integration with OptimumP2P networks, covering both P2P-only and Proxy deployment scenarios. + +## Overview + +OptimumP2P provides two gRPC client integration patterns: + +1. **P2P Direct Client** - Connect directly to P2P node sidecar (for P2P-only deployments) +2. **Proxy Client** - Connect through the Proxy service (for Proxy + P2P deployments) + +## P2P Direct Client + +The P2P direct client connects to individual P2P node sidecar gRPC endpoints for direct protocol interaction. + +### Creating the P2P Client + +The P2P client code is provided below. Create the following directory structure: + +``` +grpc_p2p_client/ +β”œβ”€β”€ p2p_client.go # Main client implementation +β”œβ”€β”€ proto/ # Protocol buffer definitions +β”œβ”€β”€ grpc/ # Generated gRPC code +└── go.mod # Go module dependencies +``` + +### Client Features + +- Direct gRPC connection to P2P node sidecar +- Real-time message publishing and subscribing +- Message trace collection for both GossipSub and OptimumP2P protocols +- Graceful shutdown handling + +### Building the P2P Client + +#### Step 1: Create the directory and initialize Go module + +```bash +mkdir grpc_p2p_client +cd grpc_p2p_client +go mod init p2p_client +``` + +#### Step 2: Add dependencies + +```bash +go get google.golang.org/grpc +go get google.golang.org/protobuf +``` + +#### Step 3: Create protobuf definition + +Create `proto/p2p_stream.proto`: + +```bash +mkdir proto +``` + +Create `proto/p2p_stream.proto` with this content: + +```protobuf +syntax = "proto3"; + +package proto; + +option go_package = "optimum-gateway/proto;proto"; + +service CommandStream { + rpc ListenCommands (stream Request) returns (stream Response) {} + rpc Health (Void) returns (HealthResponse) {} + rpc ListTopics(Void) returns (TopicList) {} +} + +message Void {} + +message HealthResponse { + bool status = 1; + string nodeMode = 2; + float memoryUsed = 3; + float cpuUsed = 4; + float diskUsed = 5; +} + +enum ResponseType { + Unknown = 0; + Message = 1; + MessageTraceOptimumP2P = 2; + MessageTraceGossipSub = 3; +} + +message Request { + int32 command = 1; + bytes data = 2; + string topic = 3; +} + +message Response { + ResponseType command = 1; + bytes data = 2; + bytes metadata = 3; +} + +message TopicList { + repeated string topics = 1; +} +``` + +#### Step 4: Generate protobuf code + +```bash +mkdir grpc +protoc --go_out=grpc --go-grpc_out=grpc proto/p2p_stream.proto +``` + +#### Step 5: Create the client code + +Create `p2p_client.go` with this content: + +```go +package main + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "flag" + "fmt" + "io" + "log" + "math" + "os" + "os/signal" + "strings" + "sync/atomic" + "syscall" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/keepalive" + "google.golang.org/grpc/status" + + protobuf "p2p_client/grpc" +) + +type P2PMessage struct { + MessageID string + Topic string + Message []byte + SourceNodeID string +} + +type Command int32 + +const ( + CommandUnknown Command = iota + CommandPublishData + CommandSubscribeToTopic + CommandUnSubscribeToTopic +) + +var ( + addr = flag.String("addr", "localhost:33212", "sidecar gRPC address") + mode = flag.String("mode", "subscribe", "mode: subscribe | publish") + topic = flag.String("topic", "", "topic name") + message = flag.String("msg", "", "message data (for publish)") + + count = flag.Int("count", 1, "number of messages to publish (for publish mode)") + sleep = flag.Duration("sleep", 0, "optional delay between publishes (e.g., 1s, 500ms)") + + keepaliveTime = flag.Duration("keepalive-interval", 2*time.Minute, "gRPC keepalive ping interval") + keepaliveTimeout = flag.Duration("keepalive-timeout", 20*time.Second, "gRPC keepalive ping timeout") +) + +func main() { + flag.Parse() + if *topic == "" { + log.Fatalf("βˆ’topic is required") + } + + conn, err := grpc.NewClient(*addr, + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithInitialWindowSize(1024*1024*1024), + grpc.WithInitialConnWindowSize(1024*1024*1024), + grpc.WithDefaultCallOptions( + grpc.MaxCallRecvMsgSize(math.MaxInt), + grpc.MaxCallSendMsgSize(math.MaxInt), + ), + grpc.WithKeepaliveParams(keepalive.ClientParameters{ + Time: *keepaliveTime, + Timeout: *keepaliveTimeout, + PermitWithoutStream: false, + })) + if err != nil { + log.Fatalf("failed to connect to node %v", err) + } + defer conn.Close() + + client := protobuf.NewCommandStreamClient(conn) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + stream, err := client.ListenCommands(ctx) + if err != nil { + log.Fatalf("ListenCommands: %v", err) + } + + c := make(chan os.Signal, 1) + signal.Notify(c, os.Interrupt, syscall.SIGTERM) + go func() { + <-c + fmt.Println("\nshutting down…") + cancel() + os.Exit(0) + }() + + switch *mode { + case "subscribe": + subReq := &protobuf.Request{ + Command: int32(CommandSubscribeToTopic), + Topic: *topic, + } + if err := stream.Send(subReq); err != nil { + log.Fatalf("send subscribe: %v", err) + } + fmt.Printf("Subscribed to topic %q, waiting for messages…\n", *topic) + + var receivedCount int32 + msgChan := make(chan *protobuf.Response, 10000) + + go func() { + for { + resp, err := stream.Recv() + if err == io.EOF { + close(msgChan) + return + } + if err != nil { + if st, ok := status.FromError(err); ok { + msg := st.Message() + if strings.Contains(msg, "ENHANCE_YOUR_CALM") || strings.Contains(msg, "too_many_pings") { + log.Printf("Connection closed due to keepalive ping limit.") + close(msgChan) + return + } + } + log.Printf("recv error: %v", err) + close(msgChan) + return + } + msgChan <- resp + } + }() + + for { + select { + case <-ctx.Done(): + log.Printf("Context canceled. Total messages received: %d", atomic.LoadInt32(&receivedCount)) + return + case resp, ok := <-msgChan: + if !ok { + log.Printf("Stream closed. Total messages received: %d", atomic.LoadInt32(&receivedCount)) + return + } + go func(resp *protobuf.Response) { + handleResponse(resp, &receivedCount) + }(resp) + } + } + + case "publish": + if *message == "" && *count == 1 { + log.Fatalf("βˆ’msg is required in publish mode") + } + for i := 0; i < *count; i++ { + var data []byte + if *count == 1 { + data = []byte(*message) + } else { + randomBytes := make([]byte, 4) + if _, err := rand.Read(randomBytes); err != nil { + log.Fatalf("failed to generate random bytes: %v", err) + } + randomSuffix := hex.EncodeToString(randomBytes) + data = []byte(fmt.Sprintf("P2P message %d - %s", i+1, randomSuffix)) + } + + pubReq := &protobuf.Request{ + Command: int32(CommandPublishData), + Topic: *topic, + Data: data, + } + if err := stream.Send(pubReq); err != nil { + log.Fatalf("send publish: %v", err) + } + fmt.Printf("Published %q to %q\n", string(data), *topic) + + if *sleep > 0 { + time.Sleep(*sleep) + } + } + + default: + log.Fatalf("unknown mode %q", *mode) + } +} + +func handleResponse(resp *protobuf.Response, counter *int32) { + switch resp.GetCommand() { + case protobuf.ResponseType_Message: + var p2pMessage P2PMessage + if err := json.Unmarshal(resp.GetData(), &p2pMessage); err != nil { + log.Printf("Error unmarshalling message: %v", err) + return + } + n := atomic.AddInt32(counter, 1) + fmt.Printf("[%d] Received message: %q\n", n, string(p2pMessage.Message)) + case protobuf.ResponseType_MessageTraceGossipSub: + case protobuf.ResponseType_MessageTraceOptimumP2P: + case protobuf.ResponseType_Unknown: + default: + log.Println("Unknown response command:", resp.GetCommand()) + } +} +``` + +#### Step 6: Build the client + +```bash +go build -o p2p-client ./p2p_client.go +``` + +### Usage Examples + +#### Subscribe to a Topic + +```bash +# Basic subscription +./p2p-client -mode=subscribe -topic=test-topic -addr=localhost:33212 +``` + +#### Publish Messages + +```bash +# Publish a single message +./p2p-client -mode=publish -topic=test-topic -msg="Hello World" -addr=localhost:33212 + +# Publish multiple random messages with delay +./p2p-client -mode=publish -topic=test-topic -addr=localhost:33212 \ + -count=100 -sleep=200ms +``` + +### Using a Convenience Script (Optional) + +You can create a helper script `p2p_client.sh` for easier usage: + +```bash +#!/usr/bin/env bash +set -e + +P2P_CLIENT_DIR="./grpc_p2p_client" + +cd "$P2P_CLIENT_DIR" + +go build -o p2p-client ./p2p_client.go + +if [ -z "${1:-}" ]; then + echo "Usage: $0 (subscribe )|(publish [options])" >&2 + exit 1 +fi + +ADDR="$1" +shift + +case "${1:-}" in + subscribe) + TOPIC="$2" + shift 2 + ./p2p-client -mode=subscribe -topic="$TOPIC" --addr="$ADDR" "$@" + ;; + publish) + TOPIC="$2" + MESSAGE="$3" + shift 3 + if [[ "$MESSAGE" == "random" ]]; then + ./p2p-client -mode=publish -topic="$TOPIC" --addr="$ADDR" "$@" + else + ./p2p-client -mode=publish -topic="$TOPIC" -msg="$MESSAGE" --addr="$ADDR" "$@" + fi + ;; + *) + echo "Usage: $0 (subscribe )|(publish [options])" >&2 + exit 1 + ;; +esac +``` + +Then use it like: + +```bash +# Subscribe to topic +./p2p_client.sh 127.0.0.1:33221 subscribe test-topic + +# Publish a message +./p2p_client.sh 127.0.0.1:33221 publish test-topic "Hello World" + +# Publish multiple random messages +./p2p_client.sh 127.0.0.1:33221 publish test-topic "random" -count=100 -sleep=200ms +``` + +### P2P Client Configuration + +| Parameter | Description | Default | Example | +|-----------|-------------|---------|---------| +| `-addr` | P2P node sidecar gRPC address | `localhost:33212` | `127.0.0.1:33221` | +| `-mode` | Operation mode | `subscribe` | `subscribe`, `publish` | +| `-topic` | Topic name for pub/sub | Required | `test-topic` | +| `-msg` | Message content (publish mode) | `""` | `"Hello World"` | +| `-count` | Number of messages to publish | `1` | `100` | +| `-sleep` | Delay between messages | `1s` | `200ms`, `2s` | + +### Message Structure + +The P2P client handles messages using this structure: + +```go +type P2PMessage struct { + MessageID string // Unique identifier for the message + Topic string // Topic name where the message was published + Message []byte // Actual message data + SourceNodeID string // ID of the node that sent the message +} +``` + +### Response Types + +The client receives different response types for metrics collection: + +- `ResponseType_Message` - Actual message content +- `ResponseType_MessageTraceGossipSub` - GossipSub protocol metrics +- `ResponseType_MessageTraceOptimumP2P` - OptimumP2P protocol metrics +- `ResponseType_Unknown` - Unknown response type + +## Proxy Client + +The Proxy client connects through the Proxy service, which provides REST API registration and gRPC streaming for message delivery. + +### Creating the Proxy Client + +The Proxy client code is provided in the [Quick Start Guide](../quick-start/first-message.md#step-4-create-the-proxy-client). Follow those steps to create the complete proxy client with all necessary files. + +### Client Features + +- REST API registration with threshold-based subscription +- gRPC streaming for real-time message delivery +- Automatic message publishing with configurable rate +- Load balancing across multiple proxy instances + +### Building the Proxy Client + +Refer to the [Quick Start Guide Steps 4a-4g](../quick-start/first-message.md#step-4-create-the-proxy-client) for complete build instructions. + +### Usage Examples + +#### Subscribe Only Mode + +```bash +# Basic subscription with threshold +./proxy-client -topic=demo -threshold=0.7 -subscribeOnly=true +``` + +#### Publish and Subscribe Mode + +```bash +# Publish 10 messages with 2-second delay +./proxy-client -topic=demo -threshold=0.5 -count=10 -delay=2s + +# High-frequency publishing +./proxy-client -topic=performance -threshold=0.5 -count=1000 -delay=100ms +``` + +### Using the Convenience Script + +Refer to the [Quick Start Guide](../quick-start/first-message.md) for examples of using the proxy client directly with command-line flags like: + +```bash +# Subscribe to topic with threshold +./proxy-client -topic=demo -threshold=0.7 -subscribeOnly + +# Publish messages with threshold and count +./proxy-client -topic=demo -threshold=0.5 -count=10 +``` + +### Proxy Client Configuration + +| Parameter | Description | Default | Example | +|-----------|-------------|---------|---------| +| `-topic` | Topic name for pub/sub | `demo` | `test-topic` | +| `-threshold` | Delivery threshold (0.0 to 1.0) | `0.1` | `0.7` | +| `-subscribeOnly` | Only subscribe, no publishing | `false` | `true` | +| `-count` | Number of messages to publish | `5` | `100` | +| `-delay` | Delay between message publishing | `2s` | `100ms`, `5s` | + +### Proxy Endpoints + +The Proxy client uses these endpoints: + +- **REST API**: `http://localhost:8081` (Proxy 1) / `http://localhost:8082` (Proxy 2) +- **gRPC Stream**: `localhost:50051` (default) + +### REST API Integration + +The client first registers via REST API before establishing gRPC streaming: + +```bash +# Example REST API call for subscription +curl -X POST http://localhost:8081/api/subscribe \ + -H "Content-Type: application/json" \ + -d '{ + "client_id": "test_client_001", + "topic": "demo", + "threshold": 0.7 + }' +``` + +## Performance Comparison Usage + +Both clients support metrics collection for comparing GossipSub vs OptimumP2P performance: + +### P2P Direct Client Metrics + +The P2P client receives protocol-specific trace messages: + +```go +switch resp.GetCommand() { +case protobuf.ResponseType_MessageTraceGossipSub: + // GossipSub protocol metrics +case protobuf.ResponseType_MessageTraceOptimumP2P: + // OptimumP2P protocol metrics +} +``` + +### Proxy Client Metrics + +The Proxy client provides simpler message delivery metrics suitable for application-level performance testing. + +### Example Performance Test + +```bash +# Terminal 1: Start OptimumP2P subscriber +./p2p-client -addr=127.0.0.1:33221 -mode=subscribe -topic=perf-test + +# Terminal 2: Start GossipSub subscriber +./p2p-client -addr=127.0.0.1:33222 -mode=subscribe -topic=perf-test + +# Terminal 3: Publish test messages +./p2p-client -addr=127.0.0.1:33221 -mode=publish -topic=perf-test -count=1000 -sleep=10ms +``` + +## Integration Examples + +### Go Application Integration + +```go +package main + +import ( + "context" + "log" + "math" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + protobuf "p2p_client/grpc" +) + +func main() { + // Connect to P2P node + conn, err := grpc.NewClient("localhost:33212", + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithDefaultCallOptions( + grpc.MaxCallRecvMsgSize(math.MaxInt), + grpc.MaxCallSendMsgSize(math.MaxInt), + )) + if err != nil { + log.Fatal(err) + } + defer conn.Close() + + client := protobuf.NewCommandStreamClient(conn) + stream, err := client.ListenCommands(context.Background()) + if err != nil { + log.Fatal(err) + } + + // Subscribe to topic + subReq := &protobuf.Request{ + Command: 2, // CommandSubscribeToTopic + Topic: "my-topic", + } + + if err := stream.Send(subReq); err != nil { + log.Fatal(err) + } + + // Listen for messages + for { + resp, err := stream.Recv() + if err != nil { + log.Fatal(err) + } + // Handle response... + } +} +``` + +### Protocol Buffer Definitions + +Both clients use Protocol Buffers for gRPC communication. The definitions are provided in the code sections above: + +- P2P Client: See the protobuf definition in the "Building the P2P Client" section +- Proxy Client: See the [Quick Start Guide](../quick-start/first-message.md#step-4d-create-protobuf-definition) + +## Troubleshooting + +### Connection Issues + +#### Error: `connection refused` + +``` +failed to connect to node dial tcp 127.0.0.1:33212: connect: connection refused +``` + +**Solutions:** +- Verify P2P nodes are running: `docker-compose ps` +- Check port mapping in docker-compose.yml +- Ensure correct sidecar port (typically 33212) + +### Message Delivery Issues + +#### No messages received + +**Troubleshooting:** +1. Verify topic names match exactly between publisher and subscriber +2. Check P2P node logs: `docker-compose logs p2pnode-1` +3. Ensure nodes are properly connected in mesh topology +4. Verify protocol mode consistency (all nodes using same protocol) + +#### Partial message delivery + +**For Proxy Client:** +- Adjust threshold parameter (lower values = faster delivery, less reliability) +- Check proxy connectivity to P2P nodes + +**For P2P Client:** +- Verify direct connection to healthy P2P node +- Check RLNC configuration parameters + +### Build Issues + +#### Proto generation errors + +```bash +# Regenerate gRPC code if needed +cd grpc_p2p_client +protoc --go_out=grpc --go-grpc_out=grpc proto/*.proto +``` + +#### Module dependency issues + +```bash +# Update Go modules +go mod tidy +go mod download +``` + +## Advanced Usage + +### Long-Running Subscriptions + +For production deployments, consider: + +1. **Connection monitoring**: Implement reconnection logic for network failures +2. **Message persistence**: Store received messages for reliability +3. **Load balancing**: Connect to multiple P2P nodes for redundancy +4. **Metrics collection**: Track message delivery rates and latency + +### Custom Message Formats + +Both clients support custom message formats. Modify the message structure in: + +- P2P Client: `P2PMessage` struct in `p2p_client.go` +- Proxy Client: Message handling in `proxy_client.go` + +### Integration with Other Languages + +The Protocol Buffer definitions can be used to generate client code for other languages: + +```bash +# Python +protoc --python_out=. --grpc_python_out=. proto/*.proto + +# JavaScript/Node.js +protoc --js_out=import_style=commonjs:. --grpc-web_out=import_style=commonjs,mode=grpcwebtext:. proto/*.proto + +# Java +protoc --java_out=. --grpc-java_out=. proto/*.proto +``` + +## See Also + +- [mump2p CLI Guide](./mump2p-cli.md) - Command-line client for OptimumP2P +- [P2P-Only Deployment](../deployment/p2p-only.md) - Setting up P2P nodes without proxy +- [Proxy + P2P Deployment](../deployment/p2p-with-proxy.md) - Full stack with proxy services +- [OptimumP2P Configuration](../configuration/optimump2p.md) - Protocol configuration parameters +- [GossipSub Configuration](../configuration/gossipsub.md) - Baseline protocol configuration \ No newline at end of file diff --git a/docs/hackathon/clients/mump2p-cli.md b/docs/hackathon/clients/mump2p-cli.md new file mode 100644 index 0000000..453a540 --- /dev/null +++ b/docs/hackathon/clients/mump2p-cli.md @@ -0,0 +1,207 @@ +# mump2p β€” OptimumP2P CLI Client + +**mump2p is the primary client entry point for interacting with OptimumP2P** β€” a high-performance RLNC-enhanced pubsub protocol. This command-line interface provides the easiest way to publish and subscribe to messages using the OptimumP2P protocol. + +**Source:** [https://github.com/getoptimum/mump2p-cli](https://github.com/getoptimum/mump2p-cli) + +## Installation + +Download the appropriate binary for your system from the [GitHub releases page](https://github.com/getoptimum/mump2p-cli/releases): + +- `mump2p-mac` for macOS +- `mump2p-linux` for Linux + +Make the binary executable (Mac/Linux): + +```sh +chmod +x mump2p-mac +# For convenience, you may want to rename and move to your PATH +mv mump2p-mac /usr/local/bin/mump2p +``` + +--- + +## Authentication + +### Login + +Before you can publish or subscribe to messages, you need to authenticate: + +```sh +./mump2p login +``` + +This will start the device authorization flow: + +1. A URL and a code will be displayed in your terminal +2. Open the URL in your browser +3. Enter the code when prompted +4. Complete the authentication process in the browser +5. The CLI will automatically receive and store your authentication token + +### Check Authentication Status + +To verify your current authentication status: + +```sh +./mump2p whoami +``` + +This will display: + +- Your client ID +- Token expiration time +- Token validity status +- Rate limits associated with your account + +**Important: After logging in, please share the email ID you used to sign up in the group, so we can activate your access.** + +### Refresh Token + +If your token is about to expire, you can refresh it: + +```sh +./mump2p refresh +``` + +### Logout + +To remove your stored authentication token: + +```sh +./mump2p logout +``` + +--- + +## Service URLs + +The CLI connects to different proxy servers around the world. By default, it uses the first available proxy, but you can specify a different one using the `--service-url` flag. + +For a complete list of available proxys and their locations, see: [Available Service URLs](https://github.com/getoptimum/mump2p-cli#available-service-urls) in the README. + +**Example using a specific proxy:** +```sh +./mump2p publish --topic=test --message="Hello" --service-url="http://35.221.118.95:8080" +./mump2p subscribe --topic=test --service-url="http://34.142.205.26:8080" +``` + +--- + +## Publishing Messages + +### Publish a Text Message + +To publish a simple text message to a topic: + +```sh +./mump2p publish --topic=your-topic-name --message="Your message content" +``` + +### Publish a File + +To publish the contents of a file: + +```sh +./mump2p publish --topic=your-topic-name --file=/path/to/your/file.json +``` + +Rate limits will be automatically applied based on your authentication token. + +## Subscribing to Messages + +### Basic Subscription + +To subscribe to a topic in real-time: + +```sh +./mump2p subscribe --topic=your-topic-name +``` + +This will open a WebSocket connection and display incoming messages in real-time. Press `Ctrl+C` to exit. + +### Save Messages to a File + +To persist messages to a local file while subscribing: + +```sh +./mump2p subscribe --topic=your-topic-name --persist=/path/to/save/ +``` + +If you provide just a directory path, messages will be saved to a file named `messages.log` in that directory. + +### Forward Messages to a Webhook + +To forward messages to an HTTP webhook: + +```sh +./mump2p subscribe --topic=your-topic-name --webhook=https://your-server.com/webhook +``` + +**Note: The webhook endpoint must be configured to accept POST requests.** + +#### Advanced Webhook Options + +For more control over webhook behavior: + +```sh +./mump2p subscribe --topic=your-topic-name \ + --webhook=https://your-server.com/webhook \ + --webhook-queue-size=200 \ + --webhook-timeout=5 + ``` + +Options: + +- `--webhook-queue-size`: Maximum number of messages to queue before dropping (default: `100`) +- `--webhook-timeout`: Timeout in seconds for each webhook POST request (default: `3`) + +### Combine Persistence and Webhook + +You can both save messages locally and forward them to a webhook: + +```sh +./mump2p subscribe --topic=your-topic-name \ + --persist=/path/to/messages.log \ + --webhook=https://your-server.com/webhook +``` + +--- + +## Checking Usage and Limits + +To see your current usage statistics and rate limits: + +```sh +./mump2p usage +``` + +This will display: + +- Number of publish operations (per hour and per second) +- Data usage (bytes, KB, or MB depending on amount) +- Quota limits +- Time until usage counters reset +- Timestamps of your last publish and subscribe operations + +## Tips for Effective Use + +1. **Topic Names:** Choose descriptive and unique topic names to avoid message conflicts +2. **Message Size:** Be aware of your maximum message size limit when publishing files +3. **Token Refresh:** For long-running operations, refresh your token before it expires +4. **Persistent Subscriptions:** Use the --persist option when you need a record of messages +5. **Webhook Reliability:** Increase the queue size for high-volume topics to prevent message drops + +## Troubleshooting + +- **Authentication Errors:** Run mump2p whoami to check token status, and mump2p login to re-authenticate +- **Rate Limit Errors:** Use mump2p usage to check your current usage against limits +- **Connection Issues:** Verify your internet connection and firewall settings +- **Webhook Failures:** Check that your webhook endpoint is accessible and properly configured to accept POST requests + +## See Also + +- [gRPC Client Examples](./grpc-examples.md) - Direct gRPC integration examples for advanced use cases +- [Proxy + P2P Deployment](../deployment/p2p-with-proxy.md) - Full stack deployment with REST/WebSocket APIs +- [P2P-Only Deployment](../deployment/p2p-only.md) - Direct P2P integration +- [OptimumP2P Configuration](../configuration/optimump2p.md) - Protocol configuration parameters \ No newline at end of file diff --git a/docs/hackathon/configuration/gossipsub.md b/docs/hackathon/configuration/gossipsub.md new file mode 100644 index 0000000..41b0e08 --- /dev/null +++ b/docs/hackathon/configuration/gossipsub.md @@ -0,0 +1,124 @@ +# GossipSub Configuration + +Understanding GossipSub configuration is essential for optimizing your P2P network performance. This page explains the core GossipSub parameters and how to configure them in Optimum P2P. + +## What is GossipSub? + +GossipSub is a pubsub protocol designed for decentralized networks that builds on the concept of a "mesh" - a subset of peers that directly exchange messages, plus "gossip" to peers outside the mesh about available messages. + +### Key Concepts + +- **Mesh**: Direct connections between peers for message forwarding +- **Gossip**: Metadata exchange about available messages with non-mesh peers +- **Fanout**: Temporary connections for publishing when not subscribed to a topic +- **Degree**: The number of peer connections each node maintains + +## Core GossipSub Parameters + +Based on the [libp2p GossipSub v1.0 specification](https://github.com/libp2p/specs/blob/master/pubsub/gossipsub/gossipsub-v1.0.md#parameters), here are the fundamental parameters: + +### Network Topology Parameters + +| Parameter | Purpose | Default | Description | +|-----------|---------|---------|-------------| +| `D` | Desired mesh degree | 6 | Target number of peers in topic mesh | +| `D_low` | Lower bound for mesh degree | 4 | Minimum connections before adding peers | +| `D_high` | Upper bound for mesh degree | 12 | Maximum connections before pruning peers | +| `D_lazy` | Gossip emission degree | 6 | Number of peers for gossip emission | + +### Timing Parameters + +| Parameter | Purpose | Default | Description | +|-----------|---------|---------|-------------| +| `heartbeat_interval` | Mesh maintenance frequency | 1 second | How often to run maintenance | +| `fanout_ttl` | Fanout topic lifetime | 60 seconds | TTL for non-subscribed topic state | +| `seen_ttl` | Message deduplication cache | 2 minutes | How long to remember seen messages | + +### Message Cache Parameters + +| Parameter | Purpose | Default | Description | +|-----------|---------|---------|-------------| +| `mcache_len` | Message history windows | 5 | Number of history windows to keep | +| `mcache_gossip` | Gossip history windows | 3 | Windows to use for gossip emission | + +## Optimum P2P GossipSub Configuration + +Optimum P2P implements GossipSub with configurable parameters through environment variables: + +### Environment Variables + +- **`GOSSIPSUB_PORT`**: Port for GossipSub protocol communication + - Purpose: Network port for GossipSub peer-to-peer communication + - Default: `6060` + - Usage: Internal P2P networking, separate from API ports + - Example: `GOSSIPSUB_PORT=6060` + +- **`GOSSIPSUB_MAX_MSG_SIZE`**: Maximum message size in bytes + - Purpose: Limits individual message size to prevent memory issues + - Default: `1048576` (1MB) + - Usage: Larger values allow bigger payloads but use more memory + - Example: `GOSSIPSUB_MAX_MSG_SIZE=1048576` + +- **`GOSSIPSUB_MESH_TARGET`**: Target number of mesh connections + - Purpose: Ideal number of direct peer connections (corresponds to `D`) + - Default: `6` + - Impact: Higher values increase redundancy but use more bandwidth + - Example: `GOSSIPSUB_MESH_TARGET=6` + +- **`GOSSIPSUB_MESH_MIN`**: Minimum mesh connections + - Purpose: Lower threshold before attempting to add peers (corresponds to `D_low`) + - Default: `4` + - Note: Should be less than `GOSSIPSUB_MESH_TARGET` + - Example: `GOSSIPSUB_MESH_MIN=4` + +- **`GOSSIPSUB_MESH_MAX`**: Maximum mesh connections + - Purpose: Upper threshold before pruning excess peers (corresponds to `D_high`) + - Default: `12` + - Note: Should be greater than `GOSSIPSUB_MESH_TARGET` + - Example: `GOSSIPSUB_MESH_MAX=12` + +### Configuration Example + +```yaml +# docker-compose.yml +services: + p2pnode-1: + environment: + - GOSSIPSUB_PORT=6060 + - GOSSIPSUB_MAX_MSG_SIZE=1048576 + - GOSSIPSUB_MESH_TARGET=6 + - GOSSIPSUB_MESH_MIN=4 + - GOSSIPSUB_MESH_MAX=12 +``` + +## GossipSub vs Optimum Protocol + +Optimum P2P runs two protocols simultaneously: + +1. **Standard GossipSub**: For baseline pub/sub functionality +2. **Optimum Protocol**: Enhanced with RLNC (Random Linear Network Coding) + +Both protocols have their own configuration parameters: + +| Aspect | GossipSub | Optimum | +|--------|-----------|---------| +| Purpose | Standard pub/sub mesh | RLNC-enhanced messaging | +| Port | `GOSSIPSUB_PORT` | `OPTIMUM_PORT` | +| Mesh Target | `GOSSIPSUB_MESH_TARGET` | `OPTIMUM_MESH_TARGET` | +| Message Size | `GOSSIPSUB_MAX_MSG_SIZE` | `OPTIMUM_MAX_MSG_SIZE` | + +### Control Messages + +GossipSub uses four types of control messages: + +- **GRAFT**: Add peer to mesh (invite to join topic mesh) +- **PRUNE**: Remove peer from mesh (remove from topic mesh) +- **IHAVE**: Announce available messages (gossip metadata) +- **IWANT**: Request specific messages (response to IHAVE) + + +## References + +- [libp2p GossipSub v1.0 Specification](https://github.com/libp2p/specs/blob/master/pubsub/gossipsub/gossipsub-v1.0.md) +- [libp2p PubSub Interface](https://github.com/libp2p/specs/blob/master/pubsub/README.md) +- [Optimum P2P Configuration Guide](/docs/hackathon/deployment/p2p-with-proxy.md#p2p-node-configuration) \ No newline at end of file diff --git a/docs/hackathon/configuration/optimump2p.md b/docs/hackathon/configuration/optimump2p.md new file mode 100644 index 0000000..c842de5 --- /dev/null +++ b/docs/hackathon/configuration/optimump2p.md @@ -0,0 +1,246 @@ +# OptimumP2P Configuration + +This page covers all OptimumP2P-specific configuration parameters, their impact on network performance, and how to configure them for different deployment scenarios. + +## Configuration Mapping + +OptimumP2P configuration can be set via YAML config files or Docker environment variables. The following tables map the configuration fields to their corresponding Docker environment variables: + +### RLNC Core Parameters + +| Config Field | Environment Variable | Default | Description | +|--------------|---------------------|---------|-------------| +| `rlnc_shard_factor` | `OPTIMUM_SHARD_FACTOR` | `4` | Number of coded shards per message | +| `publisher_shard_multiplier` | `OPTIMUM_SHARD_MULT` | `1.5` | Coded shard multiplier for publishing | +| `forward_shard_threshold` | `OPTIMUM_THRESHOLD` | `0.75` | Fraction needed before forwarding | + +### Mesh Topology Parameters + +| Config Field | Environment Variable | Default | Description | +|--------------|---------------------|---------|-------------| +| `mesh_degree_target` | `OPTIMUM_MESH_TARGET` | `6` | Target peer connections | +| `mesh_degree_min` | `OPTIMUM_MESH_MIN` | `3` | Min before grafting | +| `mesh_degree_max` | `OPTIMUM_MESH_MAX` | `12` | Max before pruning | + +### Protocol Selection + +| Config Field | Environment Variable | Default | Description | +|--------------|---------------------|---------|-------------| +| `enable_optimum_p2p` | `NODE_MODE=optimum` | `true` | Enable RLNC protocol | +| `enable_gossipsub` | `NODE_MODE=gossipsub` | `false` | Enable GossipSub baseline | + +### Message and Network Settings + +| Config Field | Environment Variable | Default | Description | +|--------------|---------------------|---------|-------------| +| `random_message_size_bytes` | `OPTIMUM_MAX_MSG_SIZE` | `1048576` | Max message size (1MB) | +| `app_port` | `API_PORT` | `9090` | HTTP API port | + +## RLNC Parameter Details + +### Shard Factor (coded shards) + +**Parameter**: `OPTIMUM_SHARD_FACTOR` +**Config Field**: `rlnc_shard_factor` +**Default**: `4` + +Controls how many coded shards each message is divided into for network coding. This is the fundamental parameter that determines RLNC granularity. + +**Technical Impact**: +- **Higher values**: More coded shards provide better redundancy and fault tolerance but increase computational overhead +- **Lower values**: Fewer coded shards reduce processing time and bandwidth usage but may reduce fault tolerance +- **Network effect**: Directly affects the minimum number of coded shards needed for message reconstruction + +### Publisher Shard Multiplier + +**Parameter**: `OPTIMUM_SHARD_MULT` +**Config Field**: `publisher_shard_multiplier` +**Default**: `1.5` + +Controls redundancy when publishing messages. Total coded shards created = `rlnc_shard_factor Γ— publisher_shard_multiplier`. + +**Technical Impact**: +- **Higher values**: More initial coded shards improve delivery guarantees but increase bandwidth usage +- **Lower values**: Fewer coded shards reduce network load but may impact reliability in lossy networks + +### Forward Shard Threshold + +**Parameter**: `OPTIMUM_THRESHOLD` +**Config Field**: `forward_shard_threshold` +**Default**: `0.75` + +Determines when nodes forward received coded shards. Nodes forward when they have more than `rlnc_shard_factor Γ— threshold` coded shards. + +**Technical Impact**: +- **Higher values** (0.8-0.9): More reliable forwarding, increased latency +- **Lower values** (0.5-0.6): Faster propagation, reduced latency, potential reliability trade-offs + +## Docker Configuration Examples + +### OptimumP2P Mode (Enhanced) +```yaml +environment: + - NODE_MODE=optimum + - OPTIMUM_SHARD_FACTOR=4 + - OPTIMUM_SHARD_MULT=1.5 + - OPTIMUM_THRESHOLD=0.75 + - OPTIMUM_MESH_TARGET=6 + - OPTIMUM_MESH_MIN=3 + - OPTIMUM_MESH_MAX=12 +``` + +### GossipSub Mode (Baseline) +```yaml +environment: + - NODE_MODE=gossipsub + - GOSSIPSUB_MESH_TARGET=6 + - GOSSIPSUB_MESH_MIN=4 + - GOSSIPSUB_MESH_MAX=12 +``` + +## Performance Tuning + +### High Throughput Setup +Optimized for maximum message throughput: +```yaml +- OPTIMUM_SHARD_FACTOR=8 +- OPTIMUM_SHARD_MULT=2.0 +- OPTIMUM_THRESHOLD=0.5 +- OPTIMUM_MESH_TARGET=8 +``` + +### Low Latency Setup +Optimized for minimum delivery latency: +```yaml +- OPTIMUM_SHARD_FACTOR=4 +- OPTIMUM_SHARD_MULT=1.2 +- OPTIMUM_THRESHOLD=0.6 +- OPTIMUM_MESH_TARGET=6 +``` + +### High Reliability Setup +Optimized for maximum fault tolerance: +```yaml +- OPTIMUM_SHARD_FACTOR=16 +- OPTIMUM_SHARD_MULT=2.5 +- OPTIMUM_THRESHOLD=0.8 +- OPTIMUM_MESH_TARGET=10 +``` + +## Configuration File Format + +Sample YAML configuration matching the actual config structure: + +```yaml +# === General App Settings === +app_port: 23422 +cluster_id: "production_cluster" +file_storage_directory: /tmp +silent_mode_enabled: false + +# === Protocol Settings === +enable_gossipsub: false +enable_optimum_p2p: true + +# === PubSub Mesh Settings === +mesh_degree_target: 6 +mesh_degree_min: 5 +mesh_degree_max: 12 +rlnc_shard_factor: 32 # number of coded shards each message is divided into + +# === Message Generation === +random_message_size_bytes: 3145728 # 3MB +publisher_shard_multiplier: 1.5 # coded shards multiplier for publishing +forward_shard_threshold: 0.75 # forward threshold for coded shards + +# === Bootstrap Nodes === +bootstrap_nodes_optimum_p2p: + - /ip4/127.0.0.1/tcp/10000/p2p/12D3KooWExample + +bootstrap_nodes_gossipsub: + - /ip4/127.0.0.1/tcp/10000/p2p/12D3KooWExample +``` + +## Protocol Mode Switching + +OptimumP2P supports running both protocols for comparison: + +**OptimumP2P Mode** (`NODE_MODE=optimum`): +- Uses RLNC with coded shards for enhanced propagation +- Provides early forwarding capabilities +- Better bandwidth efficiency through network coding +- Enhanced fault tolerance via coded shard redundancy + +**GossipSub Mode** (`NODE_MODE=gossipsub`): +- Standard libp2p GossipSub implementation +- Direct message forwarding without coding +- Lower computational overhead +- Baseline for performance comparison + +## Monitoring Configuration + +Verify your configuration is applied correctly: + +```bash +# Check applied configuration +curl http://localhost:9090/config + +# Monitor mesh formation and peer connections +curl http://localhost:9090/debug/peers + +# Check protocol mode in logs +docker logs p2pnode-1 | grep -E "(optimum|gossipsub|coded.*shard)" + +# Monitor coded shard processing +docker logs p2pnode-1 | grep -E "(shard|forward|decode)" +``` + +## Configuration Constraints + +**Enforced by code**: +- `mesh_degree_min <= mesh_degree_target <= mesh_degree_max` +- `random_message_size_bytes > 0` +- `cluster_id` must not be empty or "sample_cluster" + +**Recommended best practices**: +- `forward_shard_threshold` between 0.0 and 1.0 (values outside this range may cause unexpected behavior) +- `rlnc_shard_factor > 0` (value of 0 disables RLNC functionality) +- `publisher_shard_multiplier >= 1.0` (values < 1.0 reduce redundancy) + +## Complete Docker Compose Example + +```yaml +services: + p2pnode-1: + image: getoptimum/p2pnode:latest + environment: + # Protocol selection + - NODE_MODE=optimum + - CLUSTER_ID=p2pnode-1 + + # RLNC coded shard parameters + - OPTIMUM_SHARD_FACTOR=4 + - OPTIMUM_SHARD_MULT=1.5 + - OPTIMUM_THRESHOLD=0.75 + + # Mesh topology + - OPTIMUM_MESH_TARGET=6 + - OPTIMUM_MESH_MIN=3 + - OPTIMUM_MESH_MAX=12 + + # Network settings + - OPTIMUM_PORT=7070 + - OPTIMUM_MAX_MSG_SIZE=1048576 + - SIDECAR_PORT=33212 + - API_PORT=9090 + + # Bootstrap configuration + - BOOTSTRAP_PEERS=/ip4/172.28.0.12/tcp/7070/p2p/${BOOTSTRAP_PEER_ID} +``` + +## See Also + +- **[GossipSub Configuration](./gossipsub.md)** - Standard protocol parameters for comparison +- **[P2P-Only Deployment](../deployment/p2p-only.md)** - Complete setup with configuration +- **[Proxy + P2P Deployment](../deployment/p2p-with-proxy.md)** - Full stack configuration +- **[OptimumP2P Technical Overview](../../learn/overview/p2p.md)** - Protocol fundamentals and RLNC details \ No newline at end of file diff --git a/docs/hackathon/configuration/protocol-selection.md b/docs/hackathon/configuration/protocol-selection.md new file mode 100644 index 0000000..2c21da3 --- /dev/null +++ b/docs/hackathon/configuration/protocol-selection.md @@ -0,0 +1,271 @@ +# Protocol Selection Guide + +The `getoptimum/p2pnode:latest` Docker image supports running two different gossip protocols: + +* **OptimumP2P** (`NODE_MODE=optimum`) - RLNC-enhanced gossip protocol (recommended) +* **GossipSub** (`NODE_MODE=gossipsub`) - Standard libp2p gossip protocol for comparison + +This guide explains how to configure and deploy each protocol mode. + +## Overview + +Both protocols use the same Docker image but operate with different: + +* **Network ports** - OptimumP2P uses port 7070, GossipSub uses port 6060 +* **Configuration parameters** - Each protocol has protocol-specific environment variables +* **Bootstrap peers** - Must match the protocol and port of target nodes +* **Performance characteristics** - OptimumP2P provides enhanced throughput and fault tolerance + +## OptimumP2P Mode (Recommended) + +### Basic Configuration + +```yaml +services: + p2pnode-optimum: + image: 'getoptimum/p2pnode:latest' + environment: + - NODE_MODE=optimum + - CLUSTER_ID=my-cluster + - OPTIMUM_PORT=7070 + - IDENTITY_DIR=/identity + # Bootstrap peer using OptimumP2P protocol and port + - BOOTSTRAP_PEERS=/ip4/172.28.0.10/tcp/7070/p2p/${BOOTSTRAP_PEER_ID} + ports: + - "7070:7070" # OptimumP2P protocol port + - "33212:33212" # gRPC sidecar port + - "9090:9090" # HTTP API port +``` + +### OptimumP2P Parameters + +| Parameter | Environment Variable | Default | Description | +|-----------|---------------------|---------|-------------| +| Listen Port | `OPTIMUM_PORT` | `7070` | P2P protocol listening port | +| Max Message Size | `OPTIMUM_MAX_MSG_SIZE` | `1048576` | Maximum message size (bytes) | +| Mesh Target | `OPTIMUM_MESH_TARGET` | `6` | Target number of mesh peers | +| Mesh Min | `OPTIMUM_MESH_MIN` | `3` | Minimum mesh peers before grafting | +| Mesh Max | `OPTIMUM_MESH_MAX` | `12` | Maximum mesh peers before pruning | +| Shard Factor | `OPTIMUM_SHARD_FACTOR` | `4` | Number of coded shards per message | +| Shard Multiplier | `OPTIMUM_SHARD_MULT` | `1.5` | Publisher shard creation multiplier | +| Forward Threshold | `OPTIMUM_THRESHOLD` | `0.75` | Threshold for forwarding recoded shards | + +### Performance Benefits + +OptimumP2P provides several advantages over standard GossipSub: + +* **Lower Latency** - Coded shards can be forwarded before complete message reception +* **Bandwidth Efficiency** - RLNC reduces redundant data transmission +* **Fault Tolerance** - Messages decode from any sufficient combination of shards +* **Loss Recovery** - Built-in redundancy handles packet loss gracefully + +## GossipSub Mode (Baseline) + +### Basic Configuration + +```yaml +services: + p2pnode-gossipsub: + image: 'getoptimum/p2pnode:latest' + environment: + - NODE_MODE=gossipsub + - CLUSTER_ID=my-cluster + - GOSSIPSUB_PORT=6060 + - IDENTITY_DIR=/identity + # Bootstrap peer using GossipSub protocol and port + - BOOTSTRAP_PEERS=/ip4/172.28.0.10/tcp/6060/p2p/${BOOTSTRAP_PEER_ID} + ports: + - "6060:6060" # GossipSub protocol port + - "33212:33212" # gRPC sidecar port + - "9090:9090" # HTTP API port +``` + +### GossipSub Parameters + +| Parameter | Environment Variable | Default | Description | +|-----------|---------------------|---------|-------------| +| Listen Port | `GOSSIPSUB_PORT` | `6060` | P2P protocol listening port | +| Max Message Size | `GOSSIPSUB_MAX_MSG_SIZE` | `1048576` | Maximum message size (bytes) | +| Mesh Target | `GOSSIPSUB_MESH_TARGET` | `6` | Target number of mesh peers | +| Mesh Min | `GOSSIPSUB_MESH_MIN` | `4` | Minimum mesh peers before grafting | +| Mesh Max | `GOSSIPSUB_MESH_MAX` | `12` | Maximum mesh peers before pruning | + +For detailed GossipSub configuration, see **[GossipSub Configuration](./gossipsub.md)**. + +## Multi-Protocol Deployment + +You can run both protocols simultaneously on the same network for comparison: + +```yaml +version: '3.8' +services: + # OptimumP2P bootstrap node + p2pnode-optimum-bootstrap: + image: 'getoptimum/p2pnode:latest' + environment: + - NODE_MODE=optimum + - CLUSTER_ID=comparison-test + - OPTIMUM_PORT=7070 + - IDENTITY_DIR=/identity + ports: + - "7070:7070" + - "33212:33212" + - "9090:9090" + networks: + test-network: + ipv4_address: 172.28.0.10 + + # GossipSub bootstrap node + p2pnode-gossipsub-bootstrap: + image: 'getoptimum/p2pnode:latest' + environment: + - NODE_MODE=gossipsub + - CLUSTER_ID=comparison-test + - GOSSIPSUB_PORT=6060 + - IDENTITY_DIR=/identity + ports: + - "6060:6060" + - "33213:33212" + - "9091:9090" + networks: + test-network: + ipv4_address: 172.28.0.11 + + # OptimumP2P peer + p2pnode-optimum-peer: + image: 'getoptimum/p2pnode:latest' + environment: + - NODE_MODE=optimum + - CLUSTER_ID=comparison-test + - OPTIMUM_PORT=7070 + - BOOTSTRAP_PEERS=/ip4/172.28.0.10/tcp/7070/p2p/${OPTIMUM_BOOTSTRAP_PEER_ID} + networks: + test-network: + ipv4_address: 172.28.0.12 + + # GossipSub peer + p2pnode-gossipsub-peer: + image: 'getoptimum/p2pnode:latest' + environment: + - NODE_MODE=gossipsub + - CLUSTER_ID=comparison-test + - GOSSIPSUB_PORT=6060 + - BOOTSTRAP_PEERS=/ip4/172.28.0.11/tcp/6060/p2p/${GOSSIPSUB_BOOTSTRAP_PEER_ID} + networks: + test-network: + ipv4_address: 172.28.0.13 + +networks: + test-network: + driver: bridge + ipam: + config: + - subnet: 172.28.0.0/16 +``` + +## Validation and Troubleshooting + +### Valid NODE_MODE Values + +The p2pnode validates `NODE_MODE` at startup. Valid values are: + +* `optimum` - Enables OptimumP2P with RLNC +* `gossipsub` - Enables standard GossipSub + +Invalid values will cause the container to exit with a fatal error. + +### Common Configuration Issues + +**Port Conflicts** +```bash +# Ensure correct port mapping for each protocol +# OptimumP2P: 7070 +# GossipSub: 6060 +``` + +**Bootstrap Peer Mismatches** +```bash +# OptimumP2P nodes must bootstrap to OptimumP2P peers on port 7070 +BOOTSTRAP_PEERS=/ip4/172.28.0.10/tcp/7070/p2p/${PEER_ID} + +# GossipSub nodes must bootstrap to GossipSub peers on port 6060 +BOOTSTRAP_PEERS=/ip4/172.28.0.10/tcp/6060/p2p/${PEER_ID} +``` + +**Protocol Isolation** +* OptimumP2P and GossipSub nodes cannot directly communicate +* Each protocol forms its own mesh network +* Use separate bootstrap nodes for each protocol + +### Health Checks + +Both protocols expose the same health check endpoints: + +```bash +# Check node status +curl http://localhost:9090/health + +# Check peer connections +curl http://localhost:9090/peers + +# Check protocol-specific metrics +curl http://localhost:9090/metrics +``` + +## Configuration Experimentation + +### OptimumP2P Parameter Testing + +OptimumP2P's RLNC implementation provides several parameters that significantly impact network behavior: + +**Shard Factor Experimentation** +```yaml +# Test different sharding approaches +OPTIMUM_SHARD_FACTOR=2 # Minimal sharding +OPTIMUM_SHARD_FACTOR=4 # Default configuration +OPTIMUM_SHARD_FACTOR=8 # High redundancy +``` + +**Forward Threshold Testing** +```yaml +# Test forwarding behavior +OPTIMUM_THRESHOLD=0.5 # Aggressive forwarding +OPTIMUM_THRESHOLD=0.75 # Default behavior +OPTIMUM_THRESHOLD=0.9 # Conservative forwarding +``` + +**Mesh Density Configuration** +```yaml +# Test network connectivity +OPTIMUM_MESH_TARGET=4 # Sparse mesh +OPTIMUM_MESH_TARGET=8 # Dense mesh +OPTIMUM_MESH_TARGET=12 # Maximum connectivity +``` + +### Protocol Comparison Setup + +To observe protocol differences, deploy both modes with identical network topology: + +```yaml +# OptimumP2P network +services: + optimum-network: + environment: + - NODE_MODE=optimum + - OPTIMUM_SHARD_FACTOR=4 + - OPTIMUM_MESH_TARGET=6 + +# GossipSub network + gossipsub-network: + environment: + - NODE_MODE=gossipsub + - GOSSIPSUB_MESH_TARGET=6 +``` + + +--- + +**Next Steps:** +* **[OptimumP2P Configuration](./optimump2p.md)** - Detailed RLNC parameter tuning +* **[GossipSub Configuration](./gossipsub.md)** - Standard gossip parameters +* **[Client Integration](../clients/)** - Connect applications to either protocol \ No newline at end of file diff --git a/docs/hackathon/deployment/p2p-only.md b/docs/hackathon/deployment/p2p-only.md new file mode 100644 index 0000000..c2c30b5 --- /dev/null +++ b/docs/hackathon/deployment/p2p-only.md @@ -0,0 +1,289 @@ +# P2P Network Only Deployment + +This guide covers deploying a standalone OptimumP2P network without proxy intermediaries. In this configuration, clients connect directly to P2P nodes using gRPC sidecar connections, providing maximum performance and protocol flexibility. + + +### Network Topology + +The P2P network forms a mesh topology where each node maintains connections to a subset of peers based on the configured mesh parameters: + +- **Mesh Target (`OPTIMUM_MESH_TARGET`)**: Ideal number of peers per topic +- **Mesh Range (`OPTIMUM_MESH_MIN` to `OPTIMUM_MESH_MAX`)**: Acceptable peer count bounds +- **Bootstrap Nodes**: Initial discovery points for new nodes joining the network + + +## Deployment Configuration + +```yaml +services: + p2pnode-1: + image: 'getoptimum/p2pnode:latest' + platform: linux/amd64 + environment: + - LOG_LEVEL=debug + - CLUSTER_ID=p2pnode-1 + - NODE_MODE=optimum + - SIDECAR_PORT=33212 + - API_PORT=9090 + - IDENTITY_DIR=/identity + - OPTIMUM_PORT=7070 + - OPTIMUM_MAX_MSG_SIZE=1048576 + - OPTIMUM_MESH_TARGET=6 + - OPTIMUM_MESH_MIN=3 + - OPTIMUM_MESH_MAX=12 + - OPTIMUM_SHARD_FACTOR=4 + - OPTIMUM_SHARD_MULT=1.5 + - OPTIMUM_THRESHOLD=0.75 + - BOOTSTRAP_PEERS=/ip4/172.28.0.12/tcp/7070/p2p/${BOOTSTRAP_PEER_ID} + networks: + optimum-network: + ipv4_address: 172.28.0.12 + ports: + - "33221:33212" # gRPC sidecar + - "7071:7070" # OptimumP2P + - "9091:9090" # API + volumes: + - ./identity:/identity + + p2pnode-2: + image: 'getoptimum/p2pnode:latest' + platform: linux/amd64 + environment: + - LOG_LEVEL=debug + - CLUSTER_ID=p2pnode-2 + - NODE_MODE=optimum + - SIDECAR_PORT=33212 + - API_PORT=9090 + - IDENTITY_DIR=/identity + - OPTIMUM_PORT=7070 + - OPTIMUM_MAX_MSG_SIZE=1048576 + - OPTIMUM_MESH_TARGET=6 + - OPTIMUM_MESH_MIN=3 + - OPTIMUM_MESH_MAX=12 + - OPTIMUM_SHARD_FACTOR=4 + - OPTIMUM_SHARD_MULT=1.5 + - OPTIMUM_THRESHOLD=0.75 + - BOOTSTRAP_PEERS=/ip4/172.28.0.12/tcp/7070/p2p/${BOOTSTRAP_PEER_ID} + networks: + optimum-network: + ipv4_address: 172.28.0.13 + ports: + - "33222:33212" + - "7072:7070" + - "9092:9090" + +networks: + optimum-network: + driver: bridge + ipam: + config: + - subnet: 172.28.0.0/16 +``` + +## Environment Variables + +### Core Configuration + +- **`NODE_MODE`**: P2P protocol implementation mode + - Purpose: Selects the underlying pub/sub protocol + - Values: `optimum` (RLNC-enhanced) or `gossipsub` (standard libp2p) + - Default: `optimum` + - Usage: Use `optimum` for better performance with RLNC coding + - Note: All nodes in network should use same mode for compatibility + +- **`CLUSTER_ID`**: Unique identifier for each P2P node + - Purpose: Distinguishes nodes in logs, metrics, and network topology + - Usage: Should be unique across all P2P nodes in deployment + - Format: Alphanumeric string, typically `p2pnode-1`, `p2pnode-2`, etc. + - Required: Yes, no default value + +- **`LOG_LEVEL`**: Logging verbosity level + - Purpose: Controls the amount and detail of log output + - Values: `debug`, `info`, `warn`, `error` + - Default: `info` + - Usage Guide: + - `debug`: Most verbose, includes detailed P2P protocol events (development) + - `info`: Standard level with operational information + - `warn`: Only warnings and errors (quiet operation) + - `error`: Only error messages (minimal logging) + +### Port Configuration + +- **`SIDECAR_PORT`**: gRPC bidirectional communication port + - Purpose: Port where clients connect directly to interact with P2P node + - Default: `33212` + - Range: Any available port + - Usage: Must be accessible from client applications for direct P2P access + - Network: Used for client-to-node gRPC communication + +- **`API_PORT`**: HTTP monitoring and management API port + - Purpose: Exposes REST endpoints for health checks, node state, and metrics + - Default: `9090` + - Range: Any available port + - Endpoints: `/api/v1/health`, `/api/v1/node-state`, `/api/v1/version` + - Usage: Used for operational monitoring and debugging + +- **`OPTIMUM_PORT`**: P2P protocol communication port + - Purpose: Port for peer-to-peer communication between nodes + - Default: `7070` + - Range: Any available port + - Usage: Must be accessible between all P2P nodes in the mesh + - Network: Used for inter-node OptimumP2P protocol communication + +### RLNC Parameters + +- **`OPTIMUM_MAX_MSG_SIZE`**: Maximum message size in bytes + - Purpose: Limits individual message size to prevent memory issues + - Default: `1048576` (1MB) + - Usage: Larger values allow bigger payloads but use more memory/bandwidth + + +- **`OPTIMUM_SHARD_FACTOR`**: Number of coded shards per message + - Purpose: Controls how many pieces each message is split into for RLNC + - Default: `4` + - Impact: Higher values increase redundancy and fault tolerance but use more bandwidth + - Note: Must be non-zero to enable RLNC functionality + +- **`OPTIMUM_SHARD_MULT`**: Shard size redundancy multiplier + - Purpose: Controls redundancy factor for error recovery + - Default: `1.5` + - Impact: Higher values improve error recovery but increase bandwidth usage + - Note: Values >1.0 add redundancy (e.g., 1.5 = 50% redundancy) + + +- **`OPTIMUM_THRESHOLD`**: Forward/decode threshold ratio + - Purpose: Fraction of shards needed before forwarding or decoding message + - Default: `0.75` (75%) + - Validation: Must be between 0 and 1 (exclusive of 0, inclusive of 1) + - Impact: Critical for network performance tuning + - Note: Lower values reduce latency but may reduce reliability; higher values improve reliability but increase latency + +### Mesh Topology Configuration + +- **`OPTIMUM_MESH_TARGET`**: Target number of peer connections + - Purpose: Ideal number of peers each node connects to in the mesh + - Default: `6` + - Range: 3-50 (typically 4-12 for most deployments) + - Impact: Higher values increase redundancy and fault tolerance but use more resources + + +- **`OPTIMUM_MESH_MIN`**: Minimum mesh peer connections + - Purpose: Minimum connections before attempting to add more peers + - Default: `4` + - Note: Should be less than MESH_TARGET + - Impact: Lower values reduce fault tolerance, higher values increase resource usage + - Usage: Prevents network partitioning by maintaining minimum connectivity + +- **`OPTIMUM_MESH_MAX`**: Maximum mesh peer connections + - Purpose: Maximum connections before pruning excess peers + - Default: `12` + - Note: Should be greater than MESH_TARGET + - Impact: Prevents resource exhaustion while maintaining network connectivity + - Usage: Sets upper bound to control memory and bandwidth usage + +### Bootstrap and Identity Configuration + +- **`BOOTSTRAP_PEERS`**: Initial peer discovery addresses + - Purpose: List of known peers for joining the mesh network + - Format: `/ip4//tcp//p2p/` + - Example: `/ip4/172.28.0.12/tcp/7070/p2p/12D3KooW...` + - Usage: New nodes use these to discover and join the network + - Multiple: Can specify multiple peers separated by commas + - Note: At least one bootstrap peer must be accessible for network joining + +- **`IDENTITY_DIR`**: Node cryptographic identity directory + - Purpose: Directory containing node's P2P identity key (p2p.key) + - Default: `/identity` + - Usage: Only needed for bootstrap nodes, other nodes auto-generate identity + - Security: Contains private key material, should be properly secured + - Permissions: Ensure proper file permissions (600) for security + +### Configuration Notes + +All parameter values can be adjusted based on specific use case requirements. TWhen modifying parameters: + +- Higher OPTIMUM_THRESHOLD values improve reliability but increase latency +- Higher OPTIMUM_MESH_TARGET values improve fault tolerance but use more resources +- Higher OPTIMUM_SHARD_FACTOR values improve redundancy but increase bandwidth usage +- OPTIMUM_MESH_MIN should be less than OPTIMUM_MESH_TARGET +- OPTIMUM_MESH_MAX should be greater than OPTIMUM_MESH_TARGET + +## Starting the Network + +1. Generate peer identity (optional, will auto-generate if not present): +```bash +# Create identity directory +mkdir -p identity + +# Generate peer key +docker run --rm -v $(pwd)/identity:/identity getoptimum/p2pnode:latest generate-key +``` + +2. Set bootstrap peer ID: +```bash +export BOOTSTRAP_PEER_ID=$(docker run --rm -v $(pwd)/identity:/identity getoptimum/p2pnode:latest peer-id) +``` + +3. Start the network: +```bash +docker-compose up -d +``` + +4. Verify nodes are running: +```bash +docker-compose ps +docker-compose logs p2pnode-1 +``` + +## Client Connection + +Clients connect directly to P2P nodes via gRPC sidecar: + +```go +conn, err := grpc.Dial("localhost:33221", grpc.WithInsecure()) +if err != nil { + log.Fatal(err) +} +client := pb.NewP2PStreamClient(conn) +``` + +## Network Scaling + +To add more nodes, replicate the service definition with: +- Unique `CLUSTER_ID` +- Unique IP address in the network +- Unique port mappings +- Same `BOOTSTRAP_PEERS` configuration + +## Monitoring + +Access node metrics and status: +- Node 1: http://localhost:9091 +- Node 2: http://localhost:9092 + +## Troubleshooting + +### Peer Discovery Issues +Check bootstrap peer configuration and network connectivity: +```bash +docker-compose exec p2pnode-1 ping p2pnode-2 +docker-compose logs p2pnode-1 | grep "bootstrap" +``` + +### Mesh Formation +Verify peers are discovering each other: +```bash +curl http://localhost:9091/debug/peers +``` + +### Message Flow +Check message propagation in logs: +```bash +docker-compose logs -f | grep "message" +``` + +## See Also + +- [GossipSub Configuration Guide](/docs/hackathon/configuration/gossipsub.md) - Detailed explanation of underlying pub/sub protocol parameters +- [P2P with Proxy Deployment](/docs/hackathon/deployment/p2p-with-proxy.md) - Alternative deployment with proxy layer +- [First Message Tutorial](/docs/hackathon/quick-start/first-message.md) - Step-by-step getting started guide \ No newline at end of file diff --git a/docs/hackathon/deployment/p2p-with-proxy.md b/docs/hackathon/deployment/p2p-with-proxy.md new file mode 100644 index 0000000..3944368 --- /dev/null +++ b/docs/hackathon/deployment/p2p-with-proxy.md @@ -0,0 +1,455 @@ +# P2P Network with Proxy Deployment + +This guide covers deploying OptimumP2P with proxy services that provide HTTP/WebSocket/gRPC APIs for client applications. The proxy acts as a bridge between traditional client-server applications and the OptimumP2P network. + + +### Component Responsibilities + +**Proxy Layer:** +- Protocol translation (HTTP/WebSocket/gRPC ↔ libp2p) +- Client session management +- Load balancing across P2P nodes +- Rate limiting and authentication +- API versioning and compatibility + +**P2P Network Layer:** +- RLNC message encoding/decoding +- Mesh topology maintenance +- Peer discovery and routing +- Message propagation via GossipSub +- Network resilience and fault tolerance + + +## Deployment Configuration + +Complete setup with proxies and P2P nodes: + +```yaml +services: + proxy-1: + image: 'getoptimum/proxy:latest' + platform: linux/amd64 + ports: + - "8081:8080" # HTTP/WebSocket + - "50051:50051" # gRPC + environment: + - PROXY_PORT=:8080 + - CLUSTER_ID=proxy-1 + - ENABLE_AUTH=false + - LOG_LEVEL=debug + - P2P_NODES=p2pnode-1:33212,p2pnode-2:33212,p2pnode-3:33212,p2pnode-4:33212 + networks: + optimum-network: + ipv4_address: 172.28.0.10 + depends_on: + - p2pnode-1 + - p2pnode-2 + - p2pnode-3 + - p2pnode-4 + + proxy-2: + image: 'getoptimum/proxy:latest' + platform: linux/amd64 + ports: + - "8082:8080" + - "50052:50051" + environment: + - PROXY_PORT=:8080 + - CLUSTER_ID=proxy-2 + - ENABLE_AUTH=false + - LOG_LEVEL=debug + - P2P_NODES=p2pnode-1:33212,p2pnode-2:33212,p2pnode-3:33212,p2pnode-4:33212 + networks: + optimum-network: + ipv4_address: 172.28.0.11 + depends_on: + - p2pnode-1 + - p2pnode-2 + - p2pnode-3 + - p2pnode-4 + + p2pnode-1: + image: 'getoptimum/p2pnode:latest' + platform: linux/amd64 + volumes: + - ./identity:/identity + environment: + - LOG_LEVEL=debug + - CLUSTER_ID=p2pnode-1 + - NODE_MODE=optimum + - SIDECAR_PORT=33212 + - API_PORT=9090 + - IDENTITY_DIR=/identity + - OPTIMUM_PORT=7070 + - OPTIMUM_MAX_MSG_SIZE=1048576 + - OPTIMUM_MESH_TARGET=6 + - OPTIMUM_MESH_MIN=3 + - OPTIMUM_MESH_MAX=12 + - OPTIMUM_SHARD_FACTOR=4 + - OPTIMUM_SHARD_MULT=1.5 + - OPTIMUM_THRESHOLD=0.75 + - BOOTSTRAP_PEERS=/ip4/172.28.0.12/tcp/7070/p2p/${BOOTSTRAP_PEER_ID} + networks: + optimum-network: + ipv4_address: 172.28.0.12 + ports: + - "33221:33212" + - "7071:7070" + - "9091:9090" + + p2pnode-2: + image: 'getoptimum/p2pnode:latest' + platform: linux/amd64 + environment: + - LOG_LEVEL=debug + - CLUSTER_ID=p2pnode-2 + - NODE_MODE=optimum + - SIDECAR_PORT=33212 + - API_PORT=9090 + - IDENTITY_DIR=/identity + - OPTIMUM_PORT=7070 + - OPTIMUM_MAX_MSG_SIZE=1048576 + - OPTIMUM_MESH_TARGET=6 + - OPTIMUM_MESH_MIN=3 + - OPTIMUM_MESH_MAX=12 + - OPTIMUM_SHARD_FACTOR=4 + - OPTIMUM_SHARD_MULT=1.5 + - OPTIMUM_THRESHOLD=0.75 + - BOOTSTRAP_PEERS=/ip4/172.28.0.12/tcp/7070/p2p/${BOOTSTRAP_PEER_ID} + networks: + optimum-network: + ipv4_address: 172.28.0.13 + ports: + - "33222:33212" + - "7072:7070" + - "9092:9090" + + p2pnode-3: + image: 'getoptimum/p2pnode:latest' + platform: linux/amd64 + environment: + - LOG_LEVEL=debug + - CLUSTER_ID=p2pnode-3 + - NODE_MODE=optimum + - SIDECAR_PORT=33212 + - API_PORT=9090 + - IDENTITY_DIR=/identity + - OPTIMUM_PORT=7070 + - OPTIMUM_MAX_MSG_SIZE=1048576 + - OPTIMUM_MESH_TARGET=6 + - OPTIMUM_MESH_MIN=3 + - OPTIMUM_MESH_MAX=12 + - OPTIMUM_SHARD_FACTOR=4 + - OPTIMUM_SHARD_MULT=1.5 + - OPTIMUM_THRESHOLD=0.75 + - BOOTSTRAP_PEERS=/ip4/172.28.0.12/tcp/7070/p2p/${BOOTSTRAP_PEER_ID} + networks: + optimum-network: + ipv4_address: 172.28.0.14 + ports: + - "33223:33212" + - "7073:7070" + - "9093:9090" + + p2pnode-4: + image: 'getoptimum/p2pnode:latest' + platform: linux/amd64 + environment: + - LOG_LEVEL=debug + - CLUSTER_ID=p2pnode-4 + - NODE_MODE=optimum + - SIDECAR_PORT=33212 + - API_PORT=9090 + - IDENTITY_DIR=/identity + - OPTIMUM_PORT=7070 + - OPTIMUM_MAX_MSG_SIZE=1048576 + - OPTIMUM_MESH_TARGET=6 + - OPTIMUM_MESH_MIN=3 + - OPTIMUM_MESH_MAX=12 + - OPTIMUM_SHARD_FACTOR=4 + - OPTIMUM_SHARD_MULT=1.5 + - OPTIMUM_THRESHOLD=0.75 + - BOOTSTRAP_PEERS=/ip4/172.28.0.12/tcp/7070/p2p/${BOOTSTRAP_PEER_ID} + networks: + optimum-network: + ipv4_address: 172.28.0.15 + ports: + - "33224:33212" + - "7074:7070" + - "9094:9090" + +networks: + optimum-network: + driver: bridge + ipam: + config: + - subnet: 172.28.0.0/16 +``` + +## P2P Node Environment Variables + +### Core P2P Configuration + +- **`CLUSTER_ID`**: Unique identifier for each P2P node + - Purpose: Distinguishes nodes in logs, metrics, and network topology + - Usage: Should be unique across all P2P nodes in deployment + - Format: Alphanumeric string, typically `p2pnode-1`, `p2pnode-2`, etc. + - Required: Yes, no default value + +- **`NODE_MODE`**: P2P protocol implementation mode + - Purpose: Selects the underlying pub/sub protocol + - Values: `optimum` (RLNC-enhanced) or `gossipsub` (standard libp2p) + - Default: `optimum` + - Usage: Use `optimum` for better performance with RLNC coding + - Note: All nodes in network should use same mode for compatibility + +- **`SIDECAR_PORT`**: gRPC bidirectional communication port + - Purpose: Port where proxys and clients connect to interact with P2P node + - Default: `33212` + - Range: Any available port (1024-65535) + - Usage: Must be accessible from proxy containers for internal communication + - Network: Used for internal container-to-container communication + +- **`API_PORT`**: HTTP monitoring and management API port + - Purpose: Exposes REST endpoints for health checks, node state, and metrics + - Default: `8081` for proxy, `9090` for P2P nodes (varies by component) + - Endpoints: `/api/v1/health`, `/api/v1/node-state`, `/api/v1/version` + - Usage: Used for operational monitoring and debugging + +### Network Discovery Configuration + +- **`IDENTITY_DIR`**: Node cryptographic identity directory + - Purpose: Directory containing node's P2P identity key (p2p.key) + - Default: `/identity` + - Usage: Only needed for bootstrap nodes, other nodes auto-generate identity + - Security: Contains private key material, should be properly secured + +- **`BOOTSTRAP_PEERS`**: Initial peer discovery addresses + - Purpose: List of known peers for joining the mesh network + - Format: `/ip4//tcp//p2p/` + - Example: `/ip4/172.28.0.12/tcp/7070/p2p/12D3KooW...` + - Usage: New nodes use these to discover and join the network + - Multiple: Can specify multiple peers separated by commas + +### OptimumP2P Protocol Configuration + +- **`OPTIMUM_PORT`**: P2P protocol communication port + - Purpose: Port for peer-to-peer communication between nodes + - Default: `7070` + - Range: Any available port, commonly 7070-7080 + - Usage: Must be accessible between all P2P nodes in the mesh + - Network: Used for inter-node OptimumP2P protocol communication + +- **`OPTIMUM_MAX_MSG_SIZE`**: Maximum message size in bytes + - Purpose: Limits individual message size to prevent memory issues + - Default: `1048576` (1MB) + - Usage: Larger values allow bigger payloads but use more memory/bandwidth + +### RLNC (Random Linear Network Coding) Configuration + +- **`OPTIMUM_SHARD_FACTOR`**: Number of coded shards per message + - Purpose: Controls how many pieces each message is split into for RLNC + - Default: `4` + - Impact: Higher values increase redundancy and fault tolerance but use more bandwidth + - Note: Must be non-zero to enable RLNC functionality + +- **`OPTIMUM_SHARD_MULT`**: Shard size redundancy multiplier + - Purpose: Controls redundancy factor for error recovery + - Default: `1.5` + - Impact: Higher values improve error recovery but increase bandwidth usage + - Note: Values >1.0 add redundancy (e.g., 1.5 = 50% redundancy) + +- **`OPTIMUM_THRESHOLD`**: Forward/decode threshold ratio + - Purpose: Fraction of shards needed before forwarding or decoding message + - Default: `0.75` (75%) + - Validation: Must be between 0 and 1 (exclusive of 0, inclusive of 1) + - Impact: Critical for network performance tuning + - Note: Lower values reduce latency but may reduce reliability; higher values improve reliability but increase latency + +### Mesh Topology Configuration + +- **`OPTIMUM_MESH_TARGET`**: Target number of peer connections + - Purpose: Ideal number of peers each node connects to in the mesh + - Default: `6` + - Impact: Higher values increase redundancy and fault tolerance but use more resources + - Scaling: Adjust based on network size and reliability requirements + +- **`OPTIMUM_MESH_MIN`**: Minimum mesh peer connections + - Purpose: Minimum connections before attempting to add more peers + - Default: `4` + - Note: Should be less than `MESH_TARGET` + - Impact: Lower values reduce fault tolerance, higher values increase resource usage + - Usage: Prevents network partitioning by maintaining minimum connectivity + +- **`OPTIMUM_MESH_MAX`**: Maximum mesh peer connections + - Purpose: Maximum connections before pruning excess peers + - Default: `12` + - Note: Should be greater than `MESH_TARGET` + - Impact: Prevents resource exhaustion while maintaining network connectivity + - Usage: Sets upper bound to control memory and bandwidth usage + +### Configuration Notes + +All parameter values can be adjusted based on specific use case requirements. When modifying parameters: + +- Higher OPTIMUM_THRESHOLD values improve reliability but increase latency +- Higher OPTIMUM_MESH_TARGET values improve fault tolerance but use more resources +- Higher OPTIMUM_SHARD_FACTOR values improve redundancy but increase bandwidth usage +- OPTIMUM_MESH_MIN should be less than OPTIMUM_MESH_TARGET +- OPTIMUM_MESH_MAX should be greater than OPTIMUM_MESH_TARGET + +## Proxy Configuration + +### Environment Variables + +- **`PROXY_PORT`**: HTTP/WebSocket server port (default: `:8080`) + - Purpose: Defines the port where the proxy listens for client connections + - Usage: Internal container port for REST API and WebSocket connections + - Example: `:8080`, `:3000`, `:8081` + - Note: External port mapping is configured separately in docker-compose ports section + +- **`CLUSTER_ID`**: Unique proxy identifier + - Purpose: Distinguishes between multiple proxy instances in logs, metrics, and monitoring + - Usage: Should be unique across all proxy instances in your deployment + - Format: Alphanumeric string, no spaces + - Example: `proxy-1`, `proxy-primary`, `proxy-us-east` + - Required: Yes, no default value + +- **`ENABLE_AUTH`**: Enable Auth0 JWT authentication (true/false) + - Purpose: Controls whether API requests require JWT authentication + - Usage: Set to `false` for development/testing, `true` for production + - Values: `true` (requires valid JWT tokens) or `false` (open access) + - Default: `false` + - Note: When `true`, requires additional Auth0 configuration (AUTH0_DOMAIN, AUTH0_AUDIENCE) + +- **`P2P_NODES`**: Comma-separated list of P2P node gRPC sidecar addresses + - Purpose: Defines which P2P nodes the proxy can connect to for message routing + - Format: `hostname:port,hostname:port,...` + - Usage: Proxy load-balances requests across these nodes for high availability + - Port: Should match SIDECAR_PORT of P2P nodes (typically 33212) + - Example: `p2pnode-1:33212,p2pnode-2:33212,p2pnode-3:33212` + - Behavior: Proxy attempts to connect to all listed nodes and routes to healthy ones + +- **`LOG_LEVEL`**: Logging verbosity level + - Purpose: Controls the amount and detail of log output + - Values: `debug`, `info`, `warn`, `error` + - Default: `info` + - Usage Guide: + - `debug`: Most verbose, includes request/response details, connection events (development) + - `info`: Standard production level with operational information + - `warn`: Only warnings and errors (quiet production) + - `error`: Only error messages (minimal logging) + - Performance: Higher verbosity may impact performance and storage + +### Authentication (Optional) +For production deployments, enable authentication: +```yaml +environment: + - ENABLE_AUTH=true + - AUTH0_DOMAIN=your-domain.auth0.com + - AUTH0_AUDIENCE=your-api-audience +``` + +## API Endpoints + +### HTTP REST API +- `POST /publish` - Publish message to topic +- `GET /health` - Proxy health check +- `GET /metrics` - Prometheus metrics + +### WebSocket API +- `ws://localhost:8081/ws` - WebSocket connection for real-time messaging + +### gRPC API +- `localhost:50051` - Bidirectional streaming service + +## Starting the Network + +1. Generate peer identity: +```bash +mkdir -p identity +export BOOTSTRAP_PEER_ID=$(docker run --rm -v $(pwd)/identity:/identity getoptimum/p2pnode:latest generate-key --output-peer-id) +``` + +2. Start all services: +```bash +docker-compose up -d +``` + +3. Verify services: +```bash +docker-compose ps +curl http://localhost:8081/health +``` + +## Client Examples + +### HTTP REST +```bash +curl -X POST http://localhost:8081/publish \ + -H "Content-Type: application/json" \ + -d '{"topic": "test", "data": "Hello OptimumP2P"}' +``` + +### WebSocket (JavaScript) +```javascript +const ws = new WebSocket('ws://localhost:8081/ws'); +ws.onopen = () => { + ws.send(JSON.stringify({ + type: 'subscribe', + topic: 'test' + })); +}; +``` + +### gRPC (Go) +```go +conn, err := grpc.Dial("localhost:50051", grpc.WithInsecure()) +client := pb.NewProxyStreamClient(conn) +stream, err := client.Stream(context.Background()) +``` + +## Monitoring + +### Proxy Metrics +- Proxy 1: http://localhost:8081/metrics +- Proxy 2: http://localhost:8082/metrics + +### P2P Node Status +- Node 1: http://localhost:9091 +- Node 2: http://localhost:9092 +- Node 3: http://localhost:9093 +- Node 4: http://localhost:9094 + +## Scaling Considerations + +Scale the P2P network by adding more nodes: +- Maintain OPTIMUM_MESH_TARGET ratio to network size +- Consider network topology and latency +- Monitor mesh formation and message propagation + +## Troubleshooting + +### Proxy Connection Issues +```bash +# Check proxy logs +docker-compose logs proxy-1 + +# Test P2P node connectivity +docker-compose exec proxy-1 nc -zv p2pnode-1 33212 +``` + +### Message Delivery Issues +```bash +# Check message flow +docker-compose logs -f | grep -E "(publish|subscribe|message)" + +# Verify P2P mesh status +curl http://localhost:9091/debug/peers +``` + +## See Also + +- [GossipSub Configuration Guide](/docs/hackathon/configuration/gossipsub.md) - Detailed explanation of underlying pub/sub protocol parameters +- [P2P-Only Deployment](/docs/hackathon/deployment/p2p-only.md) - Alternative deployment without proxys +- [First Message Tutorial](/docs/hackathon/quick-start/first-message.md) - Step-by-step getting started guide diff --git a/docs/hackathon/overview.md b/docs/hackathon/overview.md new file mode 100644 index 0000000..2ff055c --- /dev/null +++ b/docs/hackathon/overview.md @@ -0,0 +1,69 @@ +# OptimumP2P Network Configuration Guide + +This guide provides everything you need to deploy, configure, and experiment with OptimumP2P. + +## What is OptimumP2P? + +OptimumP2P is a novel gossip algorithm that uses [Random Linear Network Coding (RLNC)](https://x.com/get_optimum/status/1891520664726802439) to enhance traditional gossip protocols like [GossipSub](https://github.com/libp2p/specs/tree/master/pubsub/gossipsub). For a comprehensive technical overview, see **[OptimumP2P Technical Overview](../learn/overview/p2p.md)**. + +## Quick Start + +Get OptimumP2P running in under 5 minutes: + +1. **[Installation](./quick-start/installation.md)** - Set up Docker and dependencies +2. **[First Message](./quick-start/first-message.md)** - Publish and receive your first message + +## Deployment Options + +Choose your deployment strategy: + +### P2P Network Only +- **[P2P-Only Deployment](./deployment/p2p-only.md)** - Deploy just the P2P nodes for direct integration + +### Proxy + P2P Network +- **[Proxy + P2P Deployment](./deployment/p2p-with-proxy.md)** - Full stack with REST/WebSocket APIs + +## Client Integration + +Connect to OptimumP2P using various client methods: + +- **[mump2p-cli](./clients/mump2p-cli.md)** - Command-line client for testing and automation +- **[gRPC Client Examples](./clients/grpc-examples.md)** - Direct gRPC integration examples + +## Configuration + +Fine-tune OptimumP2P for your use case: + +- **[Protocol Selection](./configuration/protocol-selection.md)** - Switch between OptimumP2P and GossipSub modes +- **[OptimumP2P Configuration](./configuration/optimump2p.md)** - RLNC parameters, mesh topology, and performance tuning +- **[GossipSub Configuration](./configuration/gossipsub.md)** - Standard GossipSub parameters for comparison + +## Performance & Comparison + +Benchmark and compare OptimumP2P: + +- **[Metrics Collection](./clients/)** - Understanding performance metrics +- **[OptimumP2P vs GossipSub](./clients/)** - Performance comparison methodology + +## Docker Images Used + +The hackathon uses these pre-built Docker images: + +- **`getoptimum/p2pnode:latest`** - Core P2P node with OptimumP2P protocol +- **`getoptimum/proxy:latest`** - Proxy service for client APIs + +Both images support protocol switching: +- `NODE_MODE=optimum` - RLNC-enhanced OptimumP2P (recommended) +- `NODE_MODE=gossipsub` - Standard GossipSub for comparison + +See **[Protocol Selection](./configuration/protocol-selection.md)** for detailed configuration and deployment examples. + +## Architecture Deep Dive + +For developers and integrators: + +- **[OptimumP2P Technical Overview](../learn/overview/p2p.md)** - Complete technical specification + +--- + +**Ready to get started?** Begin with **[Installation](./quick-start/installation.md)** or dive into the **[Technical Overview](../learn/overview/p2p.md)**. \ No newline at end of file diff --git a/docs/hackathon/quick-start/first-message.md b/docs/hackathon/quick-start/first-message.md new file mode 100644 index 0000000..9bc0da0 --- /dev/null +++ b/docs/hackathon/quick-start/first-message.md @@ -0,0 +1,809 @@ +# Your First Message + +This tutorial demonstrates how to send your first message through OptimumP2P using the proxy setup. We'll provide all the necessary configuration and code directly in this guide. + +## Prerequisites + +- Docker and Docker Compose installed +- Basic familiarity with terminal/command line +- Text editor for creating configuration files + +## Step 1: Set Up the Network + +### Step 1a: Create working directory + +```bash +mkdir optimump2p-test +``` + +### Step 1b: Navigate to directory + +```bash +cd optimump2p-test +``` + +### Step 1c: Create docker-compose.yml + +Create a `docker-compose.yml` file with the following content: + +```yaml +services: + proxy-1: + image: 'getoptimum/proxy:latest' + platform: linux/amd64 + ports: + - "8081:8080" # HTTP Port for the proxy + - "50051:50051" # gRPC Port for the proxy + environment: + - PROXY_PORT=:8080 + - CLUSTER_ID=proxy-1 + - ENABLE_AUTH=false + - LOG_LEVEL=debug + - P2P_NODES=p2pnode-1:33212,p2pnode-2:33212,p2pnode-3:33212,p2pnode-4:33212 + networks: + optimum-network: + ipv4_address: 172.28.0.10 + depends_on: + - p2pnode-1 + - p2pnode-2 + - p2pnode-3 + - p2pnode-4 + + proxy-2: + image: 'getoptimum/proxy:latest' + platform: linux/amd64 + ports: + - "8082:8080" # HTTP Port for the proxy + - "50052:50051" # gRPC Port for the proxy + environment: + - PROXY_PORT=:8080 + - CLUSTER_ID=proxy-2 + - ENABLE_AUTH=false + - LOG_LEVEL=debug + - P2P_NODES=p2pnode-1:33212,p2pnode-2:33212,p2pnode-3:33212,p2pnode-4:33212 + networks: + optimum-network: + ipv4_address: 172.28.0.11 + depends_on: + - p2pnode-1 + - p2pnode-2 + - p2pnode-3 + - p2pnode-4 + + p2pnode-1: + image: 'getoptimum/p2pnode:latest' + platform: linux/amd64 + volumes: + - ./identity:/identity + environment: + - LOG_LEVEL=debug + - CLUSTER_ID=p2pnode-1 + - NODE_MODE=optimum + - SIDECAR_PORT=33212 + - API_PORT=9090 + - IDENTITY_DIR=/identity + - BOOTSTRAP_PEERS=/ip4/172.28.0.12/tcp/7070/p2p/${BOOTSTRAP_PEER_ID} + - OPTIMUM_PORT=7070 + - OPTIMUM_MAX_MSG_SIZE=1048576 + - OPTIMUM_MESH_TARGET=6 + - OPTIMUM_MESH_MIN=3 + - OPTIMUM_MESH_MAX=12 + - OPTIMUM_SHARD_FACTOR=4 + - OPTIMUM_SHARD_MULT=1.5 + - OPTIMUM_THRESHOLD=0.75 + networks: + optimum-network: + ipv4_address: 172.28.0.12 + ports: + - "33221:33212" + - "7071:7070" + - "6061:6060" + - "9091:9090" + + p2pnode-2: + image: 'getoptimum/p2pnode:latest' + platform: linux/amd64 + environment: + - LOG_LEVEL=debug + - CLUSTER_ID=p2pnode-1 + - NODE_MODE=optimum + - SIDECAR_PORT=33212 + - API_PORT=9090 + - IDENTITY_DIR=/identity + - BOOTSTRAP_PEERS=/ip4/172.28.0.12/tcp/7070/p2p/${BOOTSTRAP_PEER_ID} + - OPTIMUM_PORT=7070 + - OPTIMUM_MAX_MSG_SIZE=1048576 + - OPTIMUM_MESH_TARGET=6 + - OPTIMUM_MESH_MIN=3 + - OPTIMUM_MESH_MAX=12 + - OPTIMUM_SHARD_FACTOR=4 + - OPTIMUM_SHARD_MULT=1.5 + - OPTIMUM_THRESHOLD=0.75 + networks: + optimum-network: + ipv4_address: 172.28.0.13 + ports: + - "33222:33212" + - "7072:7070" + - "6062:6060" + - "9092:9090" + + p2pnode-3: + image: 'getoptimum/p2pnode:latest' + platform: linux/amd64 + environment: + - LOG_LEVEL=debug + - CLUSTER_ID=p2pnode-1 + - NODE_MODE=optimum + - SIDECAR_PORT=33212 + - API_PORT=9090 + - IDENTITY_DIR=/identity + - OPTIMUM_PORT=7070 + - OPTIMUM_MAX_MSG_SIZE=1048576 + - OPTIMUM_MESH_TARGET=6 + - OPTIMUM_MESH_MIN=3 + - OPTIMUM_MESH_MAX=12 + - OPTIMUM_SHARD_FACTOR=4 + - OPTIMUM_SHARD_MULT=1.5 + - OPTIMUM_THRESHOLD=0.75 + - BOOTSTRAP_PEERS=/ip4/172.28.0.12/tcp/7070/p2p/${BOOTSTRAP_PEER_ID} + networks: + optimum-network: + ipv4_address: 172.28.0.14 + ports: + - "33223:33212" + - "7073:7070" + - "6063:6060" + - "9093:9090" + + p2pnode-4: + image: 'getoptimum/p2pnode:latest' + platform: linux/amd64 + environment: + - LOG_LEVEL=debug + - CLUSTER_ID=p2pnode-1 + - NODE_MODE=optimum + - SIDECAR_PORT=33212 + - API_PORT=9090 + - IDENTITY_DIR=/identity + - OPTIMUM_PORT=7070 + - OPTIMUM_MAX_MSG_SIZE=1048576 + - OPTIMUM_MESH_TARGET=6 + - OPTIMUM_MESH_MIN=3 + - OPTIMUM_MESH_MAX=12 + - OPTIMUM_SHARD_FACTOR=4 + - OPTIMUM_SHARD_MULT=1.5 + - OPTIMUM_THRESHOLD=0.75 + - BOOTSTRAP_PEERS=/ip4/172.28.0.12/tcp/7070/p2p/${BOOTSTRAP_PEER_ID} + networks: + optimum-network: + ipv4_address: 172.28.0.15 + ports: + - "33224:33212" + - "7074:7070" + - "6064:6060" + - "9094:9090" + +networks: + optimum-network: + driver: bridge + ipam: + config: + - subnet: 172.28.0.0/16 +``` + +## Step 2: Start the Network + +### Step 2a: Generate peer identity + +```bash +# Create identity directory if using local setup +mkdir -p identity +``` + +### Step 2b: Set environment variable + +```bash +# Set the bootstrap peer ID environment variable +export BOOTSTRAP_PEER_ID=12D3KooWExample # This will be generated automatically +``` + +### Step 2c: Start all services + +```bash +docker-compose up -d +``` + +### Step 2d: Verify services are running + +```bash +docker-compose ps +``` + +You should see containers for: +- proxy-1 (port 8081) +- proxy-2 (port 8082) +- p2pnode-1, p2pnode-2, p2pnode-3, p2pnode-4 + +## Step 3: Test Proxy Connectivity + +Before we start sending messages, let's make sure the proxy services are working correctly. The proxy provides REST API endpoints for publishing and subscribing to messages. + +### Step 3a: Test First Proxy + +Test the connection to proxy-1: + +```bash +# Test proxy-1 (should return HTTP headers if working) +curl -I http://localhost:8081 +``` + +**Expected output:** You should see HTTP response headers starting with `HTTP/1.1 200 OK` or similar. + +### Step 3b: Test Second Proxy + +Test the connection to proxy-2: + +```bash +# Test proxy-2 (should return HTTP headers if working) +curl -I http://localhost:8082 +``` + +**Expected output:** Similar HTTP response headers, confirming both proxies are running. + +**Troubleshooting:** If you get "connection refused" errors, check that your Docker containers are running with `docker-compose ps`. + +## Step 4: Create the Proxy Client + +Let's create a Go client application that makes it easy to interact with the proxy. + +### Step 4a: Create client directory + +```bash +mkdir grpc_proxy_client +cd grpc_proxy_client +``` + +### Step 4b: Initialize Go module + +```bash +go mod init proxy_client +``` + +### Step 4c: Add dependencies + +```bash +go get google.golang.org/grpc +go get google.golang.org/protobuf +``` + +### Step 4d: Create protobuf definition + +Create a `proto/proxy_stream.proto` file: + +```bash +mkdir proto +``` + +Create `proto/proxy_stream.proto` with this content: + +```protobuf +syntax = "proto3"; + +package proto; + +option go_package = "optimum-proxy/proto;proto"; + +service ProxyStream { + rpc ClientStream (stream ProxyMessage) returns (stream ProxyMessage); +} + +message ProxyMessage { + string client_id = 1; + bytes message = 2; + string topic = 3; + string message_id = 4; + string type = 5; +} +``` + +### Step 4e: Generate protobuf code + +```bash +mkdir grpc +protoc --go_out=grpc --go-grpc_out=grpc proto/proxy_stream.proto +``` + +### Step 4f: Create the client code + +Create `proxy_client.go` with this content: + +```go +package main + +import ( + "bytes" + "context" + crand "crypto/rand" + "encoding/hex" + "encoding/json" + "flag" + "fmt" + "io" + "log" + "math" + "math/rand" + "net/http" + "os" + "os/signal" + "time" + + protobuf "proxy_client/grpc" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/keepalive" +) + +const ( + proxyREST = "http://localhost:8081" + proxyGRPC = "localhost:50051" + defaultTopic = "demo" + defaultThreshold = 0.1 + defaultMsgCount = 5 + defaultDelay = 2 * time.Second +) + +var ( + topic = flag.String("topic", defaultTopic, "topic name") + threshold = flag.Float64("threshold", defaultThreshold, "delivery threshold (0.0 to 1.0)") + subscribeOnly = flag.Bool("subscribeOnly", false, "only subscribe and receive messages (no publishing)") + messageCount = flag.Int("count", defaultMsgCount, "number of messages to publish") + messageDelay = flag.Duration("delay", defaultDelay, "delay between message publishing") + + keepaliveTime = flag.Duration("keepalive-interval", 2*time.Minute, "gRPC keepalive interval") + keepaliveTimeout = flag.Duration("keepalive-timeout", 20*time.Second, "gRPC keepalive timeout") + + words = []string{"hello", "ping", "update", "broadcast", "status", "message", "event", "data", "note"} +) + +func main() { + flag.Parse() + + clientID := generateClientID() + log.Printf("[INFO] Client ID: %s | Topic: %s | Threshold: %.2f", clientID, *topic, *threshold) + + if err := subscribe(clientID, *topic, *threshold); err != nil { + log.Fatalf("subscribe error: %v", err) + } + + conn, err := grpc.NewClient(proxyGRPC, + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithInitialWindowSize(1024*1024*1024), + grpc.WithInitialConnWindowSize(1024*1024*1024), + grpc.WithDefaultCallOptions( + grpc.MaxCallRecvMsgSize(math.MaxInt), + grpc.MaxCallSendMsgSize(math.MaxInt), + ), + grpc.WithKeepaliveParams(keepalive.ClientParameters{ + Time: *keepaliveTime, + Timeout: *keepaliveTimeout, + PermitWithoutStream: false, + }), + ) + if err != nil { + log.Fatalf("gRPC connection failed: %v", err) + } + defer conn.Close() + + client := protobuf.NewProxyStreamClient(conn) + stream, err := client.ClientStream(context.Background()) + if err != nil { + log.Fatalf("stream open failed: %v", err) + } + + if err := stream.Send(&protobuf.ProxyMessage{ClientId: clientID}); err != nil { + log.Fatalf("client ID send failed: %v", err) + } + + go func() { + for { + resp, err := stream.Recv() + if err == io.EOF { + log.Println("[CLOSED] gRPC stream closed by server") + return + } + if err != nil { + log.Printf("[ERROR] stream receive: %v", err) + return + } + log.Printf("[RECEIVED] Topic: %s | Message: %s", resp.Topic, string(resp.Message)) + } + }() + + c := make(chan os.Signal, 1) + signal.Notify(c, os.Interrupt) + go func() { + <-c + log.Println("[INTERRUPTED] shutting down...") + os.Exit(0) + }() + + if *subscribeOnly { + select {} + } + + for i := 0; i < *messageCount; i++ { + msg := generateRandomMessage() + log.Printf("[PUBLISH] Message: %s", msg) + if err := publishMessage(clientID, *topic, msg); err != nil { + log.Printf("[ERROR] publish failed: %v", err) + } + time.Sleep(*messageDelay) + } + + time.Sleep(3 * time.Second) +} + +func subscribe(clientID, topic string, threshold float64) error { + body := map[string]interface{}{ + "client_id": clientID, + "topic": topic, + "threshold": threshold, + } + data, _ := json.Marshal(body) + resp, err := http.Post(proxyREST+"/api/subscribe", "application/json", bytes.NewReader(data)) + if err != nil { + return err + } + defer resp.Body.Close() + io.Copy(io.Discard, resp.Body) + return nil +} + +func publishMessage(clientID, topic, msg string) error { + body := map[string]interface{}{ + "client_id": clientID, + "topic": topic, + "message": msg, + } + data, _ := json.Marshal(body) + resp, err := http.Post(proxyREST+"/api/publish", "application/json", bytes.NewReader(data)) + if err != nil { + return err + } + defer resp.Body.Close() + io.Copy(io.Discard, resp.Body) + return nil +} + +func generateClientID() string { + b := make([]byte, 4) + _, _ = crand.Read(b) + return "client_" + hex.EncodeToString(b) +} + +func generateRandomMessage() string { + return fmt.Sprintf("%s @ %s", words[rand.Intn(len(words))], time.Now().Format("15:04:05")) +} +``` + +### Step 4g: Build the client + +```bash +go build -o proxy-client ./proxy_client.go +``` + +## Step 5: Subscribe to Messages + +Now we'll set up a message subscriber. This is like tuning into a radio station - once subscribed, you'll receive all messages published to that topic. + +### Step 5a: Open a New Terminal + +Open a **new terminal window** and navigate to your project directory: + +```bash +# Make sure you're in the right directory +cd optimump2p-test +``` + +### Step 5b: Start the Subscriber + +In this terminal, start listening for messages: + +```bash +# Subscribe to topic "demo" with threshold 0.7 +./proxy-client -topic=demo -threshold=0.7 -subscribeOnly +``` + +**What this command does:** +- `subscribe`: Tells the client to listen for messages +- `demo`: The topic name we're subscribing to +- `0.7`: The threshold value (higher = more reliable message reconstruction) + +### Step 5c: Understand the Subscriber Output + +The subscriber will: +1. **Register** with the proxy via REST API (`/api/subscribe`) +2. **Open** a gRPC stream to receive messages in real-time +3. **Display** received messages in the format: `[RECEIVED] Topic: | Message: ` + +**Keep this terminal open** - it will show incoming messages as they arrive. + +## Step 6: Publish Messages + +Now let's send some messages! We'll use a second terminal to publish messages while watching them appear in the subscriber terminal. + +### Step 6a: Open Another Terminal + +Open a **second new terminal window** and navigate to your project directory: + +```bash +# Navigate to the project directory +cd optimump2p-test +``` + +### Step 6b: Send Messages + +Publish messages to the same topic: + +```bash +# Publish 5 messages to topic "demo" with threshold 0.5 +./proxy-client -topic=demo -threshold=0.5 -count=5 +``` + +**What this command does:** +- `publish`: Tells the client to send messages +- `demo`: The topic name (must match the subscriber's topic) +- `0.5`: The threshold for message encoding +- `5`: Number of messages to send + +### Step 6c: Watch the Results + +The publisher will: +1. **Send** messages via REST API (`/api/publish`) +2. **Display** each published message in the format: `[PUBLISH] Message: ` + +**In your subscriber terminal** (from Step 5), you should see the messages being received in real-time. + +### Step 6d: Verify Message Flow + +Check that: +- The **publisher terminal** shows: `[PUBLISH] Message: Message 1`, `[PUBLISH] Message: 2`, etc. +- The **subscriber terminal** shows: `[RECEIVED] Topic: demo | Message: Message 1`, etc. + +This confirms that messages are flowing correctly through the OptimumP2P network! + +## Step 7: Alternative REST API Usage + +You can also interact directly with the REST API using curl: + +```bash +curl -X POST http://localhost:8081/api/subscribe \ + -H "Content-Type: application/json" \ + -d '{ + "client_id": "test_client_001", + "topic": "demo", + "threshold": 0.7 + }' +``` + +**What this does:** +- **POST request** to the subscription endpoint +- **client_id**: Unique identifier for your client +- **topic**: The topic you want to subscribe to +- **threshold**: Message reliability setting + +**Important note:** This only registers the subscription. To actually receive messages, you'd need to separately open a gRPC stream (which the script handles automatically). + +### Step 7c: Publish via REST API + +Send a message using a direct HTTP request: + +```bash +curl -X POST http://localhost:8081/api/publish \ + -H "Content-Type: application/json" \ + -d '{ + "client_id": "test_client_001", + "topic": "demo", + "message": "Hello from REST API" + }' +``` + +**What this does:** +- **POST request** to the publish endpoint +- **client_id**: Same client ID as used for subscription +- **topic**: Must match your subscription topic +- **message**: The actual message content to send + +### Step 7d: Test the REST API + +To see this working: + +1. **Keep your subscriber running** from Step 5 (if still open) +2. **Run the publish curl command** from Step 7c +3. **Check your subscriber terminal** - you should see: `[RECEIVED] Topic: demo | Message: Hello from REST API` + +This demonstrates that REST API publishing works with script-based subscribing (and vice versa). + +## Step 8: Multiple Topics + +OptimumP2P supports multiple topics running simultaneously. This means different applications or message types can be isolated from each other. Let's test this feature by setting up multiple subscribers and publishers. + +### Step 8a: Open Multiple Terminal Windows + +You'll need **4 separate terminal windows** for this test. Each terminal should be in your project directory: + +```bash +# In each new terminal, navigate to your project directory +cd optimump2p-test +``` + +### Step 8b: Set Up First Topic Subscriber + +In **Terminal 1**, subscribe to the "weather" topic: + +```bash +# This will listen for weather-related messages +./proxy-client -topic=weather -threshold=0.7 -subscribeOnly +``` + +**What this does:** Creates a subscriber that will only receive messages published to the "weather" topic. + +### Step 8c: Set Up Second Topic Subscriber + +In **Terminal 2**, subscribe to the "news" topic: + +```bash +# This will listen for news-related messages +./proxy-client -topic=news -threshold=0.7 -subscribeOnly +``` + +**What this does:** Creates a second subscriber for a completely different topic. This subscriber won't see weather messages. + +### Step 8d: Publish to Weather Topic + +In **Terminal 3**, publish messages to the weather topic: + +```bash +# Send 3 weather messages +./proxy-client -topic=weather -threshold=0.5 -count=3 +``` + +**Expected result:** Only Terminal 1 (weather subscriber) should show these messages. Terminal 2 (news subscriber) should remain quiet. + +### Step 8e: Publish to News Topic + +In **Terminal 4**, publish messages to the news topic: + +```bash +# Send 3 news messages +./proxy-client -topic=news -threshold=0.5 -count=3 +``` + +**Expected result:** Only Terminal 2 (news subscriber) should show these messages. Terminal 1 (weather subscriber) should not see them. + +### Step 8f: Verify Topic Isolation + +Check that each subscriber only received messages from their subscribed topic: + +- **Terminal 1** should show: `[RECEIVED] Topic: weather | Message: ...` +- **Terminal 2** should show: `[RECEIVED] Topic: news | Message: ...` + +This demonstrates that topics are completely isolated from each other. + +## Step 9: Load Testing + +Now let's test how the network handles high message volumes. This helps you understand the performance characteristics of OptimumP2P. + +### Step 9a: Prepare for Load Testing + +First, make sure you have a subscriber running to see the messages: + +```bash +# In a new terminal, subscribe to the performance topic +./proxy-client -topic=performance -threshold=0.7 -subscribeOnly +``` + +### Step 9b: Medium Load Test + +Start with a moderate load test: + +```bash +# Send 100 messages with 100ms delay between each +./proxy-client -topic=performance -threshold=0.5 -count=100 -delay=100ms +``` + +**What this does:** Sends 100 messages over about 10 seconds (100ms Γ— 100 = 10 seconds). + +### Step 9c: High Load Test + +For a more intensive test: + +```bash +# Send 1000 messages with 10ms delay (very fast) +./proxy-client -topic=stress -threshold=0.5 -count=1000 -delay=10ms +``` + +**What this does:** Sends 1000 messages over about 10 seconds (10ms Γ— 1000 = 10 seconds). + +**Note:** Make sure to set up a subscriber for the "stress" topic if you want to see these messages being received. + +## Understanding the Parameters + +### Threshold +The threshold parameter (0.0 to 1.0) controls RLNC decoding: +- Lower values: Messages decoded with fewer shards (faster, less reliable) +- Higher values: Messages require more shards (slower, more reliable) +- Typical range: 0.5 to 0.8 + +### Client ID +Each client needs a unique identifier for the proxy to track subscriptions and route messages correctly. + +### Topic Names +- Case-sensitive strings +- No special characters recommended +- Use descriptive names (e.g., "chat", "alerts", "data-feed") + +## Monitoring and Debugging + +### View logs +```bash +# Proxy logs +docker-compose logs -f proxy-1 + +# P2P node logs +docker-compose logs -f p2pnode-1 + +# All logs +docker-compose logs -f +``` + +### Check network connectivity +```bash +# Test P2P node connectivity +docker-compose exec proxy-1 nc -zv p2pnode-1 33212 + +# View container network +docker network inspect optimump2p-test_optimum-network +``` + +## Cleanup + +Stop and remove all containers: + +```bash +docker-compose down + +# Remove volumes and images (optional) +docker-compose down -v --rmi all +``` + +## Next Steps + +Now that you've successfully sent messages through OptimumP2P: + +1. **Build custom clients**: Explore the [mump2p-cli](../clients/mump2p-cli.md) or [gRPC Client Examples](../clients/grpc-examples.md) +2. **Optimize configuration**: Learn about [OptimumP2P Configuration](../configuration/optimump2p.md) +3. **Compare protocols**: Review [GossipSub Configuration](../configuration/gossipsub.md) + +## Troubleshooting + +### Build failures +- Ensure Go is installed: `go version` +- Check module dependencies: `go mod tidy` + +### Connection issues +- Verify Docker containers are running: `docker-compose ps` +- Check port availability: `netstat -an | grep 8081` +- Review firewall settings + +### Message delivery issues +- Verify topic names match exactly +- Check threshold values are between 0.0 and 1.0 +- Review proxy and P2P node logs for errors + +### Performance issues +- Monitor container resources: `docker stats` +- Adjust message delay parameters +- Check network bandwidth and latency \ No newline at end of file diff --git a/docs/hackathon/quick-start/installation.md b/docs/hackathon/quick-start/installation.md new file mode 100644 index 0000000..858601c --- /dev/null +++ b/docs/hackathon/quick-start/installation.md @@ -0,0 +1,80 @@ +# Installation & Setup + +This guide will help you install and set up OptimumP2P on your computer. We'll explain each step clearly, so don't worry if you're new to some of these tools. + +## What You'll Need + +Before we start, you need these programs on your computer: + +### 1. Docker +Docker is like a virtual container that runs applications in an isolated environment. Think of it as a box where we put our OptimumP2P network so it doesn't interfere with other programs on your computer. + +**Install Docker:** +- **Windows/Mac**: Download Docker Desktop from [docker.com](https://www.docker.com/products/docker-desktop/) +- **Linux**: Follow the instructions for your distribution on [docs.docker.com](https://docs.docker.com/engine/install/) + +**How to check if Docker is installed:** +Open your terminal (Command Prompt on Windows, Terminal on Mac/Linux) and type: +```bash +docker --version +``` + + +### 2. Docker Compose +Docker Compose helps us run multiple Docker containers together. It usually comes with Docker Desktop, but let's verify: + +```bash +docker-compose --version +``` + + +### 3. Git (Optional but recommended) +Git helps you download and manage code. Most computers have it pre-installed. + +```bash +git --version +``` + +## Getting the Code + +The OptimumP2P setup files are provided directly in this documentation. You'll create the necessary configuration files as part of the tutorial steps. + +All required configurations and code snippets are included in the [Quick Start Guide](./first-message.md) - no external repositories needed! + +## Troubleshooting + +### "Docker command not found" +- Make sure Docker is installed and running +- On Windows/Mac, start Docker Desktop application +- You might need to restart your terminal after installation + +### "Permission denied" on Linux +Add your user to the docker group: +```bash +sudo usermod -aG docker $USER +``` +Then log out and log back in. + +### "Port already in use" +If you see port conflicts, either: +- Stop other programs using those ports +- Change the port numbers in the docker-compose.yml file + +### Still having issues? +- Check Docker's own troubleshooting guide +- Make sure you have enough disk space (at least 2GB free) +- Restart your computer and try again + +## System Requirements + +**Minimum:** +- 4GB RAM +- 2GB free disk space +- Internet connection for downloading Docker images + +**Recommended:** +- 8GB RAM +- 5GB free disk space +- Stable internet connection + +You're now ready to start your OptimumP2P journey! \ No newline at end of file diff --git a/index.md b/index.md index ba25200..7c26c88 100644 --- a/index.md +++ b/index.md @@ -12,21 +12,21 @@ hero: alt: Optimum actions: - theme: brand - text: Get Started - link: /docs/how-to-guides/overview + text: Hackathon Docs + link: /docs/hackathon/overview - theme: alt - text: Introduction + text: Get Started link: /docs/learn/overview/intro features: - - title: Overview + - title: Hackathon Documentation + details: Complete guide for building applications on OptimumP2P with deployment options, client examples, and configuration. + link: /docs/hackathon/overview + icon: πŸš€ + - title: Learn Optimum details: Learn more about Optimum, the various components involved, and how it could benefit your project. link: /docs/learn/overview/intro icon: πŸŽ“ - - title: How-to Guides (Under Construction 🚧) - details: Explore step-by-step guides for running and integrating with various types of nodes, how to read and write data, and how to gossip. - # link: /docs/how-to-guides/overview - icon: βš™οΈ - title: Research details: Access resources for understanding the inner workings of Optimum and the research behind the technology. link: /docs/research/overview diff --git a/package.json b/package.json index f2211cb..f66540c 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,7 @@ }, "devDependencies": { "@eslint/js": "^9.22.0", + "@types/node": "^24.1.0", "@typescript-eslint/eslint-plugin": "^8.26.1", "@typescript-eslint/parser": "^8.26.1", "eslint": "^9.22.0", diff --git a/static/img/P2P_data_flow_dark.png b/static/img/P2P_data_flow_dark.png new file mode 100644 index 0000000..bf3a3e2 Binary files /dev/null and b/static/img/P2P_data_flow_dark.png differ diff --git a/static/img/P2P_data_flow_light.png b/static/img/P2P_data_flow_light.png new file mode 100644 index 0000000..6de7ae1 Binary files /dev/null and b/static/img/P2P_data_flow_light.png differ diff --git a/static/img/img_2.png b/static/img/img_2.png new file mode 100644 index 0000000..fda31da Binary files /dev/null and b/static/img/img_2.png differ