From 2e081aaae8a2a52d58f2a56a5cae89fbf4abd6a4 Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Thu, 4 Apr 2024 16:33:01 -0700 Subject: [PATCH 1/4] Add libp2phttp endpoints --- lib/libp2phttp.go | 71 ++++++++++++++++++++ lib/libp2phttp_test.go | 143 +++++++++++++++++++++++++++++++++++++++++ main.go | 68 +++++++++++++++++++- 3 files changed, 281 insertions(+), 1 deletion(-) create mode 100644 lib/libp2phttp.go create mode 100644 lib/libp2phttp_test.go diff --git a/lib/libp2phttp.go b/lib/libp2phttp.go new file mode 100644 index 0000000..2bd4a09 --- /dev/null +++ b/lib/libp2phttp.go @@ -0,0 +1,71 @@ +package vole + +import ( + "context" + "net" + "net/http" + "net/http/httputil" + + "github.com/libp2p/go-libp2p/core/host" + "github.com/libp2p/go-libp2p/core/peer" + libp2phttp "github.com/libp2p/go-libp2p/p2p/http" + "github.com/multiformats/go-multiaddr" +) + +func Libp2pHTTPSocketProxy(ctx context.Context, p multiaddr.Multiaddr, unixSocketPath string) error { + h, err := libp2pHost() + if err != nil { + return err + } + + httpHost := libp2phttp.Host{StreamHost: h} + + ai := peer.AddrInfo{ + Addrs: []multiaddr.Multiaddr{p}, + } + idStr, err := p.ValueForProtocol(multiaddr.P_P2P) + if err == nil { + id, err := peer.Decode(idStr) + if err != nil { + return err + } + ai.ID = id + } + + rt, err := httpHost.NewConstrainedRoundTripper(ai) + if err != nil { + return err + } + rp := &httputil.ReverseProxy{ + Transport: rt, + Director: func(r *http.Request) {}, + } + + // Serves an HTTP server on the given path using unix sockets + server := &http.Server{ + Handler: rp, + } + + l, err := net.Listen("unix", unixSocketPath) + if err != nil { + return err + } + + go func() { + <-ctx.Done() + server.Close() + }() + + return server.Serve(l) +} + +// Libp2pHTTPServer serves an libp2p enabled HTTP server +func Libp2pHTTPServer() (host.Host, *libp2phttp.Host, error) { + h, err := libp2pHost() + if err != nil { + return nil, nil, err + } + + httpHost := &libp2phttp.Host{StreamHost: h} + return h, httpHost, nil +} diff --git a/lib/libp2phttp_test.go b/lib/libp2phttp_test.go new file mode 100644 index 0000000..a5f0539 --- /dev/null +++ b/lib/libp2phttp_test.go @@ -0,0 +1,143 @@ +package vole + +import ( + "context" + "fmt" + "net" + "net/http" + "os" + "testing" + "time" + + "github.com/multiformats/go-multiaddr" +) + +func TestHTTPProxyAndServer(t *testing.T) { + // Start libp2p HTTP server + h, hh, err := Libp2pHTTPServer() + if err != nil { + t.Fatal(err) + } + + go hh.Serve() + defer hh.Close() + + serverAddr := h.Addrs()[0].Encapsulate(multiaddr.StringCast("/p2p/" + h.ID().String())) + + ctx := context.Background() + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + socketFile, err := os.CreateTemp("", "libp2phttp-*.sock") + if err != nil { + t.Fatal(err) + } + + socketFile.Close() + os.Remove(socketFile.Name()) + + go func() { + err := Libp2pHTTPSocketProxy(ctx, serverAddr, socketFile.Name()) + if err != nil { + panic(err) + } + fmt.Println("err", err) + }() + + // Wait a bit to let the proxy start up. + for i := 0; i < 10; i++ { + time.Sleep(100 * time.Millisecond) + _, err := os.Stat(socketFile.Name()) + if err == nil { + break + } + } + + client := http.Client{ + Transport: &http.Transport{ + DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { + return net.Dial("unix", socketFile.Name()) + }, + }, + } + + // TODO update this when https://github.com/libp2p/go-libp2p/pull/2757 lands + resp, err := client.Get("http://example.com" + "/.well-known/libp2p") + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("unexpected status code: %d", resp.StatusCode) + } +} + +func TestHTTPProxyAndServerOverHTTPTransport(t *testing.T) { + // Start a basic http server + s := &http.Server{ + Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + }), + } + l, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatal(err) + } + go s.Serve(l) + defer s.Close() + + // get port of listener + _, port, err := net.SplitHostPort(l.Addr().String()) + if err != nil { + t.Fatal(err) + } + + serverAddr := multiaddr.StringCast("/ip4/127.0.0.1/tcp/" + port + "/http") + + ctx := context.Background() + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + socketFile, err := os.CreateTemp("", "libp2phttp-*.sock") + if err != nil { + t.Fatal(err) + } + + socketFile.Close() + os.Remove(socketFile.Name()) + + go func() { + err := Libp2pHTTPSocketProxy(ctx, serverAddr, socketFile.Name()) + if err != nil { + panic(err) + } + fmt.Println("err", err) + }() + + // Wait a bit to let the proxy start up. + for i := 0; i < 10; i++ { + time.Sleep(100 * time.Millisecond) + _, err := os.Stat(socketFile.Name()) + if err == nil { + break + } + } + + client := http.Client{ + Transport: &http.Transport{ + DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { + return net.Dial("unix", socketFile.Name()) + }, + }, + } + + // TODO update this when https://github.com/libp2p/go-libp2p/pull/2757 lands + resp, err := client.Get("http://example.com/") + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("unexpected status code: %d", resp.StatusCode) + } +} diff --git a/main.go b/main.go index 9455084..5a45849 100644 --- a/main.go +++ b/main.go @@ -2,9 +2,13 @@ package main import ( "bytes" + "context" "encoding/json" "fmt" + "net/http" "os" + "os/signal" + "syscall" madns "github.com/multiformats/go-multiaddr-dns" @@ -332,13 +336,75 @@ Note: may not work with some transports such as p2p-circuit (not applicable) and } return vole.Ping(c.Context, c.Bool("force-relay"), ai) }, + }, { + Name: "http", + ArgsUsage: "", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "socket-path", + Usage: `Use the specified path for the unix socket instead of making a new one.`, + DefaultText: "", + Value: "", + }, + }, + Usage: "Make http requests to the given multiaddr with a unix socket", + Description: `This command creates a unix socket that can be used with curl to make HTTP requests to the provided multiaddr. +Example: + vole libp2p http + # Output: + # Proxying on: + # /tmp/libp2phttp-abc.sock + + # In another terminal + curl --unix-socket /tmp/libp2phttp-abc.sock http://.well-known/libp2p/protocols`, + Action: func(c *cli.Context) error { + if c.NArg() != 1 { + return fmt.Errorf("invalid number of arguments") + } + + socketPath := c.String("socket-path") + if socketPath == "" { + f, err := os.CreateTemp("", "libp2phttp-*.sock") + if err != nil { + return err + } + // Remove this file since the listen will create it. We just wanted a random unused file path. + f.Close() + os.Remove(f.Name()) + socketPath = f.Name() + } + + fmt.Println("Proxying on:") + fmt.Println(socketPath) + + fmt.Println("\nExample curl request:") + fmt.Println("curl --unix-socket", socketPath, "http://example.com/") + + m, err := multiaddr.NewMultiaddr(c.Args().First()) + if err != nil { + return err + } + + err = vole.Libp2pHTTPSocketProxy(c.Context, m, socketPath) + if err == http.ErrServerClosed { + return nil + } + return err + }, }, }, }, }, } - err := app.Run(os.Args) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + defer cancel() + sigs := make(chan os.Signal, 1) + signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) + <-sigs + }() + err := app.RunContext(ctx, os.Args) if err != nil { panic(err) } From 83c956311ffc005b7a7c86a5a76fda32a333bd32 Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Thu, 4 Apr 2024 16:47:20 -0700 Subject: [PATCH 2/4] Add https support --- lib/libp2phttp.go | 85 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/lib/libp2phttp.go b/lib/libp2phttp.go index 2bd4a09..c7317c8 100644 --- a/lib/libp2phttp.go +++ b/lib/libp2phttp.go @@ -2,9 +2,18 @@ package vole import ( "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "fmt" + "math/big" "net" "net/http" "net/http/httputil" + "time" "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/peer" @@ -32,6 +41,25 @@ func Libp2pHTTPSocketProxy(ctx context.Context, p multiaddr.Multiaddr, unixSocke ai.ID = id } + hasTLS := false + hasHTTP := false + multiaddr.ForEach(p, func(c multiaddr.Component) bool { + if c.Protocol().Code == multiaddr.P_HTTP { + hasHTTP = true + } + + if c.Protocol().Code == multiaddr.P_HTTPS { + hasHTTP = true + hasTLS = true + return false + } + + if c.Protocol().Code == multiaddr.P_TLS { + hasTLS = true + } + return true + }) + rt, err := httpHost.NewConstrainedRoundTripper(ai) if err != nil { return err @@ -56,6 +84,20 @@ func Libp2pHTTPSocketProxy(ctx context.Context, p multiaddr.Multiaddr, unixSocke server.Close() }() + if hasTLS && hasHTTP { + c, err := selfSignedTLSConfig() + if err != nil { + + return err + } + server.TLSConfig = c + + fmt.Println("Endpoint is an HTTPS endpoint. Using a self signed cert locally to proxy.") + fmt.Println("Curl will only work with -k flag. This is only for debugging. Do *not* use this in production.") + + return server.ServeTLS(l, "", "") + } + return server.Serve(l) } @@ -69,3 +111,46 @@ func Libp2pHTTPServer() (host.Host, *libp2phttp.Host, error) { httpHost := &libp2phttp.Host{StreamHost: h} return h, httpHost, nil } + +func selfSignedTLSConfig() (*tls.Config, error) { + priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return nil, err + } + + notBefore := time.Now() + notAfter := notBefore.Add(365 * 24 * time.Hour) + + serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) + serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) + if err != nil { + return nil, err + } + + certTemplate := x509.Certificate{ + SerialNumber: serialNumber, + Subject: pkix.Name{ + Organization: []string{"Test"}, + }, + NotBefore: notBefore, + NotAfter: notAfter, + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + } + + derBytes, err := x509.CreateCertificate(rand.Reader, &certTemplate, &certTemplate, &priv.PublicKey, priv) + if err != nil { + return nil, err + } + + cert := tls.Certificate{ + Certificate: [][]byte{derBytes}, + PrivateKey: priv, + } + + tlsConfig := &tls.Config{ + Certificates: []tls.Certificate{cert}, + } + return tlsConfig, nil +} From 650d3276be90b14ff897e0c77d9390fb54d09fd5 Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Tue, 25 Jun 2024 16:32:26 -0700 Subject: [PATCH 3/4] Don't panic if the server only closed --- lib/libp2phttp_test.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/lib/libp2phttp_test.go b/lib/libp2phttp_test.go index a5f0539..cc9e6f3 100644 --- a/lib/libp2phttp_test.go +++ b/lib/libp2phttp_test.go @@ -2,7 +2,6 @@ package vole import ( "context" - "fmt" "net" "net/http" "os" @@ -38,10 +37,9 @@ func TestHTTPProxyAndServer(t *testing.T) { go func() { err := Libp2pHTTPSocketProxy(ctx, serverAddr, socketFile.Name()) - if err != nil { + if err != http.ErrServerClosed && err != nil { panic(err) } - fmt.Println("err", err) }() // Wait a bit to let the proxy start up. @@ -108,10 +106,9 @@ func TestHTTPProxyAndServerOverHTTPTransport(t *testing.T) { go func() { err := Libp2pHTTPSocketProxy(ctx, serverAddr, socketFile.Name()) - if err != nil { + if err != http.ErrServerClosed && err != nil { panic(err) } - fmt.Println("err", err) }() // Wait a bit to let the proxy start up. From 7e0d75c59f6e75b24e467c24e919fedec8f2e660 Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Wed, 26 Jun 2024 14:27:06 -0700 Subject: [PATCH 4/4] PR Comments --- lib/libp2phttp.go | 68 ++++++++++++++++++++++++++++-------------- lib/libp2phttp_test.go | 18 +++++++---- 2 files changed, 58 insertions(+), 28 deletions(-) diff --git a/lib/libp2phttp.go b/lib/libp2phttp.go index c7317c8..845132f 100644 --- a/lib/libp2phttp.go +++ b/lib/libp2phttp.go @@ -13,6 +13,8 @@ import ( "net" "net/http" "net/http/httputil" + "net/url" + "strconv" "time" "github.com/libp2p/go-libp2p/core/host" @@ -29,44 +31,64 @@ func Libp2pHTTPSocketProxy(ctx context.Context, p multiaddr.Multiaddr, unixSocke httpHost := libp2phttp.Host{StreamHost: h} - ai := peer.AddrInfo{ - Addrs: []multiaddr.Multiaddr{p}, + ai, err := peer.AddrInfoFromP2pAddr(p) + if err == peer.ErrInvalidAddr { + ai = &peer.AddrInfo{Addrs: []multiaddr.Multiaddr{p}} // No peer id + err = nil } - idStr, err := p.ValueForProtocol(multiaddr.P_P2P) - if err == nil { - id, err := peer.Decode(idStr) - if err != nil { - return err - } - ai.ID = id + if err != nil { + return err } hasTLS := false hasHTTP := false + host := "" + port := 0 multiaddr.ForEach(p, func(c multiaddr.Component) bool { - if c.Protocol().Code == multiaddr.P_HTTP { + switch c.Protocol().Code { + case multiaddr.P_TLS: + hasTLS = true + case multiaddr.P_HTTP: hasHTTP = true - } - - if c.Protocol().Code == multiaddr.P_HTTPS { + case multiaddr.P_HTTPS: hasHTTP = true hasTLS = true + case multiaddr.P_IP4, multiaddr.P_IP6, multiaddr.P_DNS4, multiaddr.P_DNS6, multiaddr.P_DNS: + host = c.Value() + case multiaddr.P_TCP, multiaddr.P_UDP: + port, err = strconv.Atoi(c.Value()) return false } - - if c.Protocol().Code == multiaddr.P_TLS { - hasTLS = true - } return true }) + if err != nil { + return err + } + if port == 0 && hasHTTP { + port = 80 + if hasTLS { + port = 443 + } + } - rt, err := httpHost.NewConstrainedRoundTripper(ai) + rt, err := httpHost.NewConstrainedRoundTripper(*ai) if err != nil { return err } - rp := &httputil.ReverseProxy{ - Transport: rt, - Director: func(r *http.Request) {}, + + var rp http.Handler + if hasTLS && hasHTTP { + u, err := url.Parse("https://" + host + ":" + strconv.Itoa(port) + "/") + if err != nil { + return err + } + revProxy := httputil.NewSingleHostReverseProxy(u) + rp = revProxy + } else { + rp = &httputil.ReverseProxy{ + Transport: rt, + Director: func(r *http.Request) {}, + } } // Serves an HTTP server on the given path using unix sockets @@ -101,8 +123,8 @@ func Libp2pHTTPSocketProxy(ctx context.Context, p multiaddr.Multiaddr, unixSocke return server.Serve(l) } -// Libp2pHTTPServer serves an libp2p enabled HTTP server -func Libp2pHTTPServer() (host.Host, *libp2phttp.Host, error) { +// libp2pHTTPServer serves an libp2p enabled HTTP server +func libp2pHTTPServer() (host.Host, *libp2phttp.Host, error) { h, err := libp2pHost() if err != nil { return nil, nil, err diff --git a/lib/libp2phttp_test.go b/lib/libp2phttp_test.go index cc9e6f3..0ade3b4 100644 --- a/lib/libp2phttp_test.go +++ b/lib/libp2phttp_test.go @@ -13,15 +13,25 @@ import ( func TestHTTPProxyAndServer(t *testing.T) { // Start libp2p HTTP server - h, hh, err := Libp2pHTTPServer() + h, hh, err := libp2pHTTPServer() if err != nil { t.Fatal(err) } + hh.SetHTTPHandlerAtPath("/hello", "/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) go hh.Serve() defer hh.Close() serverAddr := h.Addrs()[0].Encapsulate(multiaddr.StringCast("/p2p/" + h.ID().String())) + port, err := serverAddr.ValueForProtocol(multiaddr.P_TCP) + if err != nil || port == "" { + port, err = serverAddr.ValueForProtocol(multiaddr.P_UDP) + if err != nil || port == "" { + t.Fatal("could not get port from server address") + } + } ctx := context.Background() ctx, cancel := context.WithCancel(ctx) @@ -59,8 +69,7 @@ func TestHTTPProxyAndServer(t *testing.T) { }, } - // TODO update this when https://github.com/libp2p/go-libp2p/pull/2757 lands - resp, err := client.Get("http://example.com" + "/.well-known/libp2p") + resp, err := client.Get("http://127.0.0.1:" + port + "/") if err != nil { t.Fatal(err) } @@ -128,8 +137,7 @@ func TestHTTPProxyAndServerOverHTTPTransport(t *testing.T) { }, } - // TODO update this when https://github.com/libp2p/go-libp2p/pull/2757 lands - resp, err := client.Get("http://example.com/") + resp, err := client.Get("http://127.0.0.1:" + port + "/") if err != nil { t.Fatal(err) }