From e3e561945bc606bef536f4aae8f9153d50066a4a Mon Sep 17 00:00:00 2001 From: Ryan Fowler Date: Mon, 26 Jan 2026 13:56:09 -0800 Subject: [PATCH] Add preliminary support for protobuf formatting --- README.md | 2 +- docs/USAGE.md | 1 + go.mod | 1 + go.sum | 2 + internal/fetch/fetch.go | 10 + internal/format/protobuf.go | 207 +++++++++++++++++ internal/format/protobuf_test.go | 377 +++++++++++++++++++++++++++++++ 7 files changed, 599 insertions(+), 1 deletion(-) create mode 100644 internal/format/protobuf.go create mode 100644 internal/format/protobuf_test.go diff --git a/README.md b/README.md index 760c11b..b48a855 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ ### Features include: -- **Response formatting**: automatically formats and colors output (json, html, msgpack, xml, etc.) +- **Response formatting**: automatically formats and colors output (json, html, msgpack, protobuf, xml, etc.) - **Image rendering**: render images directly in your terminal - **Compression**: automatic gzip and zstd response body decompression - **Authentication**: support for Basic Auth, Bearer Token, and AWS Signature V4 diff --git a/docs/USAGE.md b/docs/USAGE.md index 0d009dc..9dd195a 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -338,6 +338,7 @@ Supported formats for automatic formatting and syntax highlighting: - XML (`application/xml`, `text/xml`) - MessagePack (`application/msgpack`) - NDJSON/JSONLines (`application/x-ndjson`) +- Protobuf (`application/x-protobuf`, `application/protobuf`) - Server-Sent Events (`text/event-stream`) ```sh diff --git a/go.mod b/go.mod index b3a1ecd..10ec87f 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( golang.org/x/image v0.35.0 golang.org/x/net v0.49.0 golang.org/x/sys v0.40.0 + google.golang.org/protobuf v1.36.11 ) require ( diff --git a/go.sum b/go.sum index 7333aa9..8589f61 100644 --- a/go.sum +++ b/go.sum @@ -26,5 +26,7 @@ golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE= golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/fetch/fetch.go b/internal/fetch/fetch.go index e6780cb..29d3161 100644 --- a/internal/fetch/fetch.go +++ b/internal/fetch/fetch.go @@ -33,6 +33,7 @@ const ( TypeJSON TypeMsgPack TypeNDJSON + TypeProtobuf TypeSSE TypeXML ) @@ -298,6 +299,10 @@ func formatResponse(ctx context.Context, r *Request, resp *http.Response) (io.Re if format.FormatMsgPack(buf, p) == nil { buf = p.Bytes() } + case TypeProtobuf: + if format.FormatProtobuf(buf, p) == nil { + buf = p.Bytes() + } case TypeXML: if format.FormatXML(buf, p) == nil { buf = p.Bytes() @@ -329,12 +334,17 @@ func getContentType(headers http.Header) ContentType { return TypeMsgPack case "x-ndjson", "ndjson", "x-jsonl", "jsonl", "x-jsonlines": return TypeNDJSON + case "protobuf", "x-protobuf", "grpc+proto", "x-google-protobuf", "vnd.google.protobuf": + return TypeProtobuf case "xml": return TypeXML } if strings.HasSuffix(subtype, "+json") || strings.HasSuffix(subtype, "-json") { return TypeJSON } + if strings.HasSuffix(subtype, "+proto") { + return TypeProtobuf + } if strings.HasSuffix(subtype, "+xml") { return TypeXML } diff --git a/internal/format/protobuf.go b/internal/format/protobuf.go new file mode 100644 index 0000000..af3a3a1 --- /dev/null +++ b/internal/format/protobuf.go @@ -0,0 +1,207 @@ +package format + +import ( + "fmt" + "strconv" + "unicode" + "unicode/utf8" + + "github.com/ryanfowler/fetch/internal/core" + + "google.golang.org/protobuf/encoding/protowire" +) + +// FormatProtobuf formats the provided raw protobuf data to the Printer. +func FormatProtobuf(buf []byte, p *core.Printer) error { + err := formatProtobuf(buf, p, 0) + if err != nil { + p.Reset() + } + return err +} + +func formatProtobuf(buf []byte, p *core.Printer, indent int) error { + for len(buf) > 0 { + num, wtype, n := protowire.ConsumeTag(buf) + if n < 0 { + return protowire.ParseError(n) + } + buf = buf[n:] + + writeIndent(p, indent) + writeFieldNumber(p, num) + + switch wtype { + case protowire.VarintType: + v, n := protowire.ConsumeVarint(buf) + if n < 0 { + return protowire.ParseError(n) + } + buf = buf[n:] + writeWireType(p, "varint") + p.WriteString(" ") + p.WriteString(strconv.FormatUint(v, 10)) + p.WriteString("\n") + + case protowire.Fixed64Type: + v, n := protowire.ConsumeFixed64(buf) + if n < 0 { + return protowire.ParseError(n) + } + buf = buf[n:] + writeWireType(p, "fixed64") + p.WriteString(" ") + p.WriteString(fmt.Sprintf("0x%016x", v)) + p.WriteString("\n") + + case protowire.Fixed32Type: + v, n := protowire.ConsumeFixed32(buf) + if n < 0 { + return protowire.ParseError(n) + } + buf = buf[n:] + writeWireType(p, "fixed32") + p.WriteString(" ") + p.WriteString(fmt.Sprintf("0x%08x", v)) + p.WriteString("\n") + + case protowire.BytesType: + v, n := protowire.ConsumeBytes(buf) + if n < 0 { + return protowire.ParseError(n) + } + buf = buf[n:] + + // Try to parse as nested message. + if isValidProtobuf(v) { + writeWireType(p, "message") + p.WriteString(" {\n") + err := formatProtobuf(v, p, indent+1) + if err != nil { + return err + } + writeIndent(p, indent) + p.WriteString("}\n") + } else if isPrintableBytes(v) { + writeWireType(p, "bytes") + p.WriteString(" ") + writeProtobufString(p, string(v)) + p.WriteString("\n") + } else { + writeWireType(p, "bytes") + p.WriteString(" ") + writeProtobufBytes(p, v) + p.WriteString("\n") + } + + case protowire.StartGroupType, protowire.EndGroupType: + // Groups are deprecated; skip them. + return fmt.Errorf("deprecated group wire type") + + default: + return fmt.Errorf("unknown wire type: %d", wtype) + } + } + return nil +} + +// isValidProtobuf checks if the bytes can be parsed as a valid protobuf message. +func isValidProtobuf(buf []byte) bool { + if len(buf) == 0 { + return false + } + + for len(buf) > 0 { + num, wtype, n := protowire.ConsumeTag(buf) + if n < 0 || num == 0 { + return false + } + buf = buf[n:] + + switch wtype { + case protowire.VarintType: + _, n = protowire.ConsumeVarint(buf) + case protowire.Fixed64Type: + _, n = protowire.ConsumeFixed64(buf) + case protowire.Fixed32Type: + _, n = protowire.ConsumeFixed32(buf) + case protowire.BytesType: + _, n = protowire.ConsumeBytes(buf) + case protowire.StartGroupType, protowire.EndGroupType: + return false + default: + return false + } + if n < 0 { + return false + } + buf = buf[n:] + } + return true +} + +// isPrintableBytes returns true if the bytes are printable UTF-8 text. +func isPrintableBytes(b []byte) bool { + if !utf8.Valid(b) { + return false + } + for _, r := range string(b) { + if !unicode.IsPrint(r) && !unicode.IsSpace(r) { + return false + } + } + return true +} + +func writeFieldNumber(p *core.Printer, num protowire.Number) { + p.Set(core.Blue) + p.Set(core.Bold) + p.WriteString(strconv.FormatInt(int64(num), 10)) + p.Reset() + p.WriteString(":") +} + +func writeWireType(p *core.Printer, wtype string) { + p.WriteString(" ") + p.Set(core.Dim) + p.WriteString("(") + p.WriteString(wtype) + p.WriteString(")") + p.Reset() +} + +func writeProtobufString(p *core.Printer, s string) { + p.WriteString("\"") + p.Set(core.Green) + for _, c := range s { + switch c { + case '\n': + p.WriteString(`\n`) + case '\r': + p.WriteString(`\r`) + case '\t': + p.WriteString(`\t`) + case '"': + p.WriteString(`\"`) + case '\\': + p.WriteString(`\\`) + default: + p.WriteRune(c) + } + } + p.Reset() + p.WriteString("\"") +} + +func writeProtobufBytes(p *core.Printer, b []byte) { + p.Set(core.Yellow) + p.WriteString("<") + for i, byt := range b { + if i > 0 { + p.WriteString(" ") + } + p.WriteString(fmt.Sprintf("%02x", byt)) + } + p.WriteString(">") + p.Reset() +} diff --git a/internal/format/protobuf_test.go b/internal/format/protobuf_test.go new file mode 100644 index 0000000..8f5f49d --- /dev/null +++ b/internal/format/protobuf_test.go @@ -0,0 +1,377 @@ +package format + +import ( + "strings" + "testing" + + "github.com/ryanfowler/fetch/internal/core" + + "google.golang.org/protobuf/encoding/protowire" +) + +func TestFormatProtobuf(t *testing.T) { + tests := []struct { + name string + input []byte + wantErr bool + contains []string + }{ + { + name: "varint field", + input: appendVarint(nil, 1, 123), + wantErr: false, + contains: []string{"1:", "(varint)", "123"}, + }, + { + name: "fixed64 field", + input: appendFixed64(nil, 2, 0x123456789abcdef0), + wantErr: false, + contains: []string{"2:", "(fixed64)", "0x123456789abcdef0"}, + }, + { + name: "fixed32 field", + input: appendFixed32(nil, 3, 0x12345678), + wantErr: false, + contains: []string{"3:", "(fixed32)", "0x12345678"}, + }, + { + name: "string field", + input: appendBytes(nil, 4, []byte("hello world")), + wantErr: false, + contains: []string{"4:", "(bytes)", `"hello world"`}, + }, + { + name: "binary bytes field", + input: appendBytes(nil, 5, []byte{0x00, 0xff, 0x80}), + wantErr: false, + contains: []string{"5:", "(bytes)", "<00 ff 80>"}, + }, + { + name: "multiple fields", + input: func() []byte { + b := appendVarint(nil, 1, 42) + b = appendBytes(b, 2, []byte("test")) + return b + }(), + wantErr: false, + contains: []string{"1:", "42", "2:", `"test"`}, + }, + { + name: "nested message", + input: func() []byte { + inner := appendVarint(nil, 1, 456) + return appendBytes(nil, 3, inner) + }(), + wantErr: false, + contains: []string{"3:", "(message)", "{", "1:", "456", "}"}, + }, + { + name: "empty input", + input: []byte{}, + wantErr: false, + }, + { + name: "invalid tag", + input: []byte{0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x01}, + wantErr: true, + }, + { + name: "truncated varint", + input: []byte{0x08, 0x80}, // field 1, varint, but varint is incomplete + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := core.NewHandle(core.ColorOff).Stderr() + err := FormatProtobuf(tt.input, p) + if (err != nil) != tt.wantErr { + t.Errorf("FormatProtobuf() error = %v, wantErr %v", err, tt.wantErr) + return + } + if tt.wantErr { + return + } + + output := string(p.Bytes()) + for _, want := range tt.contains { + if !strings.Contains(output, want) { + t.Errorf("output should contain %q, got: %s", want, output) + } + } + }) + } +} + +func TestFormatProtobufNested(t *testing.T) { + // Create a deeply nested message: field 1 contains field 2 contains field 3 with varint + innermost := appendVarint(nil, 3, 789) + middle := appendBytes(nil, 2, innermost) + outer := appendBytes(nil, 1, middle) + + p := core.NewHandle(core.ColorOff).Stderr() + err := FormatProtobuf(outer, p) + if err != nil { + t.Fatalf("FormatProtobuf() error = %v", err) + } + + output := string(p.Bytes()) + + // Check for proper nesting structure + if !strings.Contains(output, "1:") { + t.Error("output should contain field 1") + } + if !strings.Contains(output, "2:") { + t.Error("output should contain field 2") + } + if !strings.Contains(output, "3:") { + t.Error("output should contain field 3") + } + if !strings.Contains(output, "789") { + t.Error("output should contain value 789") + } + if strings.Count(output, "{") != 2 || strings.Count(output, "}") != 2 { + t.Errorf("output should have 2 nested messages, got: %s", output) + } +} + +func TestFormatProtobufAllWireTypes(t *testing.T) { + // Build a message with all supported wire types + b := appendVarint(nil, 1, 100) + b = appendFixed64(b, 2, 200) + b = appendFixed32(b, 3, 300) + b = appendBytes(b, 4, []byte("string")) + + p := core.NewHandle(core.ColorOff).Stderr() + err := FormatProtobuf(b, p) + if err != nil { + t.Fatalf("FormatProtobuf() error = %v", err) + } + + output := string(p.Bytes()) + if !strings.Contains(output, "(varint)") { + t.Error("output should contain varint wire type") + } + if !strings.Contains(output, "(fixed64)") { + t.Error("output should contain fixed64 wire type") + } + if !strings.Contains(output, "(fixed32)") { + t.Error("output should contain fixed32 wire type") + } + if !strings.Contains(output, "(bytes)") { + t.Error("output should contain bytes wire type") + } +} + +func TestIsValidProtobuf(t *testing.T) { + tests := []struct { + name string + input []byte + want bool + }{ + { + name: "empty", + input: []byte{}, + want: false, + }, + { + name: "valid varint", + input: appendVarint(nil, 1, 123), + want: true, + }, + { + name: "valid multiple fields", + input: appendBytes(appendVarint(nil, 1, 1), 2, []byte("test")), + want: true, + }, + { + name: "invalid tag", + input: []byte{0x00}, // field number 0 is invalid + want: false, + }, + { + name: "truncated", + input: []byte{0x08}, // field 1, varint, but no value + want: false, + }, + { + name: "random bytes", + input: []byte{0xff, 0xff, 0xff}, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := isValidProtobuf(tt.input) + if got != tt.want { + t.Errorf("isValidProtobuf() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestIsPrintableBytes(t *testing.T) { + tests := []struct { + name string + input []byte + want bool + }{ + { + name: "ascii text", + input: []byte("hello world"), + want: true, + }, + { + name: "unicode text", + input: []byte("hello 世界"), + want: true, + }, + { + name: "with newline", + input: []byte("hello\nworld"), + want: true, + }, + { + name: "with tab", + input: []byte("hello\tworld"), + want: true, + }, + { + name: "binary data", + input: []byte{0x00, 0x01, 0x02}, + want: false, + }, + { + name: "invalid utf8", + input: []byte{0xff, 0xfe}, + want: false, + }, + { + name: "empty", + input: []byte{}, + want: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := isPrintableBytes(tt.input) + if got != tt.want { + t.Errorf("isPrintableBytes() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestWriteProtobufString(t *testing.T) { + tests := []struct { + name string + input string + want string + }{ + { + name: "simple string", + input: "hello", + want: `"hello"`, + }, + { + name: "with newline", + input: "hello\nworld", + want: `"hello\nworld"`, + }, + { + name: "with tab", + input: "hello\tworld", + want: `"hello\tworld"`, + }, + { + name: "with quotes", + input: `say "hello"`, + want: `"say \"hello\""`, + }, + { + name: "with backslash", + input: `path\to\file`, + want: `"path\\to\\file"`, + }, + { + name: "with carriage return", + input: "hello\rworld", + want: `"hello\rworld"`, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := core.NewHandle(core.ColorOff).Stderr() + writeProtobufString(p, tt.input) + got := string(p.Bytes()) + if got != tt.want { + t.Errorf("writeProtobufString() = %q, want %q", got, tt.want) + } + }) + } +} + +func TestWriteProtobufBytes(t *testing.T) { + tests := []struct { + name string + input []byte + want string + }{ + { + name: "single byte", + input: []byte{0xab}, + want: "", + }, + { + name: "multiple bytes", + input: []byte{0x00, 0xff, 0x80}, + want: "<00 ff 80>", + }, + { + name: "empty", + input: []byte{}, + want: "<>", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := core.NewHandle(core.ColorOff).Stderr() + writeProtobufBytes(p, tt.input) + got := string(p.Bytes()) + if got != tt.want { + t.Errorf("writeProtobufBytes() = %q, want %q", got, tt.want) + } + }) + } +} + +// Helper functions to build protobuf test data + +func appendVarint(b []byte, num protowire.Number, v uint64) []byte { + b = protowire.AppendTag(b, num, protowire.VarintType) + b = protowire.AppendVarint(b, v) + return b +} + +func appendFixed64(b []byte, num protowire.Number, v uint64) []byte { + b = protowire.AppendTag(b, num, protowire.Fixed64Type) + b = protowire.AppendFixed64(b, v) + return b +} + +func appendFixed32(b []byte, num protowire.Number, v uint32) []byte { + b = protowire.AppendTag(b, num, protowire.Fixed32Type) + b = protowire.AppendFixed32(b, v) + return b +} + +func appendBytes(b []byte, num protowire.Number, v []byte) []byte { + b = protowire.AppendTag(b, num, protowire.BytesType) + b = protowire.AppendBytes(b, v) + return b +}