diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..bd3314c --- /dev/null +++ b/.travis.yml @@ -0,0 +1,24 @@ +sudo: false + + +language: go +go: + - 'tip' + +install: + - go get github.com/whyrusleeping/gx + - go get github.com/whyrusleeping/gx-go + - gx install --global +script: + - gx test -v -race -coverprofile=coverage.txt -covermode=atomic . + +after_success: + - bash <(curl -s https://codecov.io/bash) + +cache: + directories: + - $GOPATH/src/gx + +notifications: +email: false + diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile new file mode 100644 index 0000000..b2067e6 --- /dev/null +++ b/ci/Jenkinsfile @@ -0,0 +1 @@ +golang() diff --git a/cid-fmt/main.go b/cid-fmt/main.go index 3aae3ee..8735658 100644 --- a/cid-fmt/main.go +++ b/cid-fmt/main.go @@ -1,7 +1,9 @@ package main import ( + "bufio" "fmt" + "io" "os" "strings" @@ -12,7 +14,9 @@ import ( ) func usage() { - fmt.Fprintf(os.Stderr, "usage: %s [-b multibase-code] [-v cid-version] ...\n\n", os.Args[0]) + fmt.Fprintf(os.Stderr, "usage: %s [-b multibase-code] [-v cid-version] [--filter] ...\n", os.Args[0]) + fmt.Fprintf(os.Stderr, "--filter will read from stdin and convert anything that looks like a \n") + fmt.Fprintf(os.Stderr, " -- including any non-cids that are valid Multihashes).\n") fmt.Fprintf(os.Stderr, " is either 'prefix' or a printf style format string:\n%s", cidutil.FormatRef) os.Exit(2) } @@ -24,8 +28,9 @@ func main() { newBase := mb.Encoding(-1) var verConv func(cid c.Cid) (c.Cid, error) args := os.Args[1:] + filter := false outer: - for { + for len(args) > 0 { switch args[0] { case "-b": if len(args) < 2 { @@ -52,11 +57,14 @@ outer: os.Exit(2) } args = args[2:] + case "--filter": + filter = true + args = args[1:] default: break outer } } - if len(args) < 2 { + if len(args) < 1 { usage() } fmtStr := args[0] @@ -69,41 +77,73 @@ outer: os.Exit(2) } } - for _, cidStr := range args[1:] { - cid, err := c.Decode(cidStr) - if err != nil { - fmt.Fprintf(os.Stdout, "!INVALID_CID!\n") - errorMsg("%s: %v", cidStr, err) - // Don't abort on a bad cid - continue - } + format := func(cid c.Cid, cidStr string) (string, error) { base := newBase - if newBase == -1 { + if base == -1 { base, _ = c.ExtractEncoding(cidStr) } + var err error if verConv != nil { cid, err = verConv(cid) if err != nil { - fmt.Fprintf(os.Stdout, "!ERROR!\n") - errorMsg("%s: %v", cidStr, err) - // Don't abort on a bad conversion - continue + return "", err + } + } + return cidutil.Format(fmtStr, base, cid) + } + if filter { + scanner := bufio.NewScanner(os.Stdin) + for scanner.Scan() { + buf := scanner.Bytes() + for { + i, j, cid, cidStr := cidutil.ScanForCid(buf) + os.Stdout.Write(buf[0:i]) + if i == len(buf) { + os.Stdout.Write([]byte{'\n'}) + break + } + str, err := format(cid, cidStr) + switch err.(type) { + case cidutil.FormatStringError: + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + os.Exit(2) + default: + // just use the orignal sting on non-fatal error + str = cidStr + case nil: + } + io.WriteString(os.Stdout, str) + buf = buf[j:] } } - str, err := cidutil.Format(fmtStr, base, cid) - switch err.(type) { - case cidutil.FormatStringError: + if err := scanner.Err(); err != nil { fmt.Fprintf(os.Stderr, "Error: %v\n", err) os.Exit(2) - default: - fmt.Fprintf(os.Stdout, "!ERROR!\n") - errorMsg("%s: %v", cidStr, err) - // Don't abort on cid specific errors - continue - case nil: - // no error } - fmt.Fprintf(os.Stdout, "%s\n", str) + } else { + for _, cidStr := range args[1:] { + cid, err := c.Decode(cidStr) + if err != nil { + fmt.Fprintf(os.Stdout, "!INVALID_CID!\n") + errorMsg("%s: %v", cidStr, err) + // Don't abort on a bad cid + continue + } + str, err := format(cid, cidStr) + switch err.(type) { + case cidutil.FormatStringError: + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + os.Exit(2) + default: + fmt.Fprintf(os.Stdout, "!ERROR!\n") + errorMsg("%s: %v", cidStr, err) + // Don't abort on cid specific errors + continue + case nil: + // no error + } + fmt.Fprintf(os.Stdout, "%s\n", str) + } } os.Exit(exitCode) } diff --git a/cidenc/encoder.go b/cidenc/encoder.go new file mode 100644 index 0000000..6b8cefd --- /dev/null +++ b/cidenc/encoder.go @@ -0,0 +1,59 @@ +package cidenc + +import ( + cid "github.com/ipfs/go-cid" + mbase "github.com/multiformats/go-multibase" +) + +// Encoder is a basic Encoder that will encode CIDs using a specified +// base and optionally upgrade a CIDv0 to CIDv1 +type Encoder struct { + Base mbase.Encoder // The multibase to use + Upgrade bool // If true upgrade CIDv0 to CIDv1 when encoding +} + +// Default return a new default encoder +func Default() Encoder { + return Encoder{Base: mbase.MustNewEncoder(mbase.Base58BTC)} +} + +// Encode encodes the cid using the parameters of the Encoder +func (enc Encoder) Encode(c cid.Cid) string { + if enc.Upgrade && c.Version() == 0 { + c = cid.NewCidV1(c.Type(), c.Hash()) + } + return c.Encode(enc.Base) +} + +// Recode reencodes the cid string to match the parameters of the +// encoder +func (enc Encoder) Recode(v string) (string, error) { + skip, err := enc.noopRecode(v) + if skip || err != nil { + return v, err + } + + c, err := cid.Decode(v) + if err != nil { + return v, err + } + + return enc.Encode(c), nil +} + +func (enc Encoder) noopRecode(v string) (bool, error) { + if len(v) < 2 { + return false, cid.ErrCidTooShort + } + ver := cidVer(v) + skip := ver == 0 && !enc.Upgrade || ver == 1 && v[0] == byte(enc.Base.Encoding()) + return skip, nil +} + +func cidVer(v string) int { + if len(v) == 46 && v[:2] == "Qm" { + return 0 + } else { + return 1 + } +} diff --git a/cidenc/encoder_test.go b/cidenc/encoder_test.go new file mode 100644 index 0000000..a42ef34 --- /dev/null +++ b/cidenc/encoder_test.go @@ -0,0 +1,62 @@ +package cidenc + +import ( + "testing" + + cid "github.com/ipfs/go-cid" + mbase "github.com/multiformats/go-multibase" +) + +func TestCidEncoder(t *testing.T) { + cidv0str := "QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n" + cidv1str := "zdj7Wkkhxcu2rsiN6GUyHCLsSLL47kdUNfjbFqBUUhMFTZKBi" + cidb32str := "bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku" + cidv0, _ := cid.Decode(cidv0str) + cidv1, _ := cid.Decode(cidv1str) + + testEncode := func(enc Encoder, cid cid.Cid, expect string) { + actual := enc.Encode(cid) + if actual != expect { + t.Errorf("%+v.Encode(%s): expected %s but got %s", enc, cid, expect, actual) + } + } + + testRecode := func(enc Encoder, cid string, expect string) { + actual, err := enc.Recode(cid) + if err != nil { + t.Errorf("%+v.Recode(%s): %s", enc, cid, err) + return + } + if actual != expect { + t.Errorf("%+v.Recode(%s): expected %s but got %s", enc, cid, expect, actual) + } + } + + enc := Encoder{Base: mbase.MustNewEncoder(mbase.Base58BTC), Upgrade: false} + testEncode(enc, cidv0, cidv0str) + testEncode(enc, cidv1, cidv1str) + testRecode(enc, cidv0str, cidv0str) + testRecode(enc, cidv1str, cidv1str) + testRecode(enc, cidb32str, cidv1str) + + enc = Encoder{Base: mbase.MustNewEncoder(mbase.Base58BTC), Upgrade: true} + testEncode(enc, cidv0, cidv1str) + testEncode(enc, cidv1, cidv1str) + testRecode(enc, cidv0str, cidv1str) + testRecode(enc, cidv1str, cidv1str) + testRecode(enc, cidb32str, cidv1str) + + enc = Encoder{Base: mbase.MustNewEncoder(mbase.Base32), Upgrade: false} + testEncode(enc, cidv0, cidv0str) + testEncode(enc, cidv1, cidb32str) + testRecode(enc, cidv0str, cidv0str) + testRecode(enc, cidv1str, cidb32str) + testRecode(enc, cidb32str, cidb32str) + + enc = Encoder{Base: mbase.MustNewEncoder(mbase.Base32), Upgrade: true} + testEncode(enc, cidv0, cidb32str) + testEncode(enc, cidv1, cidb32str) + testRecode(enc, cidv0str, cidb32str) + testRecode(enc, cidv1str, cidb32str) + testRecode(enc, cidb32str, cidb32str) +} diff --git a/format.go b/format.go index 873d3ec..21e157d 100644 --- a/format.go +++ b/format.go @@ -150,3 +150,48 @@ func encode(base mb.Encoder, data []byte, strip bool) string { } return str } + +// ScanForCid scans bytes for anything resembling a CID. If one is +// found `i` will point to the begging of the cid and `j` to to the +// end and the cid will be returned, otherwise `i` and `j` will point +// the end of the buffer and the cid will be `Undef`. +func ScanForCid(buf []byte) (i, j int, cid c.Cid, cidStr string) { + i = 0 + for { + i = j + for i < len(buf) && !asciiIsAlpha(buf[i]) { + i++ + } + j = i + if i == len(buf) { + return + } + for j < len(buf) && asciiIsAlpha(buf[j]) { + j++ + } + if j-i <= 1 || j-i > 128 || !supported[buf[i]] { + continue + } + var err error + cidStr = string(buf[i:j]) + cid, err = c.Decode(cidStr) + if err == nil { + return + } + } +} + +var supported = make([]bool, 256) + +func init() { + // for now base64 encoding are not supported as they contain non + // alhphanumeric characters + supportedPrefixes := []byte("QfFbBcCvVtThzZ") + for _, b := range supportedPrefixes { + supported[b] = true + } +} + +func asciiIsAlpha(b byte) bool { + return ('A' <= b && b <= 'Z') || ('a' <= b && b <= 'z') || ('0' <= b && b <= '9') +} diff --git a/format_test.go b/format_test.go index 9c77844..e102e7a 100644 --- a/format_test.go +++ b/format_test.go @@ -72,3 +72,35 @@ func testFmt(t *testing.T, cidStr string, newBase mb.Encoding, fmtStr string, re t.Error(fmt.Sprintf("expected: %s; but got: %s", result, str)) } } + +func TestScanForCid(t *testing.T) { + testStr := []byte(` +/ipfs/QmYFbmndVP7QqAVWyKhpmMuQHMaD88pkK57RgYVimmoh5H 22 45 +/ipfs/zdj7WfLr9DhLrb1hsoSi4fSdjjxuZmeqgEtBPWxMLtPbDNbFD/foobar +BAFYBEIETJGSRL3EQPQPCABV3G6IUBYTSIFVQ24XRRHD3JUETSKLTGQ7DJA +skip me (too long): QmYFbmndVP7QqAVWyKhpmMuQHMaD88pkK57RgYVimmoh5H876 +skip me (too short): bafybeietjgsrl3eqpqpcabv3g6iubytsifvq +bafybeietjgsrl3eqpqpcabv3g6iubytsifvq24xrrhd3juetskltgq7dja +bafybeietjgsrl3eqpqpcabv3g6iubytsifvq24xrrhd3juetskltgq7dja. +`) + cids := []string{ + "QmYFbmndVP7QqAVWyKhpmMuQHMaD88pkK57RgYVimmoh5H", + "zdj7WfLr9DhLrb1hsoSi4fSdjjxuZmeqgEtBPWxMLtPbDNbFD", + "BAFYBEIETJGSRL3EQPQPCABV3G6IUBYTSIFVQ24XRRHD3JUETSKLTGQ7DJA", + "bafybeietjgsrl3eqpqpcabv3g6iubytsifvq24xrrhd3juetskltgq7dja", + "bafybeietjgsrl3eqpqpcabv3g6iubytsifvq24xrrhd3juetskltgq7dja", + } + + buf := testStr + idx := 0 + offset := 0 + for len(buf) > 0 { + _, j, _, cidStr := ScanForCid(buf) + if cidStr != "" && cids[idx] != cidStr { + t.Fatalf("Scan failed, expected %s, got %s (idx=%d offset=%d)", cids[idx], cidStr, idx, offset) + } + buf = buf[j:] + offset += j + idx++ + } +}