Add --stream option to ls command
#5611
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,13 +1,12 @@ | ||
| package commands | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "io" | ||
| "os" | ||
| "text/tabwriter" | ||
|
|
||
| cmdenv "github.com/ipfs/go-ipfs/core/commands/cmdenv" | ||
| iface "github.com/ipfs/go-ipfs/core/coreapi/interface" | ||
|
|
||
| cid "gx/ipfs/QmR8BauakNcBa3RbE4nbQu76PDiJgoQgz8AJdhJuiU4TAw/go-cid" | ||
|
|
@@ -16,29 +15,36 @@ import ( | |
| unixfspb "gx/ipfs/QmUnHNqhSB1JgzVCxL1Kz3yb4bdyB4q1Z9AD5AUBVmt3fZ/go-unixfs/pb" | ||
| blockservice "gx/ipfs/QmVDTbzzTwnuBwNbJdhW3u7LoBQp46bezm9yp4z1RoEepM/go-blockservice" | ||
| offline "gx/ipfs/QmYZwey1thDTynSrvd6qQkX24UpTka6TFhQ2v569UpoqxD/go-ipfs-exchange-offline" | ||
| cmds "gx/ipfs/Qma6uuSyjkecGhMFFLfzyJDPyoDtNJSHJNweDccZhaWkgU/go-ipfs-cmds" | ||
| merkledag "gx/ipfs/QmcGt25mrjuB2kKW2zhPbXVZNHc4yoTDQ65NA8m6auP2f1/go-merkledag" | ||
| ipld "gx/ipfs/QmcKKBwfz6FyQdHR2jsXrrF6XeSBXYL86anmWNewpFpoF5/go-ipld-format" | ||
| "gx/ipfs/Qmde5VP1qUkyQXKCfmEUA7bP64V2HAptbJ7phuPp7jXWwg/go-ipfs-cmdkit" | ||
| ) | ||
|
|
||
| // LsLink contains printable data for a single ipld link in ls output | ||
| type LsLink struct { | ||
| Name, Hash string | ||
| Size uint64 | ||
| Type unixfspb.Data_DataType | ||
| } | ||
|
|
||
| // LsObject is an element of LsOutput | ||
| // It can represent all or part of a directory | ||
| type LsObject struct { | ||
| Hash string | ||
| Links []LsLink | ||
| } | ||
|
|
||
| // LsOutput is a set of printable data for directories, | ||
| // it can be complete or partial | ||
| type LsOutput struct { | ||
| Objects []LsObject | ||
| } | ||
|
|
||
| const ( | ||
| lsHeadersOptionNameTime = "headers" | ||
| lsResolveTypeOptionName = "resolve-type" | ||
| lsStreamOptionName = "stream" | ||
| ) | ||
|
|
||
| var LsCmd = &cmds.Command{ | ||
|
|
@@ -60,158 +66,235 @@ The JSON output contains type information. | |
| Options: []cmdkit.Option{ | ||
| cmdkit.BoolOption(lsHeadersOptionNameTime, "v", "Print table headers (Hash, Size, Name)."), | ||
| cmdkit.BoolOption(lsResolveTypeOptionName, "Resolve linked objects to find out their types.").WithDefault(true), | ||
| cmdkit.BoolOption(lsStreamOptionName, "s", "Enable exprimental streaming of directory entries as they are traversed."), | ||
| }, | ||
| Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { | ||
| nd, err := cmdenv.GetNode(env) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| api, err := cmdenv.GetApi(env) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| resolve, _ := req.Options[lsResolveTypeOptionName].(bool) | ||
| dserv := nd.DAG | ||
| if !resolve { | ||
| offlineexch := offline.Exchange(nd.Blockstore) | ||
| bserv := blockservice.New(nd.Blockstore, offlineexch) | ||
| dserv = merkledag.NewDAGService(bserv) | ||
| } | ||
|
|
||
| err = req.ParseBodyArgs() | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| paths := req.Arguments | ||
|
|
||
| var dagnodes []ipld.Node | ||
| for _, fpath := range paths { | ||
| p, err := iface.ParsePath(fpath) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| dagnode, err := api.ResolveNode(req.Context, p) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| dagnodes = append(dagnodes, dagnode) | ||
| } | ||
| ng := merkledag.NewSession(req.Context, nd.DAG) | ||
| ro := merkledag.NewReadOnlyDagService(ng) | ||
|
|
||
| stream, _ := req.Options[lsStreamOptionName].(bool) | ||
|
|
||
| if !stream { | ||
| output := make([]LsObject, len(req.Arguments)) | ||
|
|
||
| for i, dagnode := range dagnodes { | ||
| dir, err := uio.NewDirectoryFromNode(ro, dagnode) | ||
| if err != nil && err != uio.ErrNotADir { | ||
| return fmt.Errorf("the data in %s (at %q) is not a UnixFS directory: %s", dagnode.Cid(), paths[i], err) | ||
| } | ||
|
|
||
| var links []*ipld.Link | ||
| if dir == nil { | ||
| links = dagnode.Links() | ||
|
Member
There was a problem hiding this comment. Gah... #4186. I'm going to just break this one day... |
||
| } else { | ||
| links, err = dir.Links(req.Context) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| } | ||
| outputLinks := make([]LsLink, len(links)) | ||
| for j, link := range links { | ||
| lsLink, err := makeLsLink(req, dserv, resolve, link) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| outputLinks[j] = *lsLink | ||
| } | ||
| output[i] = LsObject{ | ||
| Hash: paths[i], | ||
| Links: outputLinks, | ||
| } | ||
| } | ||
|
|
||
| return cmds.EmitOnce(res, &LsOutput{output}) | ||
| } | ||
|
|
||
| for i, dagnode := range dagnodes { | ||
| dir, err := uio.NewDirectoryFromNode(ro, dagnode) | ||
| if err != nil && err != uio.ErrNotADir { | ||
| return fmt.Errorf("the data in %s (at %q) is not a UnixFS directory: %s", dagnode.Cid(), paths[i], err) | ||
| } | ||
|
|
||
| var linkResults <-chan unixfs.LinkResult | ||
| if dir == nil { | ||
| linkResults = makeDagNodeLinkResults(req, dagnode) | ||
| } else { | ||
| linkResults = dir.EnumLinksAsync(req.Context) | ||
| } | ||
|
|
||
| for linkResult := range linkResults { | ||
|
|
||
| if linkResult.Err != nil { | ||
| return linkResult.Err | ||
| } | ||
| link := linkResult.Link | ||
| lsLink, err := makeLsLink(req, dserv, resolve, link) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| output := []LsObject{{ | ||
| Hash: paths[i], | ||
| Links: []LsLink{*lsLink}, | ||
| }} | ||
| if err = res.Emit(&LsOutput{output}); err != nil { | ||
| return err | ||
| } | ||
| } | ||
| } | ||
| return nil | ||
| }, | ||
| PostRun: cmds.PostRunMap{ | ||
kevina marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| cmds.CLI: func(res cmds.Response, re cmds.ResponseEmitter) error { | ||
| req := res.Request() | ||
| lastObjectHash := "" | ||
|
|
||
| for { | ||
| v, err := res.Next() | ||
| if err != nil { | ||
| if err == io.EOF { | ||
| return nil | ||
| } | ||
| return err | ||
| } | ||
| out := v.(*LsOutput) | ||
| lastObjectHash = tabularOutput(req, os.Stdout, out, lastObjectHash, false) | ||
| } | ||
| }, | ||
| }, | ||
| Encoders: cmds.EncoderMap{ | ||
| cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *LsOutput) error { | ||
| // when streaming over HTTP using a text encoder, we cannot render breaks | ||
| // between directories because we don't know the hash of the last | ||
| // directory encoder | ||
| ignoreBreaks, _ := req.Options[lsStreamOptionName].(bool) | ||
| tabularOutput(req, w, out, "", ignoreBreaks) | ||
| return nil | ||
| }), | ||
| }, | ||
| Type: LsOutput{}, | ||
| } | ||
|
|
||
| func makeDagNodeLinkResults(req *cmds.Request, dagnode ipld.Node) <-chan unixfs.LinkResult { | ||
| links := dagnode.Links() | ||
| linkResults := make(chan unixfs.LinkResult, len(links)) | ||
| defer close(linkResults) | ||
| for _, l := range links { | ||
| linkResults <- unixfs.LinkResult{ | ||
| Link: l, | ||
| Err: nil, | ||
| } | ||
| } | ||
magik6k marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| return linkResults | ||
| } | ||
|
|
||
| func makeLsLink(req *cmds.Request, dserv ipld.DAGService, resolve bool, link *ipld.Link) (*LsLink, error) { | ||
| t := unixfspb.Data_DataType(-1) | ||
|
|
||
| switch link.Cid.Type() { | ||
| case cid.Raw: | ||
| // No need to check with raw leaves | ||
| t = unixfs.TFile | ||
| case cid.DagProtobuf: | ||
| linkNode, err := link.GetNode(req.Context, dserv) | ||
| if err == ipld.ErrNotFound && !resolve { | ||
| // not an error | ||
| linkNode = nil | ||
| } else if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| if pn, ok := linkNode.(*merkledag.ProtoNode); ok { | ||
| d, err := unixfs.FSNodeFromBytes(pn.Data()) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| t = d.Type() | ||
| } | ||
| } | ||
| return &LsLink{ | ||
| Name: link.Name, | ||
| Hash: link.Cid.String(), | ||
| Size: link.Size, | ||
| Type: t, | ||
| }, nil | ||
| } | ||
|
|
||
| func tabularOutput(req *cmds.Request, w io.Writer, out *LsOutput, lastObjectHash string, ignoreBreaks bool) string { | ||
kevina marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| headers, _ := req.Options[lsHeadersOptionNameTime].(bool) | ||
| stream, _ := req.Options[lsStreamOptionName].(bool) | ||
| // in streaming mode we can't automatically align the tabs | ||
| // so we take a best guess | ||
| var minTabWidth int | ||
| if stream { | ||
| minTabWidth = 10 | ||
| } else { | ||
| minTabWidth = 1 | ||
| } | ||
|
|
||
| multipleFolders := len(req.Arguments) > 1 | ||
|
|
||
| tw := tabwriter.NewWriter(w, minTabWidth, 2, 1, ' ', 0) | ||
|
|
||
| for _, object := range out.Objects { | ||
|
|
||
| if !ignoreBreaks && object.Hash != lastObjectHash { | ||
| if multipleFolders { | ||
| if lastObjectHash != "" { | ||
| fmt.Fprintln(tw) | ||
| } | ||
| fmt.Fprintf(tw, "%s:\n", object.Hash) | ||
| } | ||
| if headers { | ||
| fmt.Fprintln(tw, "Hash\tSize\tName") | ||
| } | ||
| lastObjectHash = object.Hash | ||
| } | ||
|
|
||
| for _, link := range object.Links { | ||
| if link.Type == unixfs.TDirectory { | ||
| link.Name += "/" | ||
| } | ||
|
|
||
| fmt.Fprintf(tw, "%s\t%v\t%s\n", link.Hash, link.Size, link.Name) | ||
| } | ||
| } | ||
| tw.Flush() | ||
| return lastObjectHash | ||
| } | ||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.