Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 9 additions & 11 deletions cmd/car/car.go
Original file line number Diff line number Diff line change
Expand Up @@ -100,23 +100,21 @@ func main1() int {
}},
},
{
Name: "extract",
Aliases: []string{"x"},
Usage: "Extract the contents of a car when the car encodes UnixFS data",
Action: ExtractCar,
ArgsUsage: "[output directory|-]",
Name: "extract",
Aliases: []string{"x"},
Usage: "Extract the contents of a car when the car encodes UnixFS data",
Action: ExtractCar,
Flags: []cli.Flag{
&cli.StringFlag{
Name: "file",
Aliases: []string{"f"},
Usage: "The car file to extract from, or stdin if omitted",
Required: false,
Aliases: []string{"f", "input", "i"},
Usage: "The car file to extract from",
TakesFile: true,
},
&cli.StringFlag{
Name: "path",
Aliases: []string{"p"},
Usage: "The unixfs path to extract",
Name: "output",
Aliases: []string{"o"},
Usage: "The path to write into",
Required: false,
},
&cli.BoolFlag{
Expand Down
70 changes: 29 additions & 41 deletions cmd/car/extract.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ import (
"fmt"
"io"
"os"
"runtime"
"strings"
"sync"

Expand All @@ -22,68 +21,57 @@ var ErrNotDir = fmt.Errorf("not a directory")

// ExtractCar pulls files and directories out of a car
func ExtractCar(c *cli.Context) error {
if !c.IsSet("file") {
return fmt.Errorf("a file source must be specified")
}

outputDir, err := os.Getwd()
if err != nil {
return err
}
if c.Args().Present() {
outputDir = c.Args().First()
if c.IsSet("output") {
outputDir = c.String("output")
}

var store storage.ReadableStorage
var roots []cid.Cid

if c.String("file") == "" {
if f, ok := c.App.Reader.(*os.File); ok {
stat, err := f.Stat()
if err != nil {
return err
}
if (stat.Mode() & os.ModeCharDevice) != 0 {
// Is a terminal. In reality the user is unlikely to actually paste
// CAR data into this terminal, but this message may serve to make
// them aware that they can/should pipe data into this command.
stopKeys := "Ctrl+D"
if runtime.GOOS == "windows" {
stopKeys = "Ctrl+Z, Enter"
}
fmt.Fprintf(c.App.ErrWriter, "Reading from stdin; use %s to end\n", stopKeys)
}
}
var err error
store, roots, err = NewStdinReadStorage(c.App.Reader)
if err != nil {
return err
}
} else {
carFile, err := os.Open(c.String("file"))
if err != nil {
return err
}
store, err = carstorage.OpenReadable(carFile)
if err != nil {
return err
}
roots = store.(carstorage.ReadableCar).Roots()
carFile, err := os.Open(c.String("file"))
if err != nil {
return err
}
store, err = carstorage.OpenReadable(carFile)
if err != nil {
return err
}
roots = store.(carstorage.ReadableCar).Roots()

ls := cidlink.DefaultLinkSystem()
ls.TrustedStorage = true
ls.SetReadStorage(store)

path, err := pathSegments(c.String("path"))
if err != nil {
return err
paths := c.Args().Slice()
if len(paths) == 0 {
paths = append(paths, "")
}

var extractedFiles int
for _, root := range roots {
count, err := lib.ExtractToDir(c.Context, &ls, root, outputDir, path, c.IsSet("verbose"), c.App.ErrWriter)

for _, p := range paths {
path, err := pathSegments(p)
if err != nil {
return err
}
extractedFiles += count

for _, root := range roots {
count, err := lib.ExtractToDir(c.Context, &ls, root, outputDir, path, c.IsSet("verbose"), c.App.ErrWriter)
if err != nil {
return err
}
extractedFiles += count
}
}

if extractedFiles == 0 {
return cli.Exit("no files extracted", 1)
} else {
Expand Down
2 changes: 1 addition & 1 deletion cmd/car/testdata/script/create-extract.txt
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
car create --file=out.car foo.txt bar.txt
mkdir out
car extract -v -f out.car out
car extract -v -f out.car -o out
stderr -count=2 'txt$'
stderr -count=1 '^extracted 2 file\(s\)$'
car create --file=out2.car out/foo.txt out/bar.txt
Expand Down
45 changes: 22 additions & 23 deletions cmd/car/testdata/script/extract.txt
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# full DAG export, everything in the CAR
mkdir actual-full
car extract -f ${INPUTS}/simple-unixfs.car actual-full
car extract -f ${INPUTS}/simple-unixfs.car -o actual-full
stderr '^extracted 9 file\(s\)$'
cmp actual-full/a/1/A.txt expected/a/1/A.txt
cmp actual-full/a/2/B.txt expected/a/2/B.txt
Expand All @@ -12,24 +12,9 @@ cmp actual-full/c/9/I.txt expected/c/9/I.txt
cmp actual-full/c/7/G.txt expected/c/7/G.txt
cmp actual-full/c/8/H.txt expected/c/8/H.txt

# full DAG export, everything in the CAR, accepted from stdin
mkdir actual-stdin
stdin ${INPUTS}/simple-unixfs.car
car extract actual-stdin
stderr '^extracted 9 file\(s\)$'
cmp actual-stdin/a/1/A.txt expected/a/1/A.txt
cmp actual-stdin/a/2/B.txt expected/a/2/B.txt
cmp actual-stdin/a/3/C.txt expected/a/3/C.txt
cmp actual-stdin/b/5/E.txt expected/b/5/E.txt
cmp actual-stdin/b/6/F.txt expected/b/6/F.txt
cmp actual-stdin/b/4/D.txt expected/b/4/D.txt
cmp actual-stdin/c/9/I.txt expected/c/9/I.txt
cmp actual-stdin/c/7/G.txt expected/c/7/G.txt
cmp actual-stdin/c/8/H.txt expected/c/8/H.txt

# full DAG export, everything in the CAR, but the CAR is missing blocks (incomplete DAG)
mkdir actual-missing
car extract -f ${INPUTS}/simple-unixfs-missing-blocks.car actual-missing
car extract -f ${INPUTS}/simple-unixfs-missing-blocks.car -o actual-missing
stderr -count=1 'data for entry not found: /b/4 \(skipping\.\.\.\)'
stderr -count=1 'data for entry not found: /b/5/E.txt \(skipping\.\.\.\)'
stderr -count=1 'data for entry not found: /b/6 \(skipping\.\.\.\)'
Expand All @@ -46,7 +31,7 @@ cmp actual-missing/c/8/H.txt expected/c/8/H.txt

# path-based partial export, everything under the path specified (also without leading / in path)
mkdir actual-partial
car extract -f ${INPUTS}/simple-unixfs.car -p b actual-partial
car extract -f ${INPUTS}/simple-unixfs.car -o actual-partial b
stderr '^extracted 3 file\(s\)$'
! exists actual-partial/a/1/A.txt
! exists actual-partial/a/2/B.txt
Expand All @@ -60,7 +45,7 @@ cmp actual-partial/b/4/D.txt expected/b/4/D.txt

# path-based single-file export (also with leading /)
mkdir actual-single
car extract -f ${INPUTS}/simple-unixfs.car -p /a/2/B.txt actual-single
car extract -f ${INPUTS}/simple-unixfs.car -o actual-single /a/2/B.txt
stderr '^extracted 1 file\(s\)$'
! exists actual-single/a/1/A.txt
cmp actual-single/a/2/B.txt expected/a/2/B.txt
Expand All @@ -72,18 +57,32 @@ cmp actual-single/a/2/B.txt expected/a/2/B.txt
! exists actual-single/c/7/G.txt
! exists actual-single/c/8/H.txt

# path-based multiple export
mkdir actual-multiple
car extract -f ${INPUTS}/simple-unixfs.car -o actual-multiple /a b/6 /c/7/G.txt
stderr '^extracted 5 file\(s\)$'
cmp actual-multiple/a/1/A.txt expected/a/1/A.txt
cmp actual-multiple/a/2/B.txt expected/a/2/B.txt
cmp actual-multiple/a/3/C.txt expected/a/3/C.txt
! exists actual-multiple/b/5/E.txt
cmp actual-multiple/b/6/F.txt expected/b/6/F.txt
! exists actual-multiple/b/4/D.txt
! exists actual-multiple/c/9/I.txt
cmp actual-multiple/c/7/G.txt expected/c/7/G.txt
! exists actual-multiple/c/8/H.txt

# extract that doesn't yield any files should error
! car extract -f ${INPUTS}/simple-unixfs-missing-blocks.car -p b
! car extract -f ${INPUTS}/simple-unixfs-missing-blocks.car b
stderr '^no files extracted$'

# car with only one file, nested inside sharded directory, output to stdout
car extract -f ${INPUTS}/wikipedia-cryptographic-hash-function.car -p wiki/Cryptographic_hash_function -
car extract -f ${INPUTS}/wikipedia-cryptographic-hash-function.car -o - wiki/Cryptographic_hash_function
stderr '^extracted 1 file\(s\)$'
stdout -count=1 '^ <title>Cryptographic hash function</title>$'

# car with only one file, full extract, lots of errors
mkdir actual-wiki
car extract -f ${INPUTS}/wikipedia-cryptographic-hash-function.car actual-wiki
car extract -f ${INPUTS}/wikipedia-cryptographic-hash-function.car -o actual-wiki
stderr '^extracted 1 file\(s\)$'
stderr -count=1 '^data for entry not found for 570 unknown sharded entries \(skipped\.\.\.\)$'
# random sampling of expected skip errors
Expand All @@ -110,4 +109,4 @@ c9I
-- expected/c/7/G.txt --
c7G
-- expected/c/8/H.txt --
c8H
c8H