diff --git a/cue/load/config.go b/cue/load/config.go index 0ed8052ba13..4dd246b752e 100644 --- a/cue/load/config.go +++ b/cue/load/config.go @@ -20,6 +20,7 @@ import ( "io" "io/fs" "os" + "path" "path/filepath" "runtime" @@ -164,6 +165,28 @@ type Config struct { // in the _ package. Package string + // FS, if non-nil, provides the filesystem used by the loader for discovering + // packages, resolving modules, and reading files. + // + // When FS is nil, the loader uses the host operating system filesystem + // (os.Stat, os.ReadDir, os.Open), which is the default and preserves the + // existing behavior. + // + // When FS is set, all filesystem operations performed by the loader are + // routed through FS. In this mode, paths are interpreted as slash-separated + // paths within FS, and Config.Dir is treated as a directory inside FS rather + // than a host filesystem path. + // + // The loader does not require FS to support symbolic links. When using FS, + // any operations that would rely on os.Lstat semantics on the host filesystem + // are performed using fs.Stat instead, consistent with the capabilities of + // io/fs. + // + // FS enables loading CUE packages and modules from virtual or embedded + // filesystems (for example, embed.FS or fstest.MapFS) without accessing + // the host filesystem. + FS fs.FS + // Dir is the base directory for import path resolution. // For example, it is used to determine the main module, // and rooted import paths starting with "./" are relative to it. @@ -279,7 +302,7 @@ type Config struct { // An application may supply a custom implementation of ParseFile to change // the effective file contents or the behavior of the parser, or to modify // the syntax tree. - ParseFile func(name string, src interface{}, cfg parser.Config) (*ast.File, error) + ParseFile func(name string, src any, cfg parser.Config) (*ast.File, error) // Overlay provides a mapping of absolute file paths to file contents, // which are overlaid on top of the host operating system when loading files. @@ -289,6 +312,9 @@ type Config struct { // If an overlaid file does not exist in the host filesystem, // the loader behaves as if the overlaid file exists with its contents, // and that that all of its parent directories exist too. + // + // When FS is set, overlay paths must be absolute loadFS paths + // (that is, rooted at loadFsRoot). Overlay map[string]Source // Stdin defines an alternative for os.Stdin for the file "-". When used, @@ -353,13 +379,22 @@ func addImportQualifier(pkg importPath, name string) (importPath, error) { func (c Config) complete() (cfg *Config, err error) { // Ensure [Config.Dir] is a clean and absolute path, // necessary for matching directory prefixes later. - if c.Dir == "" { - c.Dir, err = os.Getwd() - if err != nil { + if c.FS == nil { + if c.Dir == "" { + c.Dir, err = os.Getwd() + if err != nil { + return nil, err + } + } else if c.Dir, err = filepath.Abs(c.Dir); err != nil { return nil, err } - } else if c.Dir, err = filepath.Abs(c.Dir); err != nil { - return nil, err + } else { + if c.Dir == "" { + c.Dir = "." + } + if !isLoaderAbs(c.Dir) { + c.Dir = path.Clean(path.Join(loadFSRoot, c.Dir)) + } } // TODO: we could populate this already with absolute file paths, diff --git a/cue/load/fs.go b/cue/load/fs.go index 411a2f2db46..c09c5903e3d 100644 --- a/cue/load/fs.go +++ b/cue/load/fs.go @@ -21,6 +21,7 @@ import ( "io" iofs "io/fs" "os" + "path" "path/filepath" "slices" "strings" @@ -56,17 +57,179 @@ func (f *overlayFile) Mode() iofs.FileMode { } func (f *overlayFile) ModTime() time.Time { return f.modtime } func (f *overlayFile) IsDir() bool { return f.isDir } -func (f *overlayFile) Sys() interface{} { return nil } +func (f *overlayFile) Sys() any { return nil } + +// loadFSRoot is the synthetic absolute root used when loading from Config.FS. +// It is intentionally not an OS-absolute path to avoid platform-specific semantics. +// All loader paths are rooted here when FS is non-nil. +const loadFSRoot = "@fs" + +func isLoaderAbs(p string) bool { + return strings.HasPrefix(p, loadFSRoot) +} + +func canonicalOverlayPath(p string) string { + p = filepath.ToSlash(p) + return path.Clean(p) +} + +// loadFS provides access to the source filesystem used for loading CUE +// packages and modules. It abstracts over the host filesystem and +// virtual filesystems supplied via Config.FS. +// +// All direct os.* filesystem access must go through this type. +// +// loadFS operates exclusively on absolute loader paths. +// When fs != nil, all paths are rooted at loadFSRoot and use slash separators. +type loadFS struct { + fs iofs.FS +} + +type fakeDirInfo struct{} + +func (fakeDirInfo) Name() string { return "" } +func (fakeDirInfo) Size() int64 { return 0 } +func (fakeDirInfo) Mode() iofs.FileMode { return iofs.ModeDir | 0o555 } +func (fakeDirInfo) ModTime() time.Time { return time.Time{} } +func (fakeDirInfo) IsDir() bool { return true } +func (fakeDirInfo) Sys() any { return nil } + +type fakeRootFile struct{ fs iofs.FS } + +func (f fakeRootFile) Stat() (iofs.FileInfo, error) { return fakeDirInfo{}, nil } +func (f fakeRootFile) Read(p []byte) (int, error) { return 0, io.EOF } +func (f fakeRootFile) Close() error { return nil } +func (f fakeRootFile) ReadDir(n int) ([]iofs.DirEntry, error) { return iofs.ReadDir(f.fs, ".") } + +// isNotExist reports whether err indicates that a path does not exist. +// +// This helper exists to avoid using os.IsNotExist directly at call sites. +// The loader may operate on a virtual filesystem provided via Config.FS, +// whose errors are not guaranteed to satisfy os.IsNotExist. Instead, all +// filesystem access is normalized to io/fs semantics and tested using +// errors.Is(err, fs.ErrNotExist). +// +// All direct uses of os.* filesystem functions must be confined to loadFS. +func isNotExist(err error) bool { + return errors.Is(err, iofs.ErrNotExist) +} + +func stripFSRoot(path string) string { + tr := func(p string) string { + p = strings.ReplaceAll(p, `\`, `/`) + p = strings.TrimPrefix(p, loadFSRoot) + return strings.TrimPrefix(p, `/`) + } + + for strings.Contains(path, loadFSRoot) { + path = tr(path) + } + + return path +} + +func (l loadFS) Stat(name string) (iofs.FileInfo, error) { + if l.fs == nil { + return os.Stat(name) + } + + strip := stripFSRoot(name) + if strip == "" { + return fakeDirInfo{}, nil + } + return iofs.Stat(l.fs, strip) +} + +func (l loadFS) Lstat(name string) (iofs.FileInfo, error) { + if l.fs == nil { + return os.Lstat(name) + } + + strip := stripFSRoot(name) + if strip == "" { + return fakeDirInfo{}, nil + } + // fs.FS has no concept of symlinks; fall back to Stat. + return iofs.Stat(l.fs, strip) +} + +func (l loadFS) ReadDir(name string) ([]iofs.DirEntry, error) { + if l.fs == nil { + return os.ReadDir(name) + } + + strip := stripFSRoot(name) + if strip == "" { + return iofs.ReadDir(l.fs, ".") + } + return iofs.ReadDir(l.fs, strip) +} + +func (l loadFS) ReadFile(name string) ([]byte, error) { + if l.fs == nil { + return os.ReadFile(name) + } + return iofs.ReadFile(l.fs, stripFSRoot(name)) +} + +func (l loadFS) Open(name string) (iofs.File, error) { + if l.fs == nil { + return os.Open(name) + } + + strip := stripFSRoot(name) + if strip == "" { + return fakeRootFile(l), nil + } + return l.fs.Open(strip) +} + +func (l loadFS) IsAbs(p string) bool { + if l.fs == nil { + return filepath.IsAbs(p) + } + return isLoaderAbs(p) +} + +func (l loadFS) Clean(p string) string { + if l.fs == nil { + return filepath.Clean(p) + } + return path.Clean(p) +} + +func (l loadFS) Join(p ...string) string { + if l.fs == nil { + return filepath.Join(p...) + } + return path.Join(p...) +} + +func (l loadFS) Split(p string) (string, string) { + if l.fs == nil { + return filepath.Split(p) + } + return path.Split(p) +} + +func (l loadFS) Dir(p string) string { + if l.fs == nil { + return filepath.Dir(p) + } + return path.Dir(p) +} // A fileSystem specifies the supporting context for a build. type fileSystem struct { - overlayDirs map[string]map[string]*overlayFile - cwd string + cwd string + lfs loadFS + fileCache *fileCache + overlayDirs map[string]map[string]*overlayFile } func (fs *fileSystem) getDir(dir string, create bool) map[string]*overlayFile { - dir = filepath.Clean(dir) + dir = fs.lfs.Clean(dir) m, ok := fs.overlayDirs[dir] if !ok && create { m = map[string]*overlayFile{} @@ -98,22 +261,27 @@ func (fs *fileSystem) ioFS(root string, languageVersion string) iofs.FS { func newFileSystem(cfg *Config) (*fileSystem, error) { fs := &fileSystem{ cwd: cfg.Dir, + lfs: loadFS{fs: cfg.FS}, overlayDirs: map[string]map[string]*overlayFile{}, } // Organize overlay for filename, src := range cfg.Overlay { - if !filepath.IsAbs(filename) { + // Normalize overlay path to canonical loader-absolute form. + filename = fs.makeAbs(filename) + + if !fs.lfs.IsAbs(filename) { return nil, fmt.Errorf("non-absolute file path %q in overlay", filename) } // TODO: do we need to further clean the path or check that the // specified files are within the root/ absolute files? - dir, base := filepath.Split(filename) + dir, base := fs.lfs.Split(filename) m := fs.getDir(dir, true) b, file, err := src.contents() if err != nil { return nil, err } + base = canonicalOverlayPath(base) m[base] = &overlayFile{ basename: base, contents: b, @@ -123,7 +291,7 @@ func newFileSystem(cfg *Config) (*fileSystem, error) { for { prevdir := dir - dir, base = filepath.Split(filepath.Dir(dir)) + dir, base = fs.lfs.Split(fs.lfs.Dir(dir)) if dir == prevdir || dir == "" { break } @@ -141,19 +309,29 @@ func newFileSystem(cfg *Config) (*fileSystem, error) { return fs, nil } -func (fs *fileSystem) makeAbs(path string) string { - if filepath.IsAbs(path) { - return path +func (fs *fileSystem) makeAbs(p string) string { + if fs.lfs.fs == nil { + if fs.lfs.IsAbs(p) { + return p + } + return fs.lfs.Join(fs.cwd, p) + } + + // Normalize OS-rooted paths (Windows: "\foo", *NIX: "/foo") + p = strings.TrimLeft(p, string(filepath.Separator)) + + if isLoaderAbs(p) { + return path.Clean(p) } - return filepath.Join(fs.cwd, path) + return path.Join(loadFSRoot, p) } func (fs *fileSystem) readDir(path string) ([]iofs.DirEntry, errors.Error) { path = fs.makeAbs(path) m := fs.getDir(path, false) - items, err := os.ReadDir(path) + items, err := fs.lfs.ReadDir(path) if err != nil { - if !os.IsNotExist(err) || m == nil { + if !isNotExist(err) || m == nil { return nil, errors.Wrapf(err, token.NoPos, "readDir") } } @@ -179,7 +357,9 @@ func (fs *fileSystem) readDir(path string) ([]iofs.DirEntry, errors.Error) { } func (fs *fileSystem) getOverlay(path string) *overlayFile { - dir, base := filepath.Split(path) + path = canonicalOverlayPath(path) + + dir, base := fs.lfs.Split(path) if m := fs.getDir(dir, false); m != nil { return m[base] } @@ -191,7 +371,7 @@ func (fs *fileSystem) stat(path string) (iofs.FileInfo, errors.Error) { if fi := fs.getOverlay(path); fi != nil { return fi, nil } - fi, err := os.Stat(path) + fi, err := fs.lfs.Stat(path) if err != nil { return nil, errors.Wrapf(err, token.NoPos, "stat") } @@ -203,20 +383,46 @@ func (fs *fileSystem) lstat(path string) (iofs.FileInfo, errors.Error) { if fi := fs.getOverlay(path); fi != nil { return fi, nil } - fi, err := os.Lstat(path) + fi, err := fs.lfs.Lstat(path) if err != nil { return nil, errors.Wrapf(err, token.NoPos, "stat") } return fi, nil } -func (fs *fileSystem) openFile(path string) (io.ReadCloser, errors.Error) { +type fileWithStat struct { + io.ReadCloser + info iofs.FileInfo +} + +func (f fileWithStat) Stat() (iofs.FileInfo, error) { + return f.info, nil +} + +func (fs *fileSystem) openFileWithStat(path string) (iofs.File, error) { + path = fs.makeAbs(path) + + if fi := fs.getOverlay(path); fi != nil { + return fileWithStat{ + ReadCloser: io.NopCloser(bytes.NewReader(fi.contents)), + info: fi, + }, nil + } + + f, err := fs.lfs.Open(path) + if err != nil { + return nil, err + } + return f, nil +} + +func (fs *fileSystem) openFile(path string) (io.ReadCloser, error) { path = fs.makeAbs(path) if fi := fs.getOverlay(path); fi != nil { return io.NopCloser(bytes.NewReader(fi.contents)), nil } - f, err := os.Open(path) + f, err := fs.lfs.Open(path) if err != nil { return nil, errors.Wrapf(err, token.NoPos, "load") } @@ -241,7 +447,6 @@ func (fs *fileSystem) walk(root string, f walkFunc) error { return nil } return err - } func (fs *fileSystem) walkRec(path string, entry iofs.DirEntry, f walkFunc) errors.Error { @@ -264,7 +469,7 @@ func (fs *fileSystem) walkRec(path string, entry iofs.DirEntry, f walkFunc) erro } for _, entry := range dir { - filename := filepath.Join(path, entry.Name()) + filename := fs.lfs.Join(path, entry.Name()) err = fs.walkRec(filename, entry, f) if err != nil { if !entry.IsDir() || err != skipDir { @@ -318,7 +523,14 @@ func (fs *ioFS) absPathFromFSPath(name string) (string, error) { if strings.ContainsAny(name, ":\\") { return "", fmt.Errorf("invalid io/fs path %q", name) } - return filepath.Join(fs.root, name), nil + + if fs.fs.lfs.fs != nil { + if isLoaderAbs(name) { + return name, nil + } + return path.Join(loadFSRoot, name), nil + } + return fs.fs.lfs.Join(fs.root, name), nil } // ReadDir implements [io/fs.ReadDirFS]. @@ -339,7 +551,7 @@ func (fs *ioFS) ReadFile(name string) ([]byte, error) { if fi := fs.fs.getOverlay(fpath); fi != nil { return bytes.Clone(fi.contents), nil } - return os.ReadFile(fpath) + return fs.fs.lfs.ReadFile(fpath) } var _ module.ReadCUEFS = (*ioFS)(nil) @@ -370,7 +582,7 @@ func (fs *ioFS) ReadCUEFile(path string, cfg parser.Config) (*ast.File, error) { } data = fi.contents } else { - data, err = os.ReadFile(fpath) + data, err = fs.fs.lfs.ReadFile(fpath) if err != nil { cache.mu.Lock() defer cache.mu.Unlock() @@ -457,7 +669,8 @@ func (fs *fileSystem) getCUESyntax(bf *build.File, cfg parser.Config) (*ast.File } encodingCfg := fs.fileCache.config encodingCfg.ParserConfig = cfg - d := encoding.NewDecoder(fs.fileCache.ctx, bf, &encodingCfg) + openFn := fs.openFileWithStat + d := encoding.NewDecoderWithOpenFn(fs.fileCache.ctx, bf, &encodingCfg, openFn) defer d.Close() // Note: CUE files can never have multiple file parts. f, err := d.File(), d.Err() diff --git a/cue/load/fs_test.go b/cue/load/fs_test.go index 735c8c36022..01e0672047e 100644 --- a/cue/load/fs_test.go +++ b/cue/load/fs_test.go @@ -5,9 +5,13 @@ import ( "os" "path/filepath" "slices" + "strings" "testing" "testing/fstest" + "cuelang.org/go/cue" + "cuelang.org/go/cue/cuecontext" + "cuelang.org/go/internal/source" "github.com/go-quicktest/qt" ) @@ -57,6 +61,262 @@ func TestIOFS(t *testing.T) { } } +func TestLoadFromFS(t *testing.T) { + cfs := fstest.MapFS{ + "cue.mod/module.cue": &fstest.MapFile{ + Data: []byte(` + module: "example.com" + language: version: "v0.8.0" + `), + }, + "main.cue": &fstest.MapFile{ + Data: []byte(` + package main + + import "example.com/lib" + + x: lib.y + `), + }, + "lib/lib.cue": &fstest.MapFile{ + Data: []byte(` + package lib + + y: 42 + `), + }, + } + + cfg := &Config{ + FS: cfs, + Dir: ".", + } + + insts := Instances([]string{"."}, cfg) + qt.Assert(t, qt.HasLen(insts, 1)) + qt.Assert(t, qt.IsNil(insts[0].Err)) + + ctx := cuecontext.New() + val := ctx.BuildInstance(insts[0]) + qt.Assert(t, qt.IsNil(val.Err())) + + x := val.LookupPath(cue.ParsePath("x")) + qt.Assert(t, qt.IsTrue(x.Exists())) + n, err := x.Int64() + qt.Assert(t, qt.IsNil(err)) + qt.Assert(t, qt.Equals(n, int64(42))) +} + +func TestLoadFromFS_IgnoresHostFS(t *testing.T) { + tmp := t.TempDir() + + // Host FS file that must NOT be used + err := os.WriteFile(filepath.Join(tmp, "main.cue"), []byte(` + package main + x: 999 + `), 0o644) + qt.Assert(t, qt.IsNil(err)) + + cfs := fstest.MapFS{ + "main.cue": &fstest.MapFile{ + Data: []byte(` + package main + x: 42 + `), + }, + } + + cfg := &Config{ + FS: cfs, + Dir: ".", // IMPORTANT: virtual root, not tmp + } + + insts := Instances([]string{"."}, cfg) + qt.Assert(t, qt.HasLen(insts, 1)) + qt.Assert(t, qt.IsNil(insts[0].Err)) + + val := cuecontext.New().BuildInstance(insts[0]) + x := val.LookupPath(cue.ParsePath("x")) + n, err := x.Int64() + + qt.Assert(t, qt.IsNil(err)) + qt.Assert(t, qt.Equals(n, int64(42))) +} + +func TestLoadFromFS_SubdirAsDir(t *testing.T) { + cfs := fstest.MapFS{ + "app/main.cue": &fstest.MapFile{ + Data: []byte(` + package main + x: 1 + `), + }, + } + + cfg := &Config{ + FS: cfs, + Dir: "app", + } + + insts := Instances([]string{"."}, cfg) + qt.Assert(t, qt.HasLen(insts, 1)) + qt.Assert(t, qt.IsNil(insts[0].Err)) + + val := cuecontext.New().BuildInstance(insts[0]) + x := val.LookupPath(cue.ParsePath("x")) + n, err := x.Int64() + + qt.Assert(t, qt.IsNil(err)) + qt.Assert(t, qt.Equals(n, int64(1))) +} + +func TestLoadFromFS_MissingModulePackage(t *testing.T) { + cfs := fstest.MapFS{ + "cue.mod/module.cue": &fstest.MapFile{ + Data: []byte(` + module: "example.com" + language: version: "v0.8.0" + `), + }, + "main.cue": &fstest.MapFile{ + Data: []byte(` + package main + import "example.com/foo" + x: foo.y + `), + }, + } + + cfg := &Config{ + FS: cfs, + Dir: ".", + } + + insts := Instances([]string{"."}, cfg) + qt.Assert(t, qt.HasLen(insts, 1)) + qt.Assert(t, qt.IsNotNil(insts[0].Err)) + + qt.Assert(t, qt.StringContains( + insts[0].Err.Error(), + "cannot find package", + )) +} + +func TestLoadFromFS_MissingModule(t *testing.T) { + cfs := fstest.MapFS{ + "main.cue": &fstest.MapFile{ + Data: []byte(` + package main + import "example.com/foo" + x: foo.y + `), + }, + } + + cfg := &Config{ + FS: cfs, + Dir: ".", + } + + insts := Instances([]string{"."}, cfg) + qt.Assert(t, qt.HasLen(insts, 1)) + qt.Assert(t, qt.IsNotNil(insts[0].Err)) + + qt.Assert(t, qt.StringContains( + insts[0].Err.Error(), + "imports are unavailable because there is no cue.mod/module.cue file", + )) +} + +func TestLoadFromFS_OverlayOverrides(t *testing.T) { + cfs := fstest.MapFS{ + "main.cue": &fstest.MapFile{ + Data: []byte(` + package main + x: 1 + `), + }, + } + + cfg := &Config{ + FS: cfs, + Dir: ".", + Overlay: map[string]Source{ + "@fs/main.cue": FromString(` + package main + x: 99 + `), + }, + } + + insts := Instances([]string{"."}, cfg) + qt.Assert(t, qt.HasLen(insts, 1)) + qt.Assert(t, qt.IsNil(insts[0].Err)) + + val := cuecontext.New().BuildInstance(insts[0]) + x := val.LookupPath(cue.ParsePath("x")) + n, err := x.Int64() + + qt.Assert(t, qt.IsNil(err)) + qt.Assert(t, qt.Equals(n, int64(99))) +} + +func TestLoadFromFS_EncodingUsesVirtualFS(t *testing.T) { + called := false + + cfs := fstest.MapFS{ + "main.cue": &fstest.MapFile{ + Data: []byte(` + package main + x: 1 + `), + }, + } + + cfg := &Config{ + FS: cfs, + Dir: ".", + } + + // Trap host FS access + old := source.OsOpen + source.OsOpen = func(name string) (fs.File, error) { + called = true + return nil, fs.ErrNotExist + } + defer func() { source.OsOpen = old }() + + insts := Instances([]string{"."}, cfg) + qt.Assert(t, qt.HasLen(insts, 1)) + qt.Assert(t, qt.IsNil(insts[0].Err)) + + qt.Assert(t, qt.IsFalse(called)) +} + +type assertNoSyntheticFS struct { + t *testing.T +} + +func (fsys assertNoSyntheticFS) Open(name string) (fs.File, error) { + if strings.HasPrefix(name, loadFSRoot) { + fsys.t.Fatalf("fs.FS.Open called with synthetic path %q", name) + } + return nil, fs.ErrNotExist +} + +func TestLoaderDoesNotLeakSyntheticFSPrefix(t *testing.T) { + ctx := cuecontext.New() + + cfg := &Config{ + FS: assertNoSyntheticFS{t: t}, + Dir: ".", + } + + for _, inst := range Instances([]string{"whatever.cue"}, cfg) { + _ = ctx.BuildInstance(inst) + } +} + func writeFile(t *testing.T, fpath string, content string) { err := os.MkdirAll(filepath.Dir(fpath), 0o777) qt.Assert(t, qt.IsNil(err)) diff --git a/cue/load/import.go b/cue/load/import.go index 7daa40a16fa..b92479cc8bd 100644 --- a/cue/load/import.go +++ b/cue/load/import.go @@ -19,7 +19,6 @@ import ( "fmt" "io" "io/fs" - "os" pathpkg "path" "path/filepath" "slices" @@ -55,7 +54,7 @@ import ( // is present. // _ anonymous files (which may be marked with _) // * all packages -func (l *loader) importPkg(pos token.Pos, p *build.Instance) []*build.Instance { +func (l *loader) importPkg(_ token.Pos, p *build.Instance) []*build.Instance { retErr := func(errs errors.Error) []*build.Instance { // XXX: move this loop to ReportError for _, err := range errors.Errors(errs) { @@ -276,8 +275,8 @@ func setFileSource(cfg *Config, f *build.File) error { return nil } - if !filepath.IsAbs(fullPath) { - fullPath = filepath.Join(cfg.Dir, fullPath) + if !cfg.fileSystem.lfs.IsAbs(fullPath) { + fullPath = cfg.fileSystem.lfs.Join(cfg.Dir, fullPath) // Ensure that encoding.NewDecoder will work correctly. f.Filename = fullPath } @@ -292,12 +291,12 @@ func setFileSource(cfg *Config, f *build.File) error { // Note that we do this after ensuring fullPath is absolute, and after checking // whether the overlay provides the source. - info, err := os.Stat(fullPath) + info, err := cfg.fileSystem.lfs.Stat(fullPath) if err != nil { return err } if !info.Mode().IsRegular() { - b, err := os.ReadFile(fullPath) + b, err := cfg.fileSystem.lfs.ReadFile(fullPath) if err != nil { return err } @@ -424,9 +423,10 @@ func (l *loader) newInstance(pos token.Pos, p importPath) *build.Instance { i.Dir = dir parts := ast.ParseImportPath(string(p)) i.PkgName = parts.Qualifier - if i.PkgName == "" { + switch i.PkgName { + case "": i.Err = errors.Append(i.Err, l.errPkgf([]token.Pos{pos}, "cannot determine package name for %q; set it explicitly with ':'", p)) - } else if i.PkgName == "_" { + case "_": i.Err = errors.Append(i.Err, l.errPkgf([]token.Pos{pos}, "_ is not a valid import path qualifier in %q", p)) } i.DisplayPath = string(p) @@ -464,7 +464,7 @@ func (l *loader) absDirFromImportPath(pos token.Pos, p importPath) (dir string, return dir, mv, modRoot, nil } -func (l *loader) absDirFromImportPath1(pos token.Pos, p importPath) (absDir string, mv module.Version, modRoot module.SourceLoc, err error) { +func (l *loader) absDirFromImportPath1(_ token.Pos, p importPath) (absDir string, mv module.Version, modRoot module.SourceLoc, err error) { failf := func(f string, a ...any) (string, module.Version, module.SourceLoc, error) { return "", module.Version{}, module.SourceLoc{}, fmt.Errorf(f, a...) } diff --git a/internal/encoding/encoding.go b/internal/encoding/encoding.go index a2d17d7651a..67f0c63c268 100644 --- a/internal/encoding/encoding.go +++ b/internal/encoding/encoding.go @@ -62,8 +62,10 @@ type Decoder struct { err error } -type interpretFunc func(cue.Value) (file *ast.File, err error) -type rewriteFunc func(*ast.File) (file *ast.File, err error) +type ( + interpretFunc func(cue.Value) (file *ast.File, err error) + rewriteFunc func(*ast.File) (file *ast.File, err error) +) func (i *Decoder) Filename() string { return i.filename } @@ -159,6 +161,17 @@ type Config struct { // // This may change the contents of f. func NewDecoder(ctx *cue.Context, f *build.File, cfg *Config) *Decoder { + return NewDecoderWithOpenFn(ctx, f, cfg, source.OsOpen) +} + +// NewDecoderWithOpenFn returns a stream of non-rooted data expressions. The encoding +// type of f must be a data type, but does not have to be an encoding that +// can stream. stdin is used in case the file is "-". +// +// openFn is used should the underlying source-layer need to open a f.Filename (IFF f.Source == nil). +// +// This may change the contents of f. +func NewDecoderWithOpenFn(ctx *cue.Context, f *build.File, cfg *Config, openFn source.OpenFn) *Decoder { if cfg == nil { cfg = &Config{} } @@ -186,7 +199,7 @@ func NewDecoder(ctx *cue.Context, f *build.File, cfg *Config) *Decoder { r = cfg.Stdin i.size = -1 } else { - r, i.size, i.err = source.Open(f.Filename, f.Source) + r, i.size, i.err = source.OpenFunc(f.Filename, f.Source, openFn) if c, ok := r.(io.Closer); ok { i.closer = c } @@ -201,7 +214,6 @@ func NewDecoder(ctx *cue.Context, f *build.File, cfg *Config) *Decoder { openAPI := openAPIFunc(cfg, f) jsonSchema := jsonSchemaFunc(cfg, f) i.interpretFunc = func(v cue.Value) (file *ast.File, err error) { - switch i.interpretation = Detect(v); i.interpretation { case build.JSONSchema: return jsonSchema(v) diff --git a/internal/source/source.go b/internal/source/source.go index 6e43d481283..9efed51815f 100644 --- a/internal/source/source.go +++ b/internal/source/source.go @@ -20,11 +20,16 @@ import ( "bytes" "fmt" "io" + "io/fs" "math" "os" "strings" ) +type OpenFn func(name string) (fs.File, error) + +var OsOpen = func(name string) (fs.File, error) { return os.Open(name) } + // ReadAll loads the source bytes for the given arguments. If src != nil, // ReadAll converts src to a []byte if possible; otherwise it returns an // error. If src == nil, ReadAll returns the result of reading the file @@ -68,11 +73,21 @@ func ReadAllSize(r io.Reader, size int) ([]byte, error) { // Open creates a source reader for the given arguments. // If src != nil, Open converts src to an [io.Reader] if possible; otherwise it returns an error. -// If src == nil, Open returns the result of opening the file specified by filename. +// If src == nil, Open returns the result of [os.Open] using filename. // // The caller must check if the result is an [io.Closer], and if so, close it when done. // The size of the opened reader is returned if possible, or -1 otherwise. func Open(filename string, src any) (_ io.Reader, size int, _ error) { + return OpenFunc(filename, src, OsOpen) +} + +// OpenFunc creates a source reader for the given arguments. +// If src != nil, Open converts src to an [io.Reader] if possible; otherwise it returns an error. +// If src == nil, Open returns the result of openFn using filename. +// +// The caller must check if the result is an [io.Closer], and if so, close it when done. +// The size of the opened reader is returned if possible, or -1 otherwise. +func OpenFunc(filename string, src any, openFn OpenFn) (_ io.Reader, size int, _ error) { if src != nil { switch src := src.(type) { case string: @@ -86,14 +101,14 @@ func Open(filename string, src any) (_ io.Reader, size int, _ error) { } return nil, -1, fmt.Errorf("invalid source type %T", src) } - f, err := os.Open(filename) + f, err := openFn(filename) if err != nil { return nil, -1, err } return fileWithSize(f) } -func fileWithSize(f *os.File) (io.Reader, int, error) { +func fileWithSize(f fs.File) (io.Reader, int, error) { // If we just opened a regular file, return its size too. // If we can't get its size, such as non-regular files, don't give one. stat, err := f.Stat()