From 6834f6412f19082e5537333c75cd3438766f61f1 Mon Sep 17 00:00:00 2001 From: Igor Ignatyev Date: Tue, 24 Dec 2024 13:51:51 +0300 Subject: [PATCH 1/3] #55 handle empty package dir gracefully --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 30925a4..38c7c8c 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ *.gen.go .launchr/ .compose/ +.plasmactl From 3910dab711b65bbdc79e614e4f17983fe273fbf5 Mon Sep 17 00:00:00 2001 From: Igor Ignatyev Date: Tue, 24 Dec 2024 14:09:52 +0300 Subject: [PATCH 2/3] #55 handle empty package dir gracefully --- README.md | 80 ++++++----- compose/compose.go | 6 + compose/downloadManager.go | 70 ++++++--- compose/forms.go | 20 +-- compose/git.go | 287 +++++++++++++++++++++++++++++++++++-- compose/http.go | 29 ++-- compose/yaml.go | 25 ++-- plugin.go | 13 +- 8 files changed, 416 insertions(+), 114 deletions(-) diff --git a/README.md b/README.md index 021eea4..63f27c8 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ ## Composition Tool Specification + The composition tool is a command-line tool that helps developers manage dependencies for their projects. It allows developers to specify the dependencies for a project in a "plasma-compose.yaml" file, and then fetches and installs those dependencies @@ -8,28 +9,39 @@ The tool works by recursively fetching and processing the "plasma-compose.yaml" and its dependencies, and then merging the resulting filesystems into a single filesystem. ### CLI + The composition tool is invoked from the command line with the following syntax: launchr compose [options] Where options are: + * -w, --working-dir : The directory where temporary files should be stored during the composition process. Default is the .compose/packages * -s, --skip-not-versioned : Skip not versioned files from source directory (git only) -* --conflicts-verbosity: Log files conflicts in format "[curent-package] - path to file > Selectef from [domain, other package or current-package]" +* --conflicts-verbosity: Log files conflicts in format "[current-package] - path to file > Selected + from [domain, other package or current-package]" +* --interactive: Interactive mode allows to submit user credentials during action (default: true) Example usage - `launchr compose -w=./folder/something -s=1 or -s=true --conflicts-verbosity` -It's important to note that: if same file is present locally and also brought by a package, default strategy is that local file will be taken and package file ignored. [Different strategies](https://github.com/launchrctl/compose/blob/main/example/compose.example.yaml#L18-L35) can be difined to customize this behavior to your needs. - +It's important to note that: if same file is present locally and also brought by a package, default strategy is that +local file will be taken and package file +ignored. [Different strategies](https://github.com/launchrctl/compose/blob/main/example/compose.example.yaml#L18-L35) +can be difined to customize this behavior to your needs. ### `plasma-compose.yaml` File Format -The "plasma-compose.yaml" file is a text file that specifies the dependencies for a package, along with any necessary metadata and sources for those dependencies. + +The "plasma-compose.yaml" file is a text file that specifies the dependencies for a package, along with any necessary +metadata and sources for those dependencies. The file format includes the following elements: + - name: The name of the package. - version: The version number of the package. -- source: The source for the package, including the type of source (Git, HTTP), URL or file path, merge strategy and other metadata. +- source: The source for the package, including the type of source (Git, HTTP), URL or file path, merge strategy and + other metadata. - dependencies: A list of required dependencies. List of strategies: + - overwrite-local-file - remove-extra-local-files - ignore-extra-package-files @@ -38,40 +50,43 @@ List of strategies: Example: ```yaml -name: myproject -version: 1.0.0 +name: example dependencies: -- name: compose-example - source: - type: git - ref: master - tag: 0.0.1 - url: https://github.com/example/compose-example.git - strategy: - - name: remove-extra-local-files - path: - - path/to/remove-extra-local-files - - name: ignore-extra-package-files - path: - - library/inventories/platform_nodes/configuration/dev.yaml - - library/inventories/platform_nodes/configuration/prod.yaml - - library/inventories/platform_nodes/configuration/whatever.yaml + - name: compose-example + source: + type: git + ref: master # branch or tag name + url: https://github.com/example/compose-example.git + strategy: + - name: remove-extra-local-files + path: + - path/to/remove-extra-local-files + - name: ignore-extra-package-files + path: + - library/inventories/platform_nodes/configuration/dev.yaml + - library/inventories/platform_nodes/configuration/prod.yaml + - library/inventories/platform_nodes/configuration/whatever.yaml ``` - ### Fetching and Installing Dependencies -The composition tool fetches and installs dependencies for a package by recursively processing the "plasma-compose.yaml" files for each package and its dependencies. The tool follows these general steps: -1. Fetch the package source code from the specified source location. -2. Extract the package contents to a temporary directory. -3. Process the "plasma-compose.yaml" file for the package, fetching and installing any necessary dependencies recursively. -4. Merge the package filesystem into the final platform filesystem. -5. Repeat steps 1-4 for each package and its dependencies. +The composition tool fetches and installs dependencies for a package by recursively processing the "plasma-compose.yaml" +files for each package and its dependencies. The tool follows these general steps: + +1. Check if package exists locally and is up-to-date. If it's not, remove it from packages dir and proceed to next step. +2. Fetch the package from the specified location. +3. Extract the package contents to a packages directory. +4. Process the "plasma-compose.yaml" file for the package, fetching and installing any necessary dependencies + recursively. +5. Merge the package filesystem into the final platform filesystem. +6. Repeat steps 1-5 for each package and its dependencies. During this process, the composition tool keeps track of the dependencies for each package. ### Plasma-compose commands + it's possible to manipulate plasma-compose.yaml file using commands: + - plasmactl compose:add - plasmactl compose:update - plasmactl compose:delete @@ -84,8 +99,7 @@ For `compose:delete` it's possible to pass list of packaged to delete. In other cases, user will be prompted to CLI form to fill necessary data of packages. -Example of usage - +Examples of usage ``` launchr compose:add --url some-url --type http @@ -93,8 +107,6 @@ launchr compose:add --package package-name --url some-url --ref v1.0.0 launchr compose:update --package package-name --url some-url --ref v1.0.0 launchr compose:add --package package-name --url some-url --ref v1.0.0 --strategy overwrite-local-file --strategy-path "path1|path2" -launchr compose:add --package package-name --url some-url --ref v1.0.0 --strategy overwrite-local-file,remove-extra-local-files --strategy-path "path1|path2,path3|path4" +launchr compose:add --package package-name --url some-url --ref branch --strategy overwrite-local-file,remove-extra-local-files --strategy-path "path1|path2,path3|path4" launchr compose:add --package package-name --url some-url --ref v1.0.0 --strategy overwrite-local-file --strategy-path "path1|path2" --strategy remove-extra-local-files --strategy-path "path3|path4" - - ``` \ No newline at end of file diff --git a/compose/compose.go b/compose/compose.go index 6c57293..6385a31 100644 --- a/compose/compose.go +++ b/compose/compose.go @@ -46,6 +46,12 @@ func CreateComposer(pwd string, opts ComposerOptions, k keyring.Keyring) (*Compo return nil, err } + for _, dep := range config.Dependencies { + if dep.Source.Tag != "" { + launchr.Term().Warning().Printfln("found deprecated field `tag` in `%s` dependency. Use `ref` field for tags or branches.", dep.Name) + } + } + return &Composer{pwd, &opts, config, k}, nil } diff --git a/compose/downloadManager.go b/compose/downloadManager.go index eea59ae..d70887f 100644 --- a/compose/downloadManager.go +++ b/compose/downloadManager.go @@ -1,24 +1,24 @@ package compose import ( + "io" "os" "path/filepath" + + "github.com/launchrctl/launchr" ) const ( // GitType is const for GIT source type download. GitType = "git" - // SourceReferenceTag represents git tag source. - SourceReferenceTag = "tag" - // SourceReferenceBranch represents git branch source. - SourceReferenceBranch = "ref" // HTTPType is const for http source type download. HTTPType = "http" ) // Downloader interface type Downloader interface { - Download(pkg *Package, targetDir string, kw *keyringWrapper) error + Download(pkg *Package, targetDir string) error + EnsureLatest(pkg *Package, downloadPath string) (bool, error) } // DownloadManager struct, provides methods to fetch packages @@ -35,14 +35,14 @@ func CreateDownloadManager(keyring *keyringWrapper) DownloadManager { return DownloadManager{kw: keyring} } -func getDownloaderForPackage(downloadType string) Downloader { +func getDownloaderForPackage(downloadType string, kw *keyringWrapper) Downloader { switch { case downloadType == GitType: - return newGit() + return newGit(kw) case downloadType == HTTPType: - return newHTTP() + return newHTTP(kw) default: - return newGit() + return newGit(kw) } } @@ -85,16 +85,13 @@ func (m DownloadManager) recursiveDownload(yc *YamlCompose, kw *keyringWrapper, packagePath := filepath.Join(targetDir, pkg.GetName(), pkg.GetTarget()) - // Skip package download if it exists in packages dir. - if _, err := os.Stat(packagePath); os.IsNotExist(err) { - err = downloadPackage(pkg, targetDir, kw) - if err != nil { - return packages, err - } + err := downloadPackage(pkg, targetDir, kw) + if err != nil { + return packages, err } // If package has plasma-compose.yaml, proceed with it - if _, err := os.Stat(filepath.Join(packagePath, composeFile)); !os.IsNotExist(err) { + if _, err = os.Stat(filepath.Join(packagePath, composeFile)); !os.IsNotExist(err) { cfg, err := Lookup(os.DirFS(packagePath)) if err == nil { packages, err = m.recursiveDownload(cfg, kw, packages, pkg, targetDir) @@ -111,20 +108,53 @@ func (m DownloadManager) recursiveDownload(yc *YamlCompose, kw *keyringWrapper, } func downloadPackage(pkg *Package, targetDir string, kw *keyringWrapper) error { - downloader := getDownloaderForPackage(pkg.GetType()) + downloader := getDownloaderForPackage(pkg.GetType(), kw) packagePath := filepath.Join(targetDir, pkg.GetName()) downloadPath := filepath.Join(packagePath, pkg.GetTarget()) - if _, err := os.Stat(downloadPath); !os.IsNotExist(err) { - // Skip package download if folder exists in packages dir. + isLatest, err := downloader.EnsureLatest(pkg, downloadPath) + if err != nil { + return err + } + + if isLatest { return nil } + // Ensure old package doesn't exist in case of update. + err = os.RemoveAll(downloadPath) + if err != nil { + return err + } + // temporary if dtype := pkg.GetType(); dtype == HTTPType { downloadPath = packagePath } - err := downloader.Download(pkg, downloadPath, kw) + err = downloader.Download(pkg, downloadPath) + if err != nil { + errRemove := os.RemoveAll(downloadPath) + if errRemove != nil { + launchr.Log().Debug("error cleaning package folder", "path", downloadPath, "err", err) + } + } + return err } + +// IsEmptyDir check if directory has at least 1 file. +func IsEmptyDir(name string) (bool, error) { + f, err := os.Open(filepath.Clean(name)) + if err != nil { + return false, err + } + defer f.Close() + + _, err = f.Readdirnames(1) + if err == io.EOF { + return true, nil + } + + return false, err +} diff --git a/compose/forms.go b/compose/forms.go index def51fd..a804c4e 100644 --- a/compose/forms.go +++ b/compose/forms.go @@ -315,7 +315,6 @@ func processStrategiesForm(dependency *Dependency) error { } func preparePackageForm(dependency *Dependency, config *YamlCompose, isAdd bool) *huh.Form { - var refType string uniqueLimit := 1 if isAdd { uniqueLimit = 0 @@ -376,27 +375,11 @@ func preparePackageForm(dependency *Dependency, config *YamlCompose, isAdd bool) }), ), - huh.NewGroup( - huh.NewSelect[string](). - Title("- Select source reference"). - Options( - huh.NewOption("Tag", SourceReferenceTag).Selected(true), - huh.NewOption("Branch", SourceReferenceBranch), - ). - Value(&refType), - ).WithHideFunc(func() bool { return dependency.Source.Type != GitType }), - - huh.NewGroup( - huh.NewInput(). - Title("- Enter Tag"). - Value(&dependency.Source.Tag), - ).WithHideFunc(func() bool { return dependency.Source.Type != GitType || refType != SourceReferenceTag }), - huh.NewGroup( huh.NewInput(). Title("- Enter Ref"). Value(&dependency.Source.Ref), - ).WithHideFunc(func() bool { return dependency.Source.Type != GitType || refType != SourceReferenceBranch }), + ).WithHideFunc(func() bool { return dependency.Source.Type != GitType }), ) } @@ -431,5 +414,4 @@ func sanitizeDependency(dependency *Dependency) { dependency.Name = strings.TrimSpace(dependency.Name) dependency.Source.URL = strings.TrimSpace(dependency.Source.URL) dependency.Source.Ref = strings.TrimSpace(dependency.Source.Ref) - dependency.Source.Tag = strings.TrimSpace(dependency.Source.Tag) } diff --git a/compose/git.go b/compose/git.go index 43ff84b..7a0c756 100644 --- a/compose/git.go +++ b/compose/git.go @@ -2,9 +2,12 @@ package compose import ( "errors" + "fmt" "os" + "path/filepath" "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/config" "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/transport" "github.com/go-git/go-git/v5/plumbing/transport/http" @@ -12,14 +15,233 @@ import ( "github.com/launchrctl/launchr" ) -type gitDownloader struct{} +type gitDownloader struct { + k *keyringWrapper +} -func newGit() Downloader { - return &gitDownloader{} +func newGit(kw *keyringWrapper) Downloader { + return &gitDownloader{k: kw} +} + +func (g *gitDownloader) fetchRemotes(r *git.Repository, url string, refSpec []config.RefSpec) error { + remotes, errR := r.Remotes() + if errR != nil { + return errR + } + + launchr.Term().Printfln("Fetching remote %s", url) + for _, rem := range remotes { + options := git.FetchOptions{ + //RefSpecs: []config.RefSpec{"refs/*:refs/*", "HEAD:refs/heads/HEAD"}, + RefSpecs: refSpec, + Force: true, + } + + auths := []authorizationMode{authorisationNone, authorisationKeyring, authorisationManual} + for _, authType := range auths { + if authType == authorisationNone { + err := rem.Fetch(&options) + if err != nil { + if errors.Is(err, transport.ErrAuthenticationRequired) { + continue + } + + if !errors.Is(err, git.NoErrAlreadyUpToDate) { + return err + } + + return nil + } + } + + if authType == authorisationKeyring { + ci, err := g.k.getForURL(url) + if err != nil { + return err + } + + options.Auth = &http.BasicAuth{ + Username: ci.Username, + Password: ci.Password, + } + + err = rem.Fetch(&options) + if err != nil { + if errors.Is(err, transport.ErrAuthorizationFailed) || errors.Is(err, transport.ErrAuthenticationRequired) { + if g.k.interactive { + launchr.Term().Println("invalid auth, trying manual authorisation") + continue + } + } + + if !errors.Is(err, git.NoErrAlreadyUpToDate) { + return err + } + + return nil + } + } + + if authType == authorisationManual { + ci := keyring.CredentialsItem{} + ci.URL = url + ci, err := g.k.fillCredentials(ci) + if err != nil { + return err + } + + options.Auth = &http.BasicAuth{ + Username: ci.Username, + Password: ci.Password, + } + + err = rem.Fetch(&options) + if err != nil { + if !errors.Is(err, git.NoErrAlreadyUpToDate) { + return err + } + + return nil + } + } + + break + } + } + + return nil +} + +func (g *gitDownloader) EnsureLatest(pkg *Package, downloadPath string) (bool, error) { + if _, err := os.Stat(downloadPath); os.IsNotExist(err) { + // Return False in case package doesn't exist. + return false, nil + } + + emptyDir, err := IsEmptyDir(downloadPath) + if err != nil { + return false, err + } + + if emptyDir { + return false, nil + } + + r, err := git.PlainOpen(downloadPath) + if err != nil { + launchr.Log().Debug("git init error", "err", err) + return false, nil + } + + head, err := r.Head() + if err != nil { + launchr.Log().Debug("get head error", "err", err) + return false, fmt.Errorf("can't get HEAD of '%s', ensure package is valid", pkg.GetName()) + } + + headName := head.Name().Short() + pkgRefName := pkg.GetRef() + remoteRefName := pkgRefName + + if pkg.GetTarget() == TargetLatest && headName != "" { + pkgRefName = headName + remoteRefName = plumbing.HEAD.String() + } + + pullTarget := "" + isLatest := false + if headName == pkgRefName { + pullTarget = "branch" + isLatest, err = g.ensureLatestBranch(r, pkg.GetURL(), pkgRefName, remoteRefName) + if err != nil { + launchr.Term().Warning().Printfln("Couldn't check local branch, marking package %s(%s) as outdated, see debug for detailed error.", pkg.GetName(), pkgRefName) + launchr.Log().Debug("ensure branch error", "err", err) + return isLatest, nil + } + } else { + pullTarget = "tag" + isLatest, err = g.ensureLatestTag(r, pkg.GetURL(), pkgRefName) + if err != nil { + launchr.Term().Warning().Printfln("Couldn't check local tag, marking package %s(%s) as outdated, see debug for detailed error.", pkg.GetName(), pkgRefName) + launchr.Log().Debug("ensure tag error", "err", err) + return isLatest, nil + } + } + + if !isLatest { + launchr.Term().Info().Printfln("Pulling new changes from %s '%s' of %s package", pullTarget, pkgRefName, pkg.GetName()) + } + + return isLatest, nil +} + +func (g *gitDownloader) ensureLatestBranch(r *git.Repository, fetchURL, refName, remoteRefName string) (bool, error) { + refSpec := []config.RefSpec{config.RefSpec(fmt.Sprintf("refs/heads/%s:refs/heads/%s", refName, refName))} + err := g.fetchRemotes(r, fetchURL, refSpec) + if err != nil { + return false, err + } + + br, err := r.Branch(refName) + if err != nil { + return false, err + } + + localRef, err := r.Reference(plumbing.ReferenceName(br.Merge.String()), true) + if err != nil { + return false, err + } + + remote := filepath.Join("refs", "remotes", br.Remote, remoteRefName) + remoteRef, err := r.Reference(plumbing.ReferenceName(remote), false) + if err != nil { + return false, err + } + + return localRef.Hash() == remoteRef.Hash(), nil +} + +func (g *gitDownloader) ensureLatestTag(r *git.Repository, fetchURL, refName string) (bool, error) { + oldTag, err := r.Tag(refName) + if err != nil { + return false, err + } + + head, err := r.Head() + if err != nil { + return false, err + } + + refSpec := []config.RefSpec{config.RefSpec(fmt.Sprintf("refs/tags/%s:refs/tags/%s", refName, refName))} + err = g.fetchRemotes(r, fetchURL, refSpec) + if err != nil { + return false, err + } + + newTag, err := r.Tag(refName) + if err != nil { + return false, err + } + + if oldTag.Hash().String() != newTag.Hash().String() { + return false, err + } + revision := plumbing.Revision(newTag.Name().String()) + tagCommitHash, err := r.ResolveRevision(revision) + if err != nil { + return false, err + } + + commit, err := r.CommitObject(*tagCommitHash) + if err != nil { + return false, err + } + + return commit.ID() == head.Hash(), nil } // Download implements Downloader.Download interface -func (g *gitDownloader) Download(pkg *Package, targetDir string, kw *keyringWrapper) error { +func (g *gitDownloader) Download(pkg *Package, targetDir string) error { launchr.Term().Printfln("git fetch: %s", pkg.GetURL()) url := pkg.GetURL() @@ -27,17 +249,56 @@ func (g *gitDownloader) Download(pkg *Package, targetDir string, kw *keyringWrap return errNoURL } - options := &git.CloneOptions{ + ref := pkg.GetRef() + if ref == "" { + // Try to clone latest master branch. + err := g.tryDownload(targetDir, g.buildOptions(url)) + if err != nil { + return err + } + + return nil + } + + loaded := false + + // As we don't know if ref exists, iterate and try to clone both: tag and branch references. + refs := []plumbing.ReferenceName{plumbing.NewTagReferenceName(ref), plumbing.NewBranchReferenceName(ref)} + for _, r := range refs { + options := g.buildOptions(url) + options.ReferenceName = r + + err := g.tryDownload(targetDir, options) + if err != nil { + noMatchError := git.NoMatchingRefSpecError{} + if errors.Is(err, noMatchError) { + continue + } + + return err + } + + loaded = true + break + } + + if !loaded { + return fmt.Errorf("couldn't find remote ref %s", ref) + } + + return nil +} + +func (g *gitDownloader) buildOptions(url string) *git.CloneOptions { + return &git.CloneOptions{ URL: url, Progress: os.Stdout, SingleBranch: true, } - if pkg.GetRef() != "" { - options.ReferenceName = plumbing.NewBranchReferenceName(pkg.GetRef()) - } else if pkg.GetTag() != "" { - options.ReferenceName = plumbing.NewTagReferenceName(pkg.GetTag()) - } +} +func (g *gitDownloader) tryDownload(targetDir string, options *git.CloneOptions) error { + url := options.URL auths := []authorizationMode{authorisationNone, authorisationKeyring, authorisationManual} for _, authType := range auths { if authType == authorisationNone { @@ -53,7 +314,7 @@ func (g *gitDownloader) Download(pkg *Package, targetDir string, kw *keyringWrap } if authType == authorisationKeyring { - ci, err := kw.getForURL(url) + ci, err := g.k.getForURL(url) if err != nil { return err } @@ -66,7 +327,7 @@ func (g *gitDownloader) Download(pkg *Package, targetDir string, kw *keyringWrap _, err = git.PlainClone(targetDir, false, options) if err != nil { if errors.Is(err, transport.ErrAuthorizationFailed) || errors.Is(err, transport.ErrAuthenticationRequired) { - if kw.interactive { + if g.k.interactive { launchr.Term().Println("invalid auth, trying manual authorisation") continue } @@ -79,7 +340,7 @@ func (g *gitDownloader) Download(pkg *Package, targetDir string, kw *keyringWrap if authType == authorisationManual { ci := keyring.CredentialsItem{} ci.URL = url - ci, err := kw.fillCredentials(ci) + ci, err := g.k.fillCredentials(ci) if err != nil { return err } diff --git a/compose/http.go b/compose/http.go index b593dc6..bbae8f2 100644 --- a/compose/http.go +++ b/compose/http.go @@ -19,7 +19,7 @@ import ( var ( errInvalidFilepath = errors.New("invalid filepath") - errNoURL = errors.New("invalid url") + errNoURL = errors.New("invalid package url") errFailedClose = errors.New("failed to close stream") errRepositoryNotFound = errors.New("repository not found") errAuthenticationRequired = errors.New("authentication required") @@ -33,14 +33,25 @@ var ( rgxPathRoot = regexp.MustCompile(`^[^\/]*`) ) -type httpDownloader struct{} +type httpDownloader struct { + k *keyringWrapper +} + +func newHTTP(kw *keyringWrapper) Downloader { + return &httpDownloader{k: kw} +} -func newHTTP() Downloader { - return &httpDownloader{} +func (h *httpDownloader) EnsureLatest(_ *Package, downloadPath string) (bool, error) { + if _, err := os.Stat(downloadPath); !os.IsNotExist(err) { + // Skip download if package exists. + return true, nil + } + + return false, nil } // Download implements Downloader.Download interface -func (h *httpDownloader) Download(pkg *Package, targetDir string, kw *keyringWrapper) error { +func (h *httpDownloader) Download(pkg *Package, targetDir string) error { url := pkg.GetURL() name := rgxNameFromURL.FindString(url) if name == "" { @@ -92,7 +103,7 @@ func (h *httpDownloader) Download(pkg *Package, targetDir string, kw *keyringWra } if authType == authorisationKeyring { - ci, errGet := kw.getForURL(url) + ci, errGet := h.k.getForURL(url) if errGet != nil { return errGet } @@ -101,7 +112,7 @@ func (h *httpDownloader) Download(pkg *Package, targetDir string, kw *keyringWra resp, err = doRequest(client, req) if err != nil { if errors.Is(err, errAuthorizationFailed) { - if kw.interactive { + if h.k.interactive { launchr.Term().Println("invalid auth, trying manual authorisation") continue } @@ -115,7 +126,7 @@ func (h *httpDownloader) Download(pkg *Package, targetDir string, kw *keyringWra if authType == authorisationManual { ci := keyring.CredentialsItem{} ci.URL = url - ci, errFill := kw.fillCredentials(ci) + ci, errFill := h.k.fillCredentials(ci) if errFill != nil { return errFill } @@ -157,6 +168,8 @@ func (h *httpDownloader) Download(pkg *Package, targetDir string, kw *keyringWra } if archiveRootDir != "" { + defer os.Remove(fpath) + // rename root folder to package name return os.Rename( filepath.Join(targetDir, archiveRootDir), diff --git a/compose/yaml.go b/compose/yaml.go index d02531f..3d4461a 100644 --- a/compose/yaml.go +++ b/compose/yaml.go @@ -9,6 +9,11 @@ import ( "gopkg.in/yaml.v3" ) +const ( + // TargetLatest is fallback to latest master branch. + TargetLatest = "latest" +) + var ( composePermissions uint32 = 0644 ) @@ -87,22 +92,26 @@ func (p *Package) GetURL() string { // GetRef from package source func (p *Package) GetRef() string { - return p.Source.Ref + ref := p.Source.Ref + if ref == "" && p.GetTag() != "" { + ref = p.GetTag() + } + + return ref } -// GetTag from package source +// GetTag from package source. +// Deprecated: use [Package.GetRef] func (p *Package) GetTag() string { return p.Source.Tag } // GetTarget returns a target version of package func (p *Package) GetTarget() string { - target := "latest" - - if p.GetRef() != "" { - target = p.GetRef() - } else if p.GetTag() != "" { - target = p.GetTag() + target := TargetLatest + ref := p.GetRef() + if ref != "" { + target = ref } return target diff --git a/plugin.go b/plugin.go index 100febb..7b75fc6 100644 --- a/plugin.go +++ b/plugin.go @@ -143,7 +143,6 @@ func addPackageFlags(cmd *launchr.Command, dependency *compose.Dependency, strat cmd.Flags().StringVarP(&dependency.Name, "package", "", "", "Name of the package") compose.EnumVarP(cmd, &dependency.Source.Type, "type", "", compose.GitType, []string{compose.GitType, compose.HTTPType}, "Type of the package source: git, http") cmd.Flags().StringVarP(&dependency.Source.Ref, "ref", "", "", "Reference of the package source") - cmd.Flags().StringVarP(&dependency.Source.Tag, "tag", "", "", "Tag of the package source") cmd.Flags().StringVarP(&dependency.Source.URL, "url", "", "", "URL of the package source") cmd.Flags().StringSliceVarP(&strategies.Names, "strategy", "", []string{}, "Strategy name") @@ -151,23 +150,13 @@ func addPackageFlags(cmd *launchr.Command, dependency *compose.Dependency, strat } func packagePreRunValidate(cmd *launchr.Command, _ []string) error { - tagChanged := cmd.Flag("tag").Changed - refChanged := cmd.Flag("ref").Changed - if tagChanged && refChanged { - return errors.New("tag and ref cannot be used at the same time") - } - typeFlag, err := cmd.Flags().GetString("type") if err != nil { return err } if typeFlag == compose.HTTPType { - if tagChanged { - launchr.Term().Warning().Println("Tag can't be used with HTTP source") - err = cmd.Flags().Set("tag", "") - } - if refChanged { + if cmd.Flag("ref").Changed { launchr.Term().Warning().Println("Ref can't be used with HTTP source") err = cmd.Flags().Set("ref", "") } From c8f3b0a14f9c74844cf2df4a0ca0aa492943806a Mon Sep 17 00:00:00 2001 From: Igor Ignatyev Date: Tue, 24 Dec 2024 14:33:20 +0300 Subject: [PATCH 3/3] #55 handle empty package dir gracefully --- compose/builder.go | 178 ++++++++++++++++++++----------------- compose/compose.go | 64 ++++++++----- compose/downloadManager.go | 31 +++++-- compose/git.go | 16 ++-- compose/http.go | 3 +- 5 files changed, 174 insertions(+), 118 deletions(-) diff --git a/compose/builder.go b/compose/builder.go index 5e91f1c..221891c 100644 --- a/compose/builder.go +++ b/compose/builder.go @@ -1,6 +1,7 @@ package compose import ( + "context" "fmt" "io" "io/fs" @@ -150,7 +151,7 @@ func getVersionedMap(gitDir string) (map[string]bool, error) { return versionedFiles, err } -func (b *Builder) build() error { +func (b *Builder) build(ctx context.Context) error { launchr.Term().Println("Creating composition...") err := EnsureDirExists(b.targetDir) if err != nil { @@ -174,43 +175,48 @@ func (b *Builder) build() error { // @todo move to function err = fs.WalkDir(baseFs, ".", func(path string, d fs.DirEntry, err error) error { - if err != nil { - return err - } - - root := rgxPathRoot.FindString(path) - if _, ok := excludedFolders[root]; ok { - return nil - } + select { + case <-ctx.Done(): + return ctx.Err() + default: + if err != nil { + return err + } - if !d.IsDir() { - filename := filepath.Base(path) - if _, ok := excludedFiles[filename]; ok { + root := rgxPathRoot.FindString(path) + if _, ok := excludedFolders[root]; ok { return nil } - } - // Apply strategies that target local files - for _, localStrategy := range ls { - if localStrategy.s == removeExtraLocalFiles { - if ensureStrategyPrefixPath(path, localStrategy.paths) { + if !d.IsDir() { + filename := filepath.Base(path) + if _, ok := excludedFiles[filename]; ok { return nil } } - } - // Add .git folder into entriesTree whenever CheckVersioned or not - if checkVersioned && !strings.HasPrefix(path, gitPrefix) { - if _, ok := versionedMap[path]; !ok { - return nil + // Apply strategies that target local files + for _, localStrategy := range ls { + if localStrategy.s == removeExtraLocalFiles { + if ensureStrategyPrefixPath(path, localStrategy.paths) { + return nil + } + } } - } - finfo, _ := d.Info() - entry := &fsEntry{Prefix: b.platformDir, Path: path, Entry: finfo, Excluded: false, From: "domain repo"} - entriesTree = append(entriesTree, entry) - entriesMap[path] = entry - return nil + // Add .git folder into entriesTree whenever CheckVersioned or not + if checkVersioned && !strings.HasPrefix(path, gitPrefix) { + if _, ok := versionedMap[path]; !ok { + return nil + } + } + + finfo, _ := d.Info() + entry := &fsEntry{Prefix: b.platformDir, Path: path, Entry: finfo, Excluded: false, From: "domain repo"} + entriesTree = append(entriesTree, entry) + entriesMap[path] = entry + return nil + } }) if err != nil { @@ -226,72 +232,82 @@ func (b *Builder) build() error { } for i := 0; i < len(items); i++ { - pkgName := items[i] - if pkgName != DependencyRoot { - pkgPath := filepath.Join(b.sourceDir, pkgName, targetsMap[pkgName]) - packageFs := os.DirFS(pkgPath) - strategies, ok := ps[pkgName] - err = fs.WalkDir(packageFs, ".", func(path string, d fs.DirEntry, err error) error { - if err != nil { - return err - } + select { + case <-ctx.Done(): + return ctx.Err() + default: + pkgName := items[i] + if pkgName != DependencyRoot { + pkgPath := filepath.Join(b.sourceDir, pkgName, targetsMap[pkgName]) + packageFs := os.DirFS(pkgPath) + strategies, ok := ps[pkgName] + err = fs.WalkDir(packageFs, ".", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + + // Skip .git folder from packages + if strings.HasPrefix(path, gitPrefix) { + return nil + } + + var conflictReslv mergeConflictResolve + finfo, _ := d.Info() + entry := &fsEntry{Prefix: pkgPath, Path: path, Entry: finfo, Excluded: false, From: pkgName} + + if !ok { + // No strategies for package. Proceed with default merge. + entriesTree, conflictReslv = addEntries(entriesTree, entriesMap, entry, path) + } else { + entriesTree, conflictReslv = addStrategyEntries(strategies, entriesTree, entriesMap, entry, path) + } + + if b.logConflicts && !finfo.IsDir() { + logConflictResolve(conflictReslv, path, pkgName, entriesMap[path]) + } - // Skip .git folder from packages - if strings.HasPrefix(path, gitPrefix) { return nil - } - - var conflictReslv mergeConflictResolve - finfo, _ := d.Info() - entry := &fsEntry{Prefix: pkgPath, Path: path, Entry: finfo, Excluded: false, From: pkgName} - - if !ok { - // No strategies for package. Proceed with default merge. - entriesTree, conflictReslv = addEntries(entriesTree, entriesMap, entry, path) - } else { - entriesTree, conflictReslv = addStrategyEntries(strategies, entriesTree, entriesMap, entry, path) - } + }) - if b.logConflicts && !finfo.IsDir() { - logConflictResolve(conflictReslv, path, pkgName, entriesMap[path]) + if err != nil { + return err } - - return nil - }) - - if err != nil { - return err } } } // @todo check rsync for _, treeItem := range entriesTree { - sourcePath := filepath.Join(treeItem.Prefix, treeItem.Path) - destPath := filepath.Join(b.targetDir, treeItem.Path) - isSymlink := false - permissions := os.FileMode(dirPermissions) - - switch treeItem.Entry.Mode() & os.ModeType { - case os.ModeDir: - if err := createDir(destPath, treeItem.Entry.Mode()); err != nil { - return err - } - case os.ModeSymlink: - if err := lcopy(sourcePath, destPath); err != nil { - return err - } - isSymlink = true + select { + case <-ctx.Done(): + return ctx.Err() default: - permissions = treeItem.Entry.Mode() - if err := fcopy(sourcePath, destPath); err != nil { - return err + sourcePath := filepath.Join(treeItem.Prefix, treeItem.Path) + destPath := filepath.Join(b.targetDir, treeItem.Path) + isSymlink := false + permissions := os.FileMode(dirPermissions) + + switch treeItem.Entry.Mode() & os.ModeType { + case os.ModeDir: + if err := createDir(destPath, treeItem.Entry.Mode()); err != nil { + return err + } + case os.ModeSymlink: + if err := lcopy(sourcePath, destPath); err != nil { + return err + } + isSymlink = true + default: + permissions = treeItem.Entry.Mode() + if err := fcopy(sourcePath, destPath); err != nil { + return err + } } - } - if !isSymlink { - if err := os.Chmod(destPath, permissions); err != nil { - return err + if !isSymlink { + if err := os.Chmod(destPath, permissions); err != nil { + return err + } } } } diff --git a/compose/compose.go b/compose/compose.go index 6385a31..14d6e54 100644 --- a/compose/compose.go +++ b/compose/compose.go @@ -2,9 +2,12 @@ package compose import ( + "context" "errors" "os" + "os/signal" "path/filepath" + "syscall" "github.com/launchrctl/keyring" "github.com/launchrctl/launchr" @@ -107,30 +110,49 @@ func (kw *keyringWrapper) fillCredentials(ci keyring.CredentialsItem) (keyring.C // RunInstall on Composer func (c *Composer) RunInstall() error { - buildDir, packagesDir, err := c.prepareInstall() - if err != nil { - return err - } + ctx, cancel := context.WithCancel(context.Background()) - kw := &keyringWrapper{keyringService: c.getKeyring(), shouldUpdate: false, interactive: c.options.Interactive} - dm := CreateDownloadManager(kw) - packages, err := dm.Download(c.getCompose(), packagesDir) - if err != nil { - return err - } + signalChan := make(chan os.Signal, 1) + signal.Notify(signalChan, os.Interrupt, syscall.SIGTERM) - builder := createBuilder( - c.pwd, - buildDir, - packagesDir, - c.options.SkipNotVersioned, - c.options.ConflictsVerbosity, - packages, - ) - return builder.build() + go func() { + <-signalChan + launchr.Term().Printfln("\nTermination signal received. Cleaning up...") + // cleanup dir + _, _, _ = c.prepareInstall(false) + + cancel() + }() + + select { + case <-ctx.Done(): + return ctx.Err() + default: + buildDir, packagesDir, err := c.prepareInstall(c.options.Clean) + if err != nil { + return err + } + + kw := &keyringWrapper{keyringService: c.getKeyring(), shouldUpdate: false, interactive: c.options.Interactive} + dm := CreateDownloadManager(kw) + packages, err := dm.Download(ctx, c.getCompose(), packagesDir) + if err != nil { + return err + } + + builder := createBuilder( + c.pwd, + buildDir, + packagesDir, + c.options.SkipNotVersioned, + c.options.ConflictsVerbosity, + packages, + ) + return builder.build(ctx) + } } -func (c *Composer) prepareInstall() (string, string, error) { +func (c *Composer) prepareInstall(clean bool) (string, string, error) { buildPath := c.getPath(BuildDir) packagesPath := c.getPath(c.options.WorkingDir) @@ -140,7 +162,7 @@ func (c *Composer) prepareInstall() (string, string, error) { return "", "", err } - if c.options.Clean { + if clean { launchr.Term().Printfln("Cleaning packages dir: %s", packagesPath) err = os.RemoveAll(packagesPath) if err != nil { diff --git a/compose/downloadManager.go b/compose/downloadManager.go index d70887f..48e2441 100644 --- a/compose/downloadManager.go +++ b/compose/downloadManager.go @@ -1,6 +1,7 @@ package compose import ( + "context" "io" "os" "path/filepath" @@ -17,7 +18,7 @@ const ( // Downloader interface type Downloader interface { - Download(pkg *Package, targetDir string) error + Download(ctx context.Context, pkg *Package, targetDir string) error EnsureLatest(pkg *Package, downloadPath string) (bool, error) } @@ -47,7 +48,7 @@ func getDownloaderForPackage(downloadType string, kw *keyringWrapper) Downloader } // Download packages using compose file -func (m DownloadManager) Download(c *YamlCompose, targetDir string) ([]*Package, error) { +func (m DownloadManager) Download(ctx context.Context, c *YamlCompose, targetDir string) ([]*Package, error) { var packages []*Package //credentials := []keyring.CredentialsItem{} err := EnsureDirExists(targetDir) @@ -56,7 +57,7 @@ func (m DownloadManager) Download(c *YamlCompose, targetDir string) ([]*Package, } kw := m.getKeyring() - packages, err = m.recursiveDownload(c, kw, packages, nil, targetDir) + packages, err = m.recursiveDownload(ctx, c, kw, packages, nil, targetDir) if err != nil { return packages, err } @@ -69,7 +70,7 @@ func (m DownloadManager) Download(c *YamlCompose, targetDir string) ([]*Package, return packages, err } -func (m DownloadManager) recursiveDownload(yc *YamlCompose, kw *keyringWrapper, packages []*Package, parent *Package, targetDir string) ([]*Package, error) { +func (m DownloadManager) recursiveDownload(ctx context.Context, yc *YamlCompose, kw *keyringWrapper, packages []*Package, parent *Package, targetDir string) ([]*Package, error) { for _, d := range yc.Dependencies { // build package from dependency struct // add dependency if parent exists @@ -85,7 +86,7 @@ func (m DownloadManager) recursiveDownload(yc *YamlCompose, kw *keyringWrapper, packagePath := filepath.Join(targetDir, pkg.GetName(), pkg.GetTarget()) - err := downloadPackage(pkg, targetDir, kw) + err := downloadPackage(ctx, pkg, targetDir, kw) if err != nil { return packages, err } @@ -94,7 +95,7 @@ func (m DownloadManager) recursiveDownload(yc *YamlCompose, kw *keyringWrapper, if _, err = os.Stat(filepath.Join(packagePath, composeFile)); !os.IsNotExist(err) { cfg, err := Lookup(os.DirFS(packagePath)) if err == nil { - packages, err = m.recursiveDownload(cfg, kw, packages, pkg, targetDir) + packages, err = m.recursiveDownload(ctx, cfg, kw, packages, pkg, targetDir) if err != nil { return packages, err } @@ -107,7 +108,7 @@ func (m DownloadManager) recursiveDownload(yc *YamlCompose, kw *keyringWrapper, return packages, nil } -func downloadPackage(pkg *Package, targetDir string, kw *keyringWrapper) error { +func downloadPackage(ctx context.Context, pkg *Package, targetDir string, kw *keyringWrapper) error { downloader := getDownloaderForPackage(pkg.GetType(), kw) packagePath := filepath.Join(targetDir, pkg.GetName()) downloadPath := filepath.Join(packagePath, pkg.GetTarget()) @@ -132,7 +133,7 @@ func downloadPackage(pkg *Package, targetDir string, kw *keyringWrapper) error { downloadPath = packagePath } - err = downloader.Download(pkg, downloadPath) + err = downloader.Download(ctx, pkg, downloadPath) if err != nil { errRemove := os.RemoveAll(downloadPath) if errRemove != nil { @@ -156,5 +157,19 @@ func IsEmptyDir(name string) (bool, error) { return true, nil } + // Check if .git exists and nothing else + gitPath := filepath.Join(name, ".git") + if _, err = os.Stat(gitPath); err == nil { + // .git exists, now check if it's the only entry + entries, err := f.Readdirnames(2) // Read at most 2 entries + if err != nil { + return false, err + } + if len(entries) == 1 && entries[0] == ".git" { + return true, nil + } + } + + // Directory is not empty return false, err } diff --git a/compose/git.go b/compose/git.go index 7a0c756..e0689fd 100644 --- a/compose/git.go +++ b/compose/git.go @@ -1,6 +1,7 @@ package compose import ( + "context" "errors" "fmt" "os" @@ -241,7 +242,7 @@ func (g *gitDownloader) ensureLatestTag(r *git.Repository, fetchURL, refName str } // Download implements Downloader.Download interface -func (g *gitDownloader) Download(pkg *Package, targetDir string) error { +func (g *gitDownloader) Download(ctx context.Context, pkg *Package, targetDir string) error { launchr.Term().Printfln("git fetch: %s", pkg.GetURL()) url := pkg.GetURL() @@ -252,7 +253,7 @@ func (g *gitDownloader) Download(pkg *Package, targetDir string) error { ref := pkg.GetRef() if ref == "" { // Try to clone latest master branch. - err := g.tryDownload(targetDir, g.buildOptions(url)) + err := g.tryDownload(ctx, targetDir, g.buildOptions(url)) if err != nil { return err } @@ -268,7 +269,7 @@ func (g *gitDownloader) Download(pkg *Package, targetDir string) error { options := g.buildOptions(url) options.ReferenceName = r - err := g.tryDownload(targetDir, options) + err := g.tryDownload(ctx, targetDir, options) if err != nil { noMatchError := git.NoMatchingRefSpecError{} if errors.Is(err, noMatchError) { @@ -297,12 +298,13 @@ func (g *gitDownloader) buildOptions(url string) *git.CloneOptions { } } -func (g *gitDownloader) tryDownload(targetDir string, options *git.CloneOptions) error { +func (g *gitDownloader) tryDownload(ctx context.Context, targetDir string, options *git.CloneOptions) error { url := options.URL auths := []authorizationMode{authorisationNone, authorisationKeyring, authorisationManual} for _, authType := range auths { if authType == authorisationNone { - _, err := git.PlainClone(targetDir, false, options) + _, err := git.PlainCloneContext(ctx, targetDir, false, options) + launchr.Term().Println("") if err != nil { if errors.Is(err, transport.ErrAuthenticationRequired) { launchr.Term().Println("auth required, trying keyring authorisation") @@ -324,7 +326,7 @@ func (g *gitDownloader) tryDownload(targetDir string, options *git.CloneOptions) Password: ci.Password, } - _, err = git.PlainClone(targetDir, false, options) + _, err = git.PlainCloneContext(ctx, targetDir, false, options) if err != nil { if errors.Is(err, transport.ErrAuthorizationFailed) || errors.Is(err, transport.ErrAuthenticationRequired) { if g.k.interactive { @@ -350,7 +352,7 @@ func (g *gitDownloader) tryDownload(targetDir string, options *git.CloneOptions) Password: ci.Password, } - _, err = git.PlainClone(targetDir, false, options) + _, err = git.PlainCloneContext(ctx, targetDir, false, options) if err != nil { return err } diff --git a/compose/http.go b/compose/http.go index bbae8f2..128335e 100644 --- a/compose/http.go +++ b/compose/http.go @@ -4,6 +4,7 @@ import ( "archive/tar" "archive/zip" "compress/gzip" + "context" "errors" "fmt" "io" @@ -51,7 +52,7 @@ func (h *httpDownloader) EnsureLatest(_ *Package, downloadPath string) (bool, er } // Download implements Downloader.Download interface -func (h *httpDownloader) Download(pkg *Package, targetDir string) error { +func (h *httpDownloader) Download(_ context.Context, pkg *Package, targetDir string) error { url := pkg.GetURL() name := rgxNameFromURL.FindString(url) if name == "" {