Allow back ups of specific repositories

.envrc

@@ -0,0 +1 @@
+dotenv_if_exists  .env

.github/workflows/build.yml

@@ -9,7 +9,7 @@ on:
 
 jobs:
   build:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     strategy:
       matrix:
         include:
@@ -41,7 +41,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-go@v5
         with:
-          go-version: '^1.23.2'
+          go-version: "^1.24.0"
       - run: ${{ matrix.goopts }} go build -o ${{ matrix.filename }} -ldflags="-X 'main.Version=${GITHUB_REF##*/}' -X 'main.CommitHash=${GITHUB_SHA}' -X 'main.BuildTimestamp=$(date)'" ./cmd/git-backup
         env:
           GOOS: ${{ matrix.goos }}
@@ -59,7 +59,7 @@ jobs:
     needs:
       - build
     name: Build Docker
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
       - uses: actions/download-artifact@v4

.gitignore

@@ -2,3 +2,4 @@ backup/
 git-backup.yml
 .idea/
 git-backup
+.env

README.md

@@ -8,37 +8,45 @@ The latest version can be downloaded from the [releases page](https://github.com
 
 ## Configuration File
 
-Example yaml Configuration:
+The configuration file supports go templating to load secrets and other variables from the environment.
+An example `yaml`-Configuration looks like:
+
 ```yaml
 # The github section contains backup jobs for
 # GitHub and GitHub Enterprise
 github:
-    # (optional) The job name. This is used to
-    # create a subfolder in the backup folder. 
-    # (default: GitHub)
+  # (optional) The job name. This is used to
+  # create a subfolder in the backup folder.
+  # (default: GitHub)
   - job_name: github.com
+    # (optional) Set this url to connect to
+    # your self-hosted github install.
+    # (default: https://api.github.com)
+    url: https://github.mydomain.com
     # (required) The GitHub personal access
     # token. Create one with the scopes:
     # "read:org, repo"
     # https://github.com/settings/tokens/new?scopes=repo,read:org
-    access_token: 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
+    access_token: '{{ env "GITHUB_PAT" }}'
     # (optional) Back up repos you own.
     # (default: true)
     owned: true
     # (optional) Back up repos you starred.
     # (default: true)
     starred: true
-    # (optional) Back up repos on which you 
+    # (optional) Back up repos on which you
     # are a collaborator. (default: true)
     collaborator: true
-    # (optional) Back up repos owned by 
+    # (optional) Back up repos owned by
     # organisations of which you are a member.
     # (default: true)
     org_member: true
-    # (optional) Set this url to connect to
-    # your self-hosted github install.
-    # (default: https://api.github.com)
-    url: https://github.mydomain.com
+    # (optional) A list of repositories to backup.
+    # Will be merged with the other discovery mechanisms
+    # and it's result filtered with excluded repos.
+    # (default: [])
+    repositories:
+      - ChappIO/git-backup
     # (optional) Exclude this list of repos
     # or whole organizations/users
     exclude:
@@ -49,27 +57,34 @@ github:
 # GitLab.com and GitLab on premise
 gitlab:
   # (optional) The job name. This is used to
-  # create a subfolder in the backup folder. 
+  # create a subfolder in the backup folder.
   # (default: GitLab)
   - job_name: gitlab.com
+    # (optional) Set this url to connect to
+    # your self-hosted gitlab install.
+    # (default: https://gitlab.com/)
+    url: https://gitlab.mydomain.com
     # (required) The GitLab access token.
     # Create one with the scopes: "api"
     # https://gitlab.com/-/profile/personal_access_tokens?scopes=api&name=git-backup
-    access_token: glpat-6t78yuihy789uy8t768
+    access_token: '{{ env "GITLAB_PAT" }}'
     # (optional) Back up repos you own.
     # (default: true)
     owned: true
     # (optional) Back up repos you starred.
     # (default: true)
     starred: true
-    # (optional) Back up repos owned by 
+    # (optional) Back up repos owned by
     # teams of which you are a member.
     # (default: true)
     member: true
-    # (optional) Set this url to connect to
-    # your self-hosted gitlab install.
-    # (default: https://gitlab.com/)
-    url: https://gitlab.mydomain.com
+    # (optional) A list of repositories to backup.
+    # Will be merged with the other discovery mechanisms
+    # and it's result filtered with excluded repos.
+    # (default: [])
+    repositories:
+      - gnuwget/wget2
+      - gitlab-org/api/client-go
     # (optional) Exclude this list of repos
     # or whole organizations/users
     exclude:
@@ -113,8 +128,8 @@ docker run -v /path/to/backups:/backups ghcr.io/chappio/git-backup:1
 You can specify several parameters when starting this container.
 
 | **Parameter**                  | **Description**                                                                        |
-|--------------------------------|----------------------------------------------------------------------------------------|
+| ------------------------------ | -------------------------------------------------------------------------------------- |
 | `-v /path/to/backups:/backups` | Mount the folder where you want to store your backups and read you configuration file. |
 | `-e TZ=Europe/Amsterdam`       | Set the timezone used for logging.                                                     |
 | `-e PUID=0`                    | Set the user id of the unix user who will own the backup files in /backup.             |
-| `-e PGID=0`                    | Set the group id of the unix user's group who will own the backup files.               | 
+| `-e PGID=0`                    | Set the group id of the unix user's group who will own the backup files.               |

cmd/git-backup/main.go

@@ -59,7 +59,7 @@ func main() {
 			os.Exit(100)
 		}
 		for _, repo := range repos {
-			log.Printf("Discovered %s", repo.FullName)
+			log.Printf("Backing up %s", repo.FullName)
 			targetPath := filepath.Join(*targetPath, sourceName, repo.FullName)
 			err := os.MkdirAll(targetPath, os.ModePerm)
 			if err != nil {

config.go

@@ -1,9 +1,12 @@
 package git_backup
 
 import (
-	"gopkg.in/yaml.v3"
+	"bytes"
 	"io"
 	"os"
+	"text/template"
+
+	"gopkg.in/yaml.v3"
 )
 
 type Config struct {
@@ -53,9 +56,37 @@ func LoadFile(path string) (out Config, err error) {
 }
 
 func LoadReader(reader io.Reader) (out Config, err error) {
-	dec := yaml.NewDecoder(reader)
+	data, err := io.ReadAll(reader)
+	if err != nil {
+		return
+	}
+	rendered, err := parse(string(data))
+	if err != nil {
+		return
+	}
+
+	dec := yaml.NewDecoder(rendered)
 	dec.KnownFields(true)
 	err = dec.Decode(&out)
 	out.setDefaults()
 	return
 }
+
+func parse(rawTemplate string) (rendered io.Reader, err error) {
+	fmap := template.FuncMap{
+		"env": os.Getenv,
+	}
+
+	tmpl := template.New("").Funcs(fmap).Option("missingkey=error")
+
+	tmpl, err = tmpl.Parse(rawTemplate)
+	if err != nil {
+		return
+	}
+
+	buf := &bytes.Buffer{}
+	err = tmpl.Execute(buf, nil)
+	rendered = bytes.NewReader(buf.Bytes())
+
+	return
+}

github.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"log"
+	"maps"
 	"net/url"
 	"slices"
 	"strings"
@@ -21,6 +22,7 @@ type GithubConfig struct {
 	Collaborator *bool    `yaml:"collaborator,omitempty"`
 	Owned        *bool    `yaml:"owned,omitempty"`
 	Exclude      []string `yaml:"exclude,omitempty"`
+	Repositories []string `yaml:"repositories,omitempty"`
 	client       *github.Client
 }
 
@@ -44,34 +46,22 @@ func (c *GithubConfig) ListRepositories() ([]*Repository, error) {
 	}
 	out := make([]*Repository, 0, len(repos))
 	for _, repo := range repos {
+		if isExcluded(c.Exclude, *repo.FullName) {
+			log.Printf("Skipping excluded repository: %s", *repo.FullName)
+			continue
+		}
+
 		gitUrl, err := url.Parse(*repo.CloneURL)
 		if err != nil {
 			return out, err
 		}
 		gitUrl.User = url.UserPassword("github", c.AccessToken)
 
-		isExcluded := slices.ContainsFunc(c.Exclude, func(s string) bool {
-			if strings.EqualFold(s, *repo.FullName) {
-				return true
-			}
-
-			if strings.Contains(s, "/") {
-				return false
-			}
-
-			repoFullName := *repo.FullName
-
-			repoOwner := repoFullName[:strings.Index(repoFullName, "/")]
-			return strings.EqualFold(s, repoOwner)
+		out = append(out, &Repository{
+			FullName: *repo.FullName,
+			GitURL:   *gitUrl,
 		})
-		if isExcluded {
-			log.Printf("Skipping excluded repository: %s", *repo.FullName)
-		} else {
-			out = append(out, &Repository{
-				FullName: *repo.FullName,
-				GitURL:   *gitUrl,
-			})
-		}
+
 	}
 	return out, nil
 }
@@ -110,23 +100,38 @@ func (c *GithubConfig) getMe() (*github.User, error) {
 }
 
 func (c *GithubConfig) getAllRepos() ([]*github.Repository, error) {
-	all := make([]*github.Repository, 0)
+	all := make(map[string]*github.Repository, 0)
+
+	// fetch the configured repos
+	if len(c.Repositories) > 0 {
+		if repos, err := c.getRepoList(c.Repositories); err != nil {
+			return nil, err
+		} else {
+			for _, repo := range repos {
+				all[*repo.FullName] = repo
+			}
+		}
+	}
+
+	// use discovery mechanisms
 	var err error
 
-	for repos, response, apiErr := c.getRepos(1); true; repos, response, apiErr = c.getRepos(response.NextPage) {
+	for repos, response, apiErr := c.discoverRepos(1); true; repos, response, apiErr = c.discoverRepos(response.NextPage) {
 		if apiErr != nil {
 			err = apiErr
 			break
 		} else {
-			all = append(all, repos...)
+			for _, repo := range repos {
+				all[*repo.FullName] = repo
+			}
 		}
 
 		if len(repos) == 0 || response.NextPage == 0 {
 			break
 		}
 	}
 	if err != nil {
-		return all, err
+		return nil, err
 	}
 
 	if *c.Starred {
@@ -135,7 +140,9 @@ func (c *GithubConfig) getAllRepos() ([]*github.Repository, error) {
 				err = apiErr
 				break
 			} else {
-				all = append(all, repos...)
+				for _, repo := range repos {
+					all[*repo.FullName] = repo
+				}
 			}
 
 			if len(repos) == 0 || response.NextPage == 0 {
@@ -144,10 +151,31 @@ func (c *GithubConfig) getAllRepos() ([]*github.Repository, error) {
 		}
 	}
 
-	return all, err
+	return slices.Collect(maps.Values(all)), err
+}
+
+func (c *GithubConfig) getRepoList(repos []string) ([]*github.Repository, error) {
+
+	ghRepos := make([]*github.Repository, 0, len(repos))
+
+	for _, repo := range repos {
+		parts := strings.Split(repo, "/")
+
+		if len(parts) != 2 {
+			return nil, fmt.Errorf("invalid repository name '%v' must be of schema owner/repo", repo)
+		}
+
+		ghRepo, _, err := c.client.Repositories.Get(context.Background(), parts[0], parts[1])
+		if err != nil {
+			return nil, err
+		}
+		ghRepos = append(ghRepos, ghRepo)
+	}
+
+	return ghRepos, nil
 }
 
-func (c *GithubConfig) getRepos(page int) ([]*github.Repository, *github.Response, error) {
+func (c *GithubConfig) discoverRepos(page int) ([]*github.Repository, *github.Response, error) {
 	affiliations := make([]string, 0)
 
 	if *c.Owned {