From 2e003351af2631c5af512735daf9d0b8d4c4a2df Mon Sep 17 00:00:00 2001 From: Massaki Archambault Date: Wed, 24 Dec 2025 13:01:55 -0500 Subject: [PATCH 1/8] refactor to have all forge share a common cache implementation --- .vscode/launch.json | 3 +- cache/cache.go | 93 +++++++++++++++ config/loader.go | 4 +- config/loader_test.go | 10 +- forges/gitea/client.go | 75 +++++------- forges/gitea/organization.go | 96 +++++----------- forges/gitea/repository.go | 7 +- forges/gitea/user.go | 96 +++++----------- forges/github/client.go | 74 +++++------- forges/github/organization.go | 102 ++++++----------- forges/github/repository.go | 7 +- forges/github/user.go | 102 ++++++----------- forges/gitlab/client.go | 81 +++++-------- forges/gitlab/group.go | 207 +++++++++++----------------------- forges/gitlab/project.go | 8 +- forges/gitlab/user.go | 106 ++++++----------- fstree/group.go | 24 ++-- fstree/refresh.go | 8 +- fstree/repository.go | 12 +- fstree/root.go | 10 +- git/client.go | 4 +- main.go | 10 +- types/types.go | 27 +++++ 23 files changed, 483 insertions(+), 683 deletions(-) create mode 100644 cache/cache.go create mode 100644 types/types.go diff --git a/.vscode/launch.json b/.vscode/launch.json index 5500823..1e3f28e 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -10,8 +10,9 @@ "request": "launch", "mode": "debug", "program": "${workspaceRoot}", + "buildFlags": "-race", "args": [ - "-debug", + // "-debug", "-config", "${workspaceRoot}/config.yaml" ] diff --git a/cache/cache.go b/cache/cache.go new file mode 100644 index 0000000..b4001f3 --- /dev/null +++ b/cache/cache.go @@ -0,0 +1,93 @@ +package cache + +import ( + "context" + "sync" + "time" + + "github.com/badjware/gitforgefs/types" +) + +type Cache struct { + backend types.GitForge + + rootContentLock sync.RWMutex + cachedRootContent map[string]types.GroupSource + + contentLock sync.RWMutex + cachedContent map[string]CachedContent +} + +func NewForgeCache(backend types.GitForge) types.GitForge { + return &Cache{ + backend: backend, + + cachedContent: map[string]CachedContent{}, + } +} + +type CachedContent struct { + types.GroupContent + creationTime time.Time +} + +func (c *Cache) FetchRootGroupContent(ctx context.Context) (map[string]types.GroupSource, error) { + c.rootContentLock.RLock() + if c.cachedRootContent == nil { + c.rootContentLock.RUnlock() + + // acquire write lock + c.rootContentLock.Lock() + defer c.rootContentLock.Unlock() + + // check to make sure the data is still not there, + // since RWMutex is not upgradeable and another thread may have grabbed the lock in the meantime + if c.cachedRootContent == nil { + content, err := c.backend.FetchRootGroupContent(ctx) + if err != nil { + return nil, err + } + c.cachedRootContent = content + } + return c.cachedRootContent, nil + } + c.rootContentLock.RUnlock() + return c.cachedRootContent, nil +} + +func (c *Cache) FetchGroupContent(ctx context.Context, source types.GroupSource) (types.GroupContent, error) { + c.contentLock.RLock() + if cachedContent, found := c.cachedContent[source.GetGroupPath()]; !found { + c.contentLock.RUnlock() + + // acquire write lock + c.contentLock.Lock() + defer c.contentLock.Unlock() + + // read the map again to make sure the data is still not there + if cachedContent, found := c.cachedContent[source.GetGroupPath()]; found { + return cachedContent.GroupContent, nil + } + + // fetch content from backend and cache it + content, err := c.backend.FetchGroupContent(ctx, source) + if err != nil { + return types.GroupContent{}, err + } + c.cachedContent[source.GetGroupPath()] = CachedContent{ + GroupContent: content, + creationTime: time.Now(), + } + return content, nil + } else { + c.contentLock.RUnlock() + return cachedContent.GroupContent, nil + } +} + +func (c *Cache) InvalidateCache(path string) { + c.contentLock.Lock() + defer c.contentLock.Unlock() + + delete(c.cachedContent, path) +} diff --git a/config/loader.go b/config/loader.go index 456e8d8..02d0b6b 100644 --- a/config/loader.go +++ b/config/loader.go @@ -39,7 +39,7 @@ type ( URL string `yaml:"url,omitempty"` Token string `yaml:"token,omitempty"` - GroupIDs []int `yaml:"group_ids,omitempty"` + GroupIDs []uint64 `yaml:"group_ids,omitempty"` UserNames []string `yaml:"user_names,omitempty"` ArchivedProjectHandling string `yaml:"archived_project_handling,omitempty"` @@ -97,7 +97,7 @@ func LoadConfig(configPath string) (*Config, error) { URL: "https://gitlab.com", Token: "", PullMethod: "http", - GroupIDs: []int{9970}, + GroupIDs: []uint64{9970}, UserNames: []string{}, ArchivedProjectHandling: "hide", IncludeCurrentUser: true, diff --git a/config/loader_test.go b/config/loader_test.go index cba0a03..0f4defa 100644 --- a/config/loader_test.go +++ b/config/loader_test.go @@ -24,7 +24,7 @@ func TestLoadConfig(t *testing.T) { URL: "https://example.com", Token: "12345", PullMethod: "ssh", - GroupIDs: []int{123}, + GroupIDs: []uint64{123}, UserNames: []string{"test-user"}, ArchivedProjectHandling: "hide", IncludeCurrentUser: true, @@ -147,7 +147,7 @@ func TestMakeGitlabConfig(t *testing.T) { URL: "https://gitlab.com", PullMethod: "http", Token: "", - GroupIDs: []int{9970}, + GroupIDs: []uint64{9970}, UserNames: []string{}, ArchivedProjectHandling: "hide", IncludeCurrentUser: true, @@ -157,7 +157,7 @@ func TestMakeGitlabConfig(t *testing.T) { URL: "https://gitlab.com", PullMethod: "http", Token: "", - GroupIDs: []int{9970}, + GroupIDs: []uint64{9970}, UserNames: []string{}, ArchivedProjectHandling: "hide", IncludeCurrentUser: true, @@ -172,7 +172,7 @@ func TestMakeGitlabConfig(t *testing.T) { URL: "https://gitlab.com", PullMethod: "invalid", Token: "", - GroupIDs: []int{9970}, + GroupIDs: []uint64{9970}, UserNames: []string{}, ArchivedProjectHandling: "hide", IncludeCurrentUser: true, @@ -186,7 +186,7 @@ func TestMakeGitlabConfig(t *testing.T) { URL: "https://gitlab.com", PullMethod: "http", Token: "", - GroupIDs: []int{9970}, + GroupIDs: []uint64{9970}, UserNames: []string{}, IncludeCurrentUser: true, ArchivedProjectHandling: "invalid", diff --git a/forges/gitea/client.go b/forges/gitea/client.go index b568579..1b40645 100644 --- a/forges/gitea/client.go +++ b/forges/gitea/client.go @@ -4,11 +4,10 @@ import ( "context" "fmt" "log/slog" - "sync" "code.gitea.io/sdk/gitea" "github.com/badjware/gitforgefs/config" - "github.com/badjware/gitforgefs/fstree" + "github.com/badjware/gitforgefs/types" ) type giteaClient struct { @@ -17,15 +16,8 @@ type giteaClient struct { logger *slog.Logger - rootContent map[string]fstree.GroupSource - - // API response cache - organizationCacheMux sync.RWMutex - organizationNameToIDMap map[string]int64 - organizationCache map[int64]*Organization - userCacheMux sync.RWMutex - userNameToIDMap map[string]int64 - userCache map[int64]*User + // use a map without values for efficient lookups + users map[string]struct{} } func NewClient(logger *slog.Logger, config config.GiteaClientConfig) (*giteaClient, error) { @@ -40,12 +32,7 @@ func NewClient(logger *slog.Logger, config config.GiteaClientConfig) (*giteaClie logger: logger, - rootContent: nil, - - organizationNameToIDMap: map[string]int64{}, - organizationCache: map[int64]*Organization{}, - userNameToIDMap: map[string]int64{}, - userCache: map[int64]*User{}, + users: make(map[string]struct{}), } // Fetch current user and add it to the list @@ -53,45 +40,41 @@ func NewClient(logger *slog.Logger, config config.GiteaClientConfig) (*giteaClie if err != nil { logger.Warn("failed to fetch the current user:", "error", err.Error()) } else { - giteaClient.UserNames = append(giteaClient.UserNames, *¤tUser.UserName) + giteaClient.UserNames = append(giteaClient.UserNames, currentUser.UserName) } return giteaClient, nil } -func (c *giteaClient) FetchRootGroupContent(ctx context.Context) (map[string]fstree.GroupSource, error) { - if c.rootContent == nil { - rootContent := make(map[string]fstree.GroupSource) - - for _, orgName := range c.GiteaClientConfig.OrgNames { - org, err := c.fetchOrganization(ctx, orgName) - if err != nil { - c.logger.Warn(err.Error()) - } else { - rootContent[org.Name] = org - } - } +func (c *giteaClient) FetchRootGroupContent(ctx context.Context) (map[string]types.GroupSource, error) { + rootContent := make(map[string]types.GroupSource) - for _, userName := range c.GiteaClientConfig.UserNames { - user, err := c.fetchUser(ctx, userName) - if err != nil { - c.logger.Warn(err.Error()) - } else { - rootContent[user.Name] = user - } + for _, orgName := range c.GiteaClientConfig.OrgNames { + org, err := c.fetchOrganization(ctx, orgName) + if err != nil { + c.logger.Warn(err.Error()) + } else { + rootContent[org.Name] = org } + } - c.rootContent = rootContent + for _, userName := range c.GiteaClientConfig.UserNames { + user, err := c.fetchUser(ctx, userName) + if err != nil { + c.logger.Warn(err.Error()) + } else { + rootContent[user.Name] = user + c.users[user.Name] = struct{}{} + } } - return c.rootContent, nil + + return rootContent, nil } -func (c *giteaClient) FetchGroupContent(ctx context.Context, gid uint64) (map[string]fstree.GroupSource, map[string]fstree.RepositorySource, error) { - if org, found := c.organizationCache[int64(gid)]; found { - return c.fetchOrganizationContent(ctx, org) - } - if user, found := c.userCache[int64(gid)]; found { - return c.fetchUserContent(ctx, user) +func (c *giteaClient) FetchGroupContent(ctx context.Context, source types.GroupSource) (types.GroupContent, error) { + if _, found := c.users[source.GetGroupPath()]; found { + return c.fetchUserContent(ctx, source.GetGroupPath()) + } else { + return c.fetchOrganizationContent(ctx, source.GetGroupPath()) } - return nil, nil, fmt.Errorf("invalid gid: %v", gid) } diff --git a/forges/gitea/organization.go b/forges/gitea/organization.go index e0f5b65..63ed10f 100644 --- a/forges/gitea/organization.go +++ b/forges/gitea/organization.go @@ -3,51 +3,25 @@ package gitea import ( "context" "fmt" - "sync" "code.gitea.io/sdk/gitea" - "github.com/badjware/gitforgefs/fstree" + "github.com/badjware/gitforgefs/types" ) type Organization struct { ID int64 Name string - - mux sync.Mutex - - // hold org content - childRepositories map[string]fstree.RepositorySource } func (o *Organization) GetGroupID() uint64 { return uint64(o.ID) } -func (o *Organization) InvalidateContentCache() { - o.mux.Lock() - defer o.mux.Unlock() - - // clear child repositories from cache - o.childRepositories = nil +func (o *Organization) GetGroupPath() string { + return o.Name } func (c *giteaClient) fetchOrganization(ctx context.Context, orgName string) (*Organization, error) { - c.organizationCacheMux.RLock() - cachedId, found := c.organizationNameToIDMap[orgName] - if found { - cachedOrg := c.organizationCache[cachedId] - c.organizationCacheMux.RUnlock() - - // if found in cache, return the cached reference - c.logger.Debug("Organization cache hit", "org_name", orgName) - return cachedOrg, nil - } else { - c.organizationCacheMux.RUnlock() - - c.logger.Debug("Organization cache miss", "org_name", orgName) - } - - // If not found in cache, fetch organization infos from API giteaOrg, _, err := c.client.GetOrg(orgName) if err != nil { return nil, fmt.Errorf("failed to fetch organization with name %v: %v", orgName, err) @@ -55,51 +29,43 @@ func (c *giteaClient) fetchOrganization(ctx context.Context, orgName string) (*O newOrg := Organization{ ID: giteaOrg.ID, Name: giteaOrg.UserName, - - childRepositories: nil, } - // save in cache - c.organizationCacheMux.Lock() - c.organizationCache[newOrg.ID] = &newOrg - c.organizationNameToIDMap[newOrg.Name] = newOrg.ID - c.organizationCacheMux.Unlock() - return &newOrg, nil } -func (c *giteaClient) fetchOrganizationContent(ctx context.Context, org *Organization) (map[string]fstree.GroupSource, map[string]fstree.RepositorySource, error) { - org.mux.Lock() - defer org.mux.Unlock() +func (c *giteaClient) fetchOrganizationContent(ctx context.Context, orgName string) (types.GroupContent, error) { + org, err := c.fetchOrganization(ctx, orgName) + if err != nil { + return types.GroupContent{}, err + } - // Get cached data if available - // TODO: cache cache invalidation? - if org.childRepositories == nil { - childRepositories := make(map[string]fstree.RepositorySource) + repositories := make(map[string]types.RepositorySource) - // Fetch the organization repositories - listReposOptions := gitea.ListReposOptions{ - ListOptions: gitea.ListOptions{PageSize: 100}, + // Fetch the organization repositories + listReposOptions := gitea.ListReposOptions{ + ListOptions: gitea.ListOptions{PageSize: 100}, + } + for { + giteaRepositories, response, err := c.client.ListOrgRepos(org.Name, gitea.ListOrgReposOptions(listReposOptions)) + if err != nil { + return types.GroupContent{}, fmt.Errorf("failed to fetch repository in gitea: %v", err) } - for { - giteaRepositories, response, err := c.client.ListOrgRepos(org.Name, gitea.ListOrgReposOptions(listReposOptions)) - if err != nil { - return nil, nil, fmt.Errorf("failed to fetch repository in gitea: %v", err) - } - for _, giteaRepository := range giteaRepositories { - repository := c.newRepositoryFromGiteaRepository(ctx, giteaRepository) - if repository != nil { - childRepositories[repository.Path] = repository - } + for _, giteaRepository := range giteaRepositories { + repository := c.newRepositoryFromGiteaRepository(giteaRepository) + if repository != nil { + repositories[repository.GetRepositoryName()] = repository } - if response.NextPage == 0 { - break - } - // Get the next page - listReposOptions.Page = response.NextPage } - - org.childRepositories = childRepositories + if response.NextPage == 0 { + break + } + // Get the next page + listReposOptions.Page = response.NextPage } - return make(map[string]fstree.GroupSource), org.childRepositories, nil + + return types.GroupContent{ + Groups: make(map[string]types.GroupSource), + Repositories: repositories, + }, nil } diff --git a/forges/gitea/repository.go b/forges/gitea/repository.go index d2c6cec..790324f 100644 --- a/forges/gitea/repository.go +++ b/forges/gitea/repository.go @@ -1,7 +1,6 @@ package gitea import ( - "context" "path" "code.gitea.io/sdk/gitea" @@ -19,6 +18,10 @@ func (r *Repository) GetRepositoryID() uint64 { return uint64(r.ID) } +func (r *Repository) GetRepositoryName() string { + return r.Path +} + func (r *Repository) GetCloneURL() string { return r.CloneURL } @@ -27,7 +30,7 @@ func (r *Repository) GetDefaultBranch() string { return r.DefaultBranch } -func (c *giteaClient) newRepositoryFromGiteaRepository(ctx context.Context, repository *gitea.Repository) *Repository { +func (c *giteaClient) newRepositoryFromGiteaRepository(repository *gitea.Repository) *Repository { if c.ArchivedRepoHandling == config.ArchivedProjectIgnore && repository.Archived { return nil } diff --git a/forges/gitea/user.go b/forges/gitea/user.go index d392355..dea98f3 100644 --- a/forges/gitea/user.go +++ b/forges/gitea/user.go @@ -3,51 +3,25 @@ package gitea import ( "context" "fmt" - "sync" "code.gitea.io/sdk/gitea" - "github.com/badjware/gitforgefs/fstree" + "github.com/badjware/gitforgefs/types" ) type User struct { ID int64 Name string - - mux sync.Mutex - - // hold user content - childRepositories map[string]fstree.RepositorySource } func (u *User) GetGroupID() uint64 { return uint64(u.ID) } -func (u *User) InvalidateContentCache() { - u.mux.Lock() - defer u.mux.Unlock() - - // clear child repositories from cache - u.childRepositories = nil +func (u *User) GetGroupPath() string { + return u.Name } func (c *giteaClient) fetchUser(ctx context.Context, userName string) (*User, error) { - c.userCacheMux.RLock() - cachedId, found := c.userNameToIDMap[userName] - if found { - cachedUser := c.userCache[cachedId] - c.userCacheMux.RUnlock() - - // if found in cache, return the cached reference - c.logger.Debug("User cache hit", "user_name", userName) - return cachedUser, nil - } else { - c.userCacheMux.RUnlock() - - c.logger.Debug("User cache miss", "user_name", userName) - } - - // If not found in cache, fetch user infos from API giteaUser, _, err := c.client.GetUserInfo(userName) if err != nil { return nil, fmt.Errorf("failed to fetch user with name %v: %v", userName, err) @@ -55,51 +29,43 @@ func (c *giteaClient) fetchUser(ctx context.Context, userName string) (*User, er newUser := User{ ID: giteaUser.ID, Name: giteaUser.UserName, - - childRepositories: nil, } - // save in cache - c.userCacheMux.Lock() - c.userCache[newUser.ID] = &newUser - c.userNameToIDMap[newUser.Name] = newUser.ID - c.userCacheMux.Unlock() - return &newUser, nil } -func (c *giteaClient) fetchUserContent(ctx context.Context, user *User) (map[string]fstree.GroupSource, map[string]fstree.RepositorySource, error) { - user.mux.Lock() - defer user.mux.Unlock() +func (c *giteaClient) fetchUserContent(ctx context.Context, userName string) (types.GroupContent, error) { + user, err := c.fetchUser(ctx, userName) + if err != nil { + return types.GroupContent{}, err + } - // Get cached data if available - // TODO: cache cache invalidation? - if user.childRepositories == nil { - childRepositories := make(map[string]fstree.RepositorySource) + repositories := make(map[string]types.RepositorySource) - // Fetch the user repositories - listReposOptions := gitea.ListReposOptions{ - ListOptions: gitea.ListOptions{PageSize: 100}, + // Fetch the user repositories + listReposOptions := gitea.ListReposOptions{ + ListOptions: gitea.ListOptions{PageSize: 100}, + } + for { + giteaRepositories, response, err := c.client.ListUserRepos(user.Name, listReposOptions) + if err != nil { + return types.GroupContent{}, fmt.Errorf("failed to fetch repository in gitea: %v", err) } - for { - giteaRepositories, response, err := c.client.ListUserRepos(user.Name, listReposOptions) - if err != nil { - return nil, nil, fmt.Errorf("failed to fetch repository in gitea: %v", err) - } - for _, giteaRepository := range giteaRepositories { - repository := c.newRepositoryFromGiteaRepository(ctx, giteaRepository) - if repository != nil { - childRepositories[repository.Path] = repository - } + for _, giteaRepository := range giteaRepositories { + repository := c.newRepositoryFromGiteaRepository(giteaRepository) + if repository != nil { + repositories[repository.Path] = repository } - if response.NextPage == 0 { - break - } - // Get the next page - listReposOptions.Page = response.NextPage } - - user.childRepositories = childRepositories + if response.NextPage == 0 { + break + } + // Get the next page + listReposOptions.Page = response.NextPage } - return make(map[string]fstree.GroupSource), user.childRepositories, nil + + return types.GroupContent{ + Groups: make(map[string]types.GroupSource), + Repositories: repositories, + }, nil } diff --git a/forges/github/client.go b/forges/github/client.go index a95827f..498967a 100644 --- a/forges/github/client.go +++ b/forges/github/client.go @@ -2,12 +2,10 @@ package github import ( "context" - "fmt" "log/slog" - "sync" "github.com/badjware/gitforgefs/config" - "github.com/badjware/gitforgefs/fstree" + "github.com/badjware/gitforgefs/types" "github.com/google/go-github/v63/github" ) @@ -17,15 +15,8 @@ type githubClient struct { logger *slog.Logger - rootContent map[string]fstree.GroupSource - - // API response cache - organizationCacheMux sync.RWMutex - organizationNameToIDMap map[string]int64 - organizationCache map[int64]*Organization - userCacheMux sync.RWMutex - userNameToIDMap map[string]int64 - userCache map[int64]*User + // use a map without values for efficient lookups + users map[string]struct{} } func NewClient(logger *slog.Logger, config config.GithubClientConfig) (*githubClient, error) { @@ -40,12 +31,7 @@ func NewClient(logger *slog.Logger, config config.GithubClientConfig) (*githubCl logger: logger, - rootContent: nil, - - organizationNameToIDMap: map[string]int64{}, - organizationCache: map[int64]*Organization{}, - userNameToIDMap: map[string]int64{}, - userCache: map[int64]*User{}, + users: make(map[string]struct{}), } // Fetch current user and add it to the list @@ -59,39 +45,35 @@ func NewClient(logger *slog.Logger, config config.GithubClientConfig) (*githubCl return gitHubClient, nil } -func (c *githubClient) FetchRootGroupContent(ctx context.Context) (map[string]fstree.GroupSource, error) { - if c.rootContent == nil { - rootContent := make(map[string]fstree.GroupSource) - - for _, orgName := range c.GithubClientConfig.OrgNames { - org, err := c.fetchOrganization(ctx, orgName) - if err != nil { - c.logger.Warn(err.Error()) - } else { - rootContent[org.Name] = org - } - } +func (c *githubClient) FetchRootGroupContent(ctx context.Context) (map[string]types.GroupSource, error) { + rootContent := make(map[string]types.GroupSource) - for _, userName := range c.GithubClientConfig.UserNames { - user, err := c.fetchUser(ctx, userName) - if err != nil { - c.logger.Warn(err.Error()) - } else { - rootContent[user.Name] = user - } + for _, orgName := range c.GithubClientConfig.OrgNames { + org, err := c.fetchOrganization(ctx, orgName) + if err != nil { + c.logger.Warn(err.Error()) + } else { + rootContent[org.Name] = org } + } - c.rootContent = rootContent + for _, userName := range c.GithubClientConfig.UserNames { + user, err := c.fetchUser(ctx, userName) + if err != nil { + c.logger.Warn(err.Error()) + } else { + rootContent[user.Name] = user + c.users[user.Name] = struct{}{} + } } - return c.rootContent, nil + + return rootContent, nil } -func (c *githubClient) FetchGroupContent(ctx context.Context, gid uint64) (map[string]fstree.GroupSource, map[string]fstree.RepositorySource, error) { - if org, found := c.organizationCache[int64(gid)]; found { - return c.fetchOrganizationContent(ctx, org) - } - if user, found := c.userCache[int64(gid)]; found { - return c.fetchUserContent(ctx, user) +func (c *githubClient) FetchGroupContent(ctx context.Context, source types.GroupSource) (types.GroupContent, error) { + if _, found := c.users[source.GetGroupPath()]; found { + return c.fetchUserContent(ctx, source.GetGroupPath()) + } else { + return c.fetchOrganizationContent(ctx, source.GetGroupPath()) } - return nil, nil, fmt.Errorf("invalid gid: %v", gid) } diff --git a/forges/github/organization.go b/forges/github/organization.go index bef3ce2..59c2054 100644 --- a/forges/github/organization.go +++ b/forges/github/organization.go @@ -3,103 +3,67 @@ package github import ( "context" "fmt" - "sync" - "github.com/badjware/gitforgefs/fstree" + "github.com/badjware/gitforgefs/types" "github.com/google/go-github/v63/github" ) type Organization struct { ID int64 Name string - - mux sync.Mutex - - // hold org content - childRepositories map[string]fstree.RepositorySource } func (o *Organization) GetGroupID() uint64 { return uint64(o.ID) } -func (o *Organization) InvalidateContentCache() { - o.mux.Lock() - defer o.mux.Unlock() - - // clear child repositories from cache - o.childRepositories = nil +func (o *Organization) GetGroupPath() string { + return o.Name } func (c *githubClient) fetchOrganization(ctx context.Context, orgName string) (*Organization, error) { - c.organizationCacheMux.RLock() - cachedId, found := c.organizationNameToIDMap[orgName] - if found { - cachedOrg := c.organizationCache[cachedId] - c.organizationCacheMux.RUnlock() - - // if found in cache, return the cached reference - c.logger.Debug("Organization cache hit", "org_name", orgName) - return cachedOrg, nil - } else { - c.organizationCacheMux.RUnlock() - - c.logger.Debug("Organization cache miss", "org_name", orgName) - } - - // If not found in cache, fetch organization infos from API githubOrg, _, err := c.client.Organizations.Get(ctx, orgName) if err != nil { return nil, fmt.Errorf("failed to fetch organization with name %v: %v", orgName, err) } - newOrg := Organization{ + return &Organization{ ID: *githubOrg.ID, Name: *githubOrg.Login, - - childRepositories: nil, - } - - // save in cache - c.organizationCacheMux.Lock() - c.organizationCache[newOrg.ID] = &newOrg - c.organizationNameToIDMap[newOrg.Name] = newOrg.ID - c.organizationCacheMux.Unlock() - - return &newOrg, nil + }, nil } -func (c *githubClient) fetchOrganizationContent(ctx context.Context, org *Organization) (map[string]fstree.GroupSource, map[string]fstree.RepositorySource, error) { - org.mux.Lock() - defer org.mux.Unlock() +func (c *githubClient) fetchOrganizationContent(ctx context.Context, orgName string) (types.GroupContent, error) { + org, err := c.fetchOrganization(ctx, orgName) + if err != nil { + return types.GroupContent{}, err + } - // Get cached data if available - // TODO: cache cache invalidation? - if org.childRepositories == nil { - childRepositories := make(map[string]fstree.RepositorySource) + repositories := make(map[string]types.RepositorySource) - // Fetch the organization repositories - repositoryListOpt := &github.RepositoryListByOrgOptions{ - ListOptions: github.ListOptions{PerPage: 100}, + // Fetch the organization repositories + repositoryListOpt := &github.RepositoryListByOrgOptions{ + ListOptions: github.ListOptions{PerPage: 100}, + } + for { + githubRepositories, response, err := c.client.Repositories.ListByOrg(ctx, org.Name, repositoryListOpt) + if err != nil { + return types.GroupContent{}, fmt.Errorf("failed to fetch repository in github: %v", err) } - for { - githubRepositories, response, err := c.client.Repositories.ListByOrg(ctx, org.Name, repositoryListOpt) - if err != nil { - return nil, nil, fmt.Errorf("failed to fetch repository in github: %v", err) - } - for _, githubRepository := range githubRepositories { - repository := c.newRepositoryFromGithubRepository(ctx, githubRepository) - if repository != nil { - childRepositories[repository.Path] = repository - } + for _, githubRepository := range githubRepositories { + repository := c.newRepositoryFromGithubRepository(githubRepository) + if repository != nil { + repositories[repository.GetRepositoryName()] = repository } - if response.NextPage == 0 { - break - } - // Get the next page - repositoryListOpt.Page = response.NextPage } - - org.childRepositories = childRepositories + if response.NextPage == 0 { + break + } + // Get the next page + repositoryListOpt.Page = response.NextPage } - return make(map[string]fstree.GroupSource), org.childRepositories, nil + + return types.GroupContent{ + Groups: make(map[string]types.GroupSource), + Repositories: repositories, + }, nil } diff --git a/forges/github/repository.go b/forges/github/repository.go index ecfe720..8fc120d 100644 --- a/forges/github/repository.go +++ b/forges/github/repository.go @@ -1,7 +1,6 @@ package github import ( - "context" "path" "github.com/badjware/gitforgefs/config" @@ -19,6 +18,10 @@ func (r *Repository) GetRepositoryID() uint64 { return uint64(r.ID) } +func (r *Repository) GetRepositoryName() string { + return r.Path +} + func (r *Repository) GetCloneURL() string { return r.CloneURL } @@ -27,7 +30,7 @@ func (r *Repository) GetDefaultBranch() string { return r.DefaultBranch } -func (c *githubClient) newRepositoryFromGithubRepository(ctx context.Context, repository *github.Repository) *Repository { +func (c *githubClient) newRepositoryFromGithubRepository(repository *github.Repository) *Repository { if c.ArchivedRepoHandling == config.ArchivedProjectIgnore && *repository.Archived { return nil } diff --git a/forges/github/user.go b/forges/github/user.go index 5e16b90..9cfc16f 100644 --- a/forges/github/user.go +++ b/forges/github/user.go @@ -3,103 +3,67 @@ package github import ( "context" "fmt" - "sync" - "github.com/badjware/gitforgefs/fstree" + "github.com/badjware/gitforgefs/types" "github.com/google/go-github/v63/github" ) type User struct { ID int64 Name string - - mux sync.Mutex - - // hold user content - childRepositories map[string]fstree.RepositorySource } func (u *User) GetGroupID() uint64 { return uint64(u.ID) } -func (u *User) InvalidateContentCache() { - u.mux.Lock() - defer u.mux.Unlock() - - // clear child repositories from cache - u.childRepositories = nil +func (u *User) GetGroupPath() string { + return u.Name } func (c *githubClient) fetchUser(ctx context.Context, userName string) (*User, error) { - c.userCacheMux.RLock() - cachedId, found := c.userNameToIDMap[userName] - if found { - cachedUser := c.userCache[cachedId] - c.userCacheMux.RUnlock() - - // if found in cache, return the cached reference - c.logger.Debug("User cache hit", "user_name", userName) - return cachedUser, nil - } else { - c.userCacheMux.RUnlock() - - c.logger.Debug("User cache miss", "user_name", userName) - } - - // If not found in cache, fetch user infos from API githubUser, _, err := c.client.Users.Get(ctx, userName) if err != nil { return nil, fmt.Errorf("failed to fetch user with name %v: %v", userName, err) } - newUser := User{ + return &User{ ID: *githubUser.ID, Name: *githubUser.Login, - - childRepositories: nil, - } - - // save in cache - c.userCacheMux.Lock() - c.userCache[newUser.ID] = &newUser - c.userNameToIDMap[newUser.Name] = newUser.ID - c.userCacheMux.Unlock() - - return &newUser, nil + }, nil } -func (c *githubClient) fetchUserContent(ctx context.Context, user *User) (map[string]fstree.GroupSource, map[string]fstree.RepositorySource, error) { - user.mux.Lock() - defer user.mux.Unlock() +func (c *githubClient) fetchUserContent(ctx context.Context, userName string) (types.GroupContent, error) { + user, err := c.fetchUser(ctx, userName) + if err != nil { + return types.GroupContent{}, err + } - // Get cached data if available - // TODO: cache cache invalidation? - if user.childRepositories == nil { - childRepositories := make(map[string]fstree.RepositorySource) + repositories := make(map[string]types.RepositorySource) - // Fetch the user repositories - repositoryListOpt := &github.RepositoryListByUserOptions{ - ListOptions: github.ListOptions{PerPage: 100}, + // Fetch the user repositories + repositoryListOpt := &github.RepositoryListByUserOptions{ + ListOptions: github.ListOptions{PerPage: 100}, + } + for { + githubRepositories, response, err := c.client.Repositories.ListByUser(ctx, user.Name, repositoryListOpt) + if err != nil { + return types.GroupContent{}, fmt.Errorf("failed to fetch repository in github: %v", err) } - for { - githubRepositories, response, err := c.client.Repositories.ListByUser(ctx, user.Name, repositoryListOpt) - if err != nil { - return nil, nil, fmt.Errorf("failed to fetch repository in github: %v", err) - } - for _, githubRepository := range githubRepositories { - repository := c.newRepositoryFromGithubRepository(ctx, githubRepository) - if repository != nil { - childRepositories[repository.Path] = repository - } + for _, githubRepository := range githubRepositories { + repository := c.newRepositoryFromGithubRepository(githubRepository) + if repository != nil { + repositories[repository.Path] = repository } - if response.NextPage == 0 { - break - } - // Get the next page - repositoryListOpt.Page = response.NextPage } - - user.childRepositories = childRepositories + if response.NextPage == 0 { + break + } + // Get the next page + repositoryListOpt.Page = response.NextPage } - return make(map[string]fstree.GroupSource), user.childRepositories, nil + + return types.GroupContent{ + Groups: make(map[string]types.GroupSource), + Repositories: repositories, + }, nil } diff --git a/forges/gitlab/client.go b/forges/gitlab/client.go index 1fb42e9..18e71e8 100644 --- a/forges/gitlab/client.go +++ b/forges/gitlab/client.go @@ -4,11 +4,9 @@ import ( "context" "fmt" "log/slog" - "slices" - "sync" "github.com/badjware/gitforgefs/config" - "github.com/badjware/gitforgefs/fstree" + "github.com/badjware/gitforgefs/types" gitlab "gitlab.com/gitlab-org/api/client-go" ) @@ -18,15 +16,8 @@ type gitlabClient struct { logger *slog.Logger - rootContent map[string]fstree.GroupSource - - userIDs []int - - // API response cache - groupCacheMux sync.RWMutex - groupCache map[int]*Group - userCacheMux sync.RWMutex - userCache map[int]*User + // use a map without values for efficient lookups + users map[string]int } func NewClient(logger *slog.Logger, config config.GitlabClientConfig) (*gitlabClient, error) { @@ -44,12 +35,7 @@ func NewClient(logger *slog.Logger, config config.GitlabClientConfig) (*gitlabCl logger: logger, - rootContent: nil, - - userIDs: []int{}, - - groupCache: map[int]*Group{}, - userCache: map[int]*User{}, + users: make(map[string]int), } // Fetch current user and add it to the list @@ -57,7 +43,7 @@ func NewClient(logger *slog.Logger, config config.GitlabClientConfig) (*gitlabCl if err != nil { logger.Warn("failed to fetch the current user:", "error", err.Error()) } else { - gitlabClient.userIDs = append(gitlabClient.userIDs, currentUser.ID) + gitlabClient.users[currentUser.Username] = currentUser.ID } // Fetch the configured users and add them to the list @@ -66,54 +52,39 @@ func NewClient(logger *slog.Logger, config config.GitlabClientConfig) (*gitlabCl if err != nil || len(user) != 1 { logger.Warn("failed to fetch the user", "userName", userName, "error", err.Error()) } else { - gitlabClient.userIDs = append(gitlabClient.userIDs, user[0].ID) + gitlabClient.users[userName] = user[0].ID } } return gitlabClient, nil } -func (c *gitlabClient) FetchRootGroupContent(ctx context.Context) (map[string]fstree.GroupSource, error) { - // use cached values if available - if c.rootContent == nil { - rootGroupCache := make(map[string]fstree.GroupSource) +func (c *gitlabClient) FetchRootGroupContent(ctx context.Context) (map[string]types.GroupSource, error) { + rootContent := make(map[string]types.GroupSource) - // fetch root groups - for _, gid := range c.GroupIDs { - group, err := c.fetchGroup(ctx, gid) - if err != nil { - return nil, err - } - rootGroupCache[group.Name] = group + // fetch root groups + for _, gid := range c.GroupIDs { + group, err := c.fetchGroup(ctx, gid) + if err != nil { + return nil, err } - // fetch users - for _, uid := range c.userIDs { - user, err := c.fetchUser(ctx, uid) - if err != nil { - return nil, err - } - rootGroupCache[user.Name] = user + rootContent[group.Name] = group + } + // fetch users + for _, uid := range c.users { + user, err := c.fetchUser(ctx, uid) + if err != nil { + return nil, err } - - c.rootContent = rootGroupCache + rootContent[user.Name] = user } - return c.rootContent, nil + return rootContent, nil } -func (c *gitlabClient) FetchGroupContent(ctx context.Context, gid uint64) (map[string]fstree.GroupSource, map[string]fstree.RepositorySource, error) { - if slices.Contains[[]int, int](c.userIDs, int(gid)) { - // gid is a user - user, err := c.fetchUser(ctx, int(gid)) - if err != nil { - return nil, nil, err - } - return c.fetchUserContent(ctx, user) +func (c *gitlabClient) FetchGroupContent(ctx context.Context, source types.GroupSource) (types.GroupContent, error) { + if _, found := c.users[source.GetGroupPath()]; found { + return c.fetchUserContent(ctx, source.GetGroupID()) } else { - // gid is a group - group, err := c.fetchGroup(ctx, int(gid)) - if err != nil { - return nil, nil, err - } - return c.fetchGroupContent(ctx, group) + return c.fetchGroupContent(ctx, source.GetGroupID()) } } diff --git a/forges/gitlab/group.go b/forges/gitlab/group.go index 9ee48d5..48b71c9 100644 --- a/forges/gitlab/group.go +++ b/forges/gitlab/group.go @@ -3,180 +3,101 @@ package gitlab import ( "context" "fmt" - "sync" - "github.com/badjware/gitforgefs/fstree" + "github.com/badjware/gitforgefs/types" gitlab "gitlab.com/gitlab-org/api/client-go" ) type Group struct { ID int Name string - - gitlabClient *gitlabClient - - mux sync.Mutex - - // hold group content - childGroups map[string]fstree.GroupSource - childProjects map[string]fstree.RepositorySource + Path string } func (g *Group) GetGroupID() uint64 { return uint64(g.ID) } -func (g *Group) InvalidateContentCache() { - g.mux.Lock() - defer g.mux.Unlock() - - // clear child group from cache - g.gitlabClient.groupCacheMux.Lock() - for _, childGroup := range g.childGroups { - gid := int(childGroup.GetGroupID()) - delete(g.gitlabClient.groupCache, gid) - } - g.gitlabClient.groupCacheMux.Unlock() - g.childGroups = nil +func (g *Group) GetGroupName() string { + return g.Name +} - // clear child repositories from cache - g.childGroups = nil +func (g *Group) GetGroupPath() string { + return g.Path } -func (c *gitlabClient) fetchGroup(ctx context.Context, gid int) (*Group, error) { - // start by searching the cache - // TODO: cache invalidation? - c.groupCacheMux.RLock() - group, found := c.groupCache[gid] - c.groupCacheMux.RUnlock() - if found { - c.logger.Debug("Group cache hit", "gid", gid) - return group, nil - } else { - c.logger.Debug("Group cache miss; fetching group", "gid", gid) +func (c *gitlabClient) newGroupFromGitlabGroup(gitlabGroup *gitlab.Group) *Group { + return &Group{ + ID: gitlabGroup.ID, + Name: gitlabGroup.Path, + Path: gitlabGroup.FullPath, } +} - // If not in cache, fetch group infos from API +func (c *gitlabClient) fetchGroup(ctx context.Context, gid uint64) (*Group, error) { gitlabGroup, _, err := c.client.Groups.GetGroup(gid, &gitlab.GetGroupOptions{}) if err != nil { return nil, fmt.Errorf("failed to fetch group with id %v: %v", gid, err) } c.logger.Debug("Fetched group", "gid", gid) - newGroup := Group{ - ID: gitlabGroup.ID, - Name: gitlabGroup.Path, - - gitlabClient: c, - - childGroups: nil, - childProjects: nil, - } - - // save in cache - c.groupCacheMux.Lock() - c.groupCache[gid] = &newGroup - c.groupCacheMux.Unlock() - - return &newGroup, nil + return c.newGroupFromGitlabGroup(gitlabGroup), nil } -func (c *gitlabClient) newGroupFromGitlabGroup(gitlabGroup *gitlab.Group) (*Group, error) { - gid := gitlabGroup.ID - - // start by searching the cache - c.groupCacheMux.RLock() - group, found := c.groupCache[gid] - c.groupCacheMux.RUnlock() - if found { - // if found in cache, return the cached reference - c.logger.Debug("Group cache hit", "gid", gid) - return group, nil - } else { - c.logger.Debug("Group cache miss; registering group", "gid", gid) - } - - // if not found in cache, convert and save to cache now - newGroup := Group{ - ID: gitlabGroup.ID, - Name: gitlabGroup.Path, - - gitlabClient: c, - - childGroups: nil, - childProjects: nil, +func (c *gitlabClient) fetchGroupContent(ctx context.Context, gid uint64) (types.GroupContent, error) { + childGroups := make(map[string]types.GroupSource) + childProjects := make(map[string]types.RepositorySource) + + // List subgroups in path + listGroupsOpt := &gitlab.ListSubGroupsOptions{ + ListOptions: gitlab.ListOptions{ + Page: 1, + PerPage: 100, + }, + AllAvailable: gitlab.Ptr(true), } - - // save in cache - c.groupCacheMux.Lock() - c.groupCache[gid] = &newGroup - c.groupCacheMux.Unlock() - - return &newGroup, nil -} - -func (c *gitlabClient) fetchGroupContent(ctx context.Context, group *Group) (map[string]fstree.GroupSource, map[string]fstree.RepositorySource, error) { - // Only a single routine can fetch the group content at the time. - // We lock for the whole duration of the function to avoid fetching the same data from the API - // multiple times if concurrent calls where to occur. - group.mux.Lock() - defer group.mux.Unlock() - - // Get cached data if available - // TODO: cache cache invalidation? - if group.childGroups == nil || group.childProjects == nil { - childGroups := make(map[string]fstree.GroupSource) - childProjects := make(map[string]fstree.RepositorySource) - - // List subgroups in path - listGroupsOpt := &gitlab.ListSubGroupsOptions{ - ListOptions: gitlab.ListOptions{ - Page: 1, - PerPage: 100, - }, - AllAvailable: gitlab.Ptr(true), + for { + gitlabGroups, response, err := c.client.Groups.ListSubGroups(gid, listGroupsOpt) + if err != nil { + return types.GroupContent{}, fmt.Errorf("failed to fetch groups in gitlab: %v", err) } - for { - gitlabGroups, response, err := c.client.Groups.ListSubGroups(group.ID, listGroupsOpt) - if err != nil { - return nil, nil, fmt.Errorf("failed to fetch groups in gitlab: %v", err) + for _, gitlabGroup := range gitlabGroups { + group := c.newGroupFromGitlabGroup(gitlabGroup) + if group != nil { + childGroups[group.GetGroupName()] = group } - for _, gitlabGroup := range gitlabGroups { - group, _ := c.newGroupFromGitlabGroup(gitlabGroup) - childGroups[group.Name] = group - } - if response.CurrentPage >= response.TotalPages { - break - } - // Get the next page - listGroupsOpt.Page = response.NextPage } + if response.CurrentPage >= response.TotalPages { + break + } + // Get the next page + listGroupsOpt.Page = response.NextPage + } - // List projects in path - listProjectOpt := &gitlab.ListGroupProjectsOptions{ - ListOptions: gitlab.ListOptions{ - Page: 1, - PerPage: 100, - }} - for { - gitlabProjects, response, err := c.client.Groups.ListGroupProjects(group.ID, listProjectOpt) - if err != nil { - return nil, nil, fmt.Errorf("failed to fetch projects in gitlab: %v", err) - } - for _, gitlabProject := range gitlabProjects { - project := c.newProjectFromGitlabProject(ctx, gitlabProject) - if project != nil { - childProjects[project.Path] = project - } - } - if response.CurrentPage >= response.TotalPages { - break + // List projects in path + listProjectOpt := &gitlab.ListGroupProjectsOptions{ + ListOptions: gitlab.ListOptions{ + Page: 1, + PerPage: 100, + }} + for { + gitlabProjects, response, err := c.client.Groups.ListGroupProjects(gid, listProjectOpt) + if err != nil { + return types.GroupContent{}, fmt.Errorf("failed to fetch projects in gitlab: %v", err) + } + for _, gitlabProject := range gitlabProjects { + project := c.newProjectFromGitlabProject(gitlabProject) + if project != nil { + childProjects[project.GetRepositoryName()] = project } - // Get the next page - listProjectOpt.Page = response.NextPage } - - group.childGroups = childGroups - group.childProjects = childProjects + if response.CurrentPage >= response.TotalPages { + break + } + // Get the next page + listProjectOpt.Page = response.NextPage } - return group.childGroups, group.childProjects, nil + return types.GroupContent{ + Groups: childGroups, + Repositories: childProjects, + }, nil } diff --git a/forges/gitlab/project.go b/forges/gitlab/project.go index 2587265..9dec86d 100644 --- a/forges/gitlab/project.go +++ b/forges/gitlab/project.go @@ -1,7 +1,6 @@ package gitlab import ( - "context" "path" "github.com/badjware/gitforgefs/config" @@ -10,6 +9,7 @@ import ( type Project struct { ID int + Name string Path string CloneURL string DefaultBranch string @@ -19,6 +19,10 @@ func (p *Project) GetRepositoryID() uint64 { return uint64(p.ID) } +func (p *Project) GetRepositoryName() string { + return p.Name +} + func (p *Project) GetCloneURL() string { return p.CloneURL } @@ -27,7 +31,7 @@ func (p *Project) GetDefaultBranch() string { return p.DefaultBranch } -func (c *gitlabClient) newProjectFromGitlabProject(ctx context.Context, project *gitlab.Project) *Project { +func (c *gitlabClient) newProjectFromGitlabProject(project *gitlab.Project) *Project { // https://godoc.org/github.com/xanzy/go-gitlab#Project if c.ArchivedProjectHandling == config.ArchivedProjectIgnore && project.Archived { return nil diff --git a/forges/gitlab/user.go b/forges/gitlab/user.go index 726e96b..0a26dcf 100644 --- a/forges/gitlab/user.go +++ b/forges/gitlab/user.go @@ -3,105 +3,63 @@ package gitlab import ( "context" "fmt" - "sync" - "github.com/badjware/gitforgefs/fstree" + "github.com/badjware/gitforgefs/types" gitlab "gitlab.com/gitlab-org/api/client-go" ) type User struct { ID int Name string - - mux sync.Mutex - - // hold user content - childProjects map[string]fstree.RepositorySource } func (u *User) GetGroupID() uint64 { return uint64(u.ID) } -func (u *User) InvalidateContentCache() { - u.mux.Lock() - defer u.mux.Unlock() - - // clear child repositories from cache - u.childProjects = nil +func (u *User) GetGroupPath() string { + return u.Name } func (c *gitlabClient) fetchUser(ctx context.Context, uid int) (*User, error) { - // start by searching the cache - // TODO: cache invalidation? - c.userCacheMux.RLock() - user, found := c.userCache[uid] - c.userCacheMux.RUnlock() - if found { - // if found in cache, return the cached reference - c.logger.Debug("User cache hit", "uid", uid) - return user, nil - } else { - c.logger.Debug("User cache miss", "uid", uid) - } - - // If not found in cache, fetch group infos from API gitlabUser, _, err := c.client.Users.GetUser(uid, gitlab.GetUsersOptions{}) if err != nil { return nil, fmt.Errorf("failed to fetch user with id %v: %v", uid, err) } - newUser := User{ + return &User{ ID: gitlabUser.ID, Name: gitlabUser.Username, - - childProjects: nil, - } - - // save in cache - c.userCacheMux.Lock() - c.userCache[uid] = &newUser - c.userCacheMux.Unlock() - - return &newUser, nil + }, nil } -func (c *gitlabClient) fetchUserContent(ctx context.Context, user *User) (map[string]fstree.GroupSource, map[string]fstree.RepositorySource, error) { - // Only a single routine can fetch the user content at the time. - // We lock for the whole duration of the function to avoid fetching the same data from the API - // multiple times if concurrent calls where to occur. - user.mux.Lock() - defer user.mux.Unlock() - - // Get cached data if available - // TODO: cache cache invalidation? - if user.childProjects == nil { - childProjects := make(map[string]fstree.RepositorySource) - - // Fetch the user repositories - listProjectOpt := &gitlab.ListProjectsOptions{ - ListOptions: gitlab.ListOptions{ - Page: 1, - PerPage: 100, - }} - for { - gitlabProjects, response, err := c.client.Projects.ListUserProjects(user.ID, listProjectOpt) - if err != nil { - return nil, nil, fmt.Errorf("failed to fetch projects in gitlab: %v", err) - } - for _, gitlabProject := range gitlabProjects { - project := c.newProjectFromGitlabProject(ctx, gitlabProject) - if project != nil { - childProjects[project.Path] = project - } - } - if response.CurrentPage >= response.TotalPages { - break +func (c *gitlabClient) fetchUserContent(ctx context.Context, uid uint64) (types.GroupContent, error) { + childProjects := make(map[string]types.RepositorySource) + + // Fetch the user repositories + listProjectOpt := &gitlab.ListProjectsOptions{ + ListOptions: gitlab.ListOptions{ + Page: 1, + PerPage: 100, + }} + for { + gitlabProjects, response, err := c.client.Projects.ListUserProjects(uid, listProjectOpt) + if err != nil { + return types.GroupContent{}, fmt.Errorf("failed to fetch projects in gitlab: %v", err) + } + for _, gitlabProject := range gitlabProjects { + project := c.newProjectFromGitlabProject(gitlabProject) + if project != nil { + childProjects[project.GetRepositoryName()] = project } - // Get the next page - listProjectOpt.Page = response.NextPage } - - user.childProjects = childProjects + if response.CurrentPage >= response.TotalPages { + break + } + // Get the next page + listProjectOpt.Page = response.NextPage } - return make(map[string]fstree.GroupSource), user.childProjects, nil + return types.GroupContent{ + Groups: make(map[string]types.GroupSource), + Repositories: childProjects, + }, nil } diff --git a/fstree/group.go b/fstree/group.go index b58d874..81cc1fa 100644 --- a/fstree/group.go +++ b/fstree/group.go @@ -4,6 +4,7 @@ import ( "context" "syscall" + "github.com/badjware/gitforgefs/types" "github.com/hanwen/go-fuse/v2/fs" "github.com/hanwen/go-fuse/v2/fuse" ) @@ -16,22 +17,17 @@ type groupNode struct { fs.Inode param *FSParam - source GroupSource + source types.GroupSource staticNodes map[string]staticNode } -type GroupSource interface { - GetGroupID() uint64 - InvalidateContentCache() -} - // Ensure we are implementing the NodeReaddirer interface var _ = (fs.NodeReaddirer)((*groupNode)(nil)) // Ensure we are implementing the NodeLookuper interface var _ = (fs.NodeLookuper)((*groupNode)(nil)) -func newGroupNodeFromSource(ctx context.Context, source GroupSource, param *FSParam) (fs.InodeEmbedder, error) { +func newGroupNodeFromSource(ctx context.Context, source types.GroupSource, param *FSParam) (fs.InodeEmbedder, error) { node := &groupNode{ param: param, source: source, @@ -43,20 +39,20 @@ func newGroupNodeFromSource(ctx context.Context, source GroupSource, param *FSPa } func (n *groupNode) Readdir(ctx context.Context) (fs.DirStream, syscall.Errno) { - groups, repositories, err := n.param.GitForge.FetchGroupContent(ctx, n.source.GetGroupID()) + content, err := n.param.GitForge.FetchGroupContent(ctx, n.source) if err != nil { n.param.logger.Error(err.Error()) } - entries := make([]fuse.DirEntry, 0, len(groups)+len(repositories)+len(n.staticNodes)) - for groupName, group := range groups { + entries := make([]fuse.DirEntry, 0, len(content.Groups)+len(content.Repositories)+len(n.staticNodes)) + for groupName, group := range content.Groups { entries = append(entries, fuse.DirEntry{ Name: groupName, Ino: group.GetGroupID() + groupBaseInode, Mode: fuse.S_IFDIR, }) } - for repositoryName, repository := range repositories { + for repositoryName, repository := range content.Repositories { entries = append(entries, fuse.DirEntry{ Name: repositoryName, Ino: repository.GetRepositoryID() + repositoryBaseInode, @@ -74,12 +70,12 @@ func (n *groupNode) Readdir(ctx context.Context) (fs.DirStream, syscall.Errno) { } func (n *groupNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) (*fs.Inode, syscall.Errno) { - groups, repositories, err := n.param.GitForge.FetchGroupContent(ctx, n.source.GetGroupID()) + content, err := n.param.GitForge.FetchGroupContent(ctx, n.source) if err != nil { n.param.logger.Error(err.Error()) } else { // Check if the map of groups contains it - group, found := groups[name] + group, found := content.Groups[name] if found { attrs := fs.StableAttr{ Ino: group.GetGroupID() + groupBaseInode, @@ -90,7 +86,7 @@ func (n *groupNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) } // Check if the map of projects contains it - repository, found := repositories[name] + repository, found := content.Repositories[name] if found { attrs := fs.StableAttr{ Ino: repository.GetRepositoryID() + repositoryBaseInode, diff --git a/fstree/refresh.go b/fstree/refresh.go index 25671e9..e52ae4b 100644 --- a/fstree/refresh.go +++ b/fstree/refresh.go @@ -4,6 +4,7 @@ import ( "context" "syscall" + "github.com/badjware/gitforgefs/types" "github.com/hanwen/go-fuse/v2/fs" "github.com/hanwen/go-fuse/v2/fuse" ) @@ -12,7 +13,7 @@ type refreshNode struct { fs.Inode ino uint64 - source GroupSource + source types.GroupSource } // Ensure we are implementing the NodeSetattrer interface @@ -21,7 +22,7 @@ var _ = (fs.NodeSetattrer)((*refreshNode)(nil)) // Ensure we are implementing the NodeOpener interface var _ = (fs.NodeOpener)((*refreshNode)(nil)) -func newRefreshNode(source GroupSource, param *FSParam) *refreshNode { +func newRefreshNode(source types.GroupSource, param *FSParam) *refreshNode { return &refreshNode{ ino: 0, source: source, @@ -41,6 +42,7 @@ func (n *refreshNode) Setattr(ctx context.Context, fh fs.FileHandle, in *fuse.Se } func (n *refreshNode) Open(ctx context.Context, flags uint32) (fh fs.FileHandle, fuseFlags uint32, errno syscall.Errno) { - n.source.InvalidateContentCache() + // FIXME + // n.source.InvalidateContentCache() return nil, 0, 0 } diff --git a/fstree/repository.go b/fstree/repository.go index ce1f49f..192f548 100644 --- a/fstree/repository.go +++ b/fstree/repository.go @@ -8,6 +8,7 @@ import ( "syscall" "time" + "github.com/badjware/gitforgefs/types" "github.com/hanwen/go-fuse/v2/fs" ) @@ -19,20 +20,13 @@ type repositorySymlinkNode struct { fs.Inode param *FSParam - source RepositorySource -} - -type RepositorySource interface { - // GetName() string - GetRepositoryID() uint64 - GetCloneURL() string - GetDefaultBranch() string + source types.RepositorySource } // Ensure we are implementing the NodeReaddirer interface var _ = (fs.NodeReadlinker)((*repositorySymlinkNode)(nil)) -func newRepositoryNodeFromSource(ctx context.Context, source RepositorySource, param *FSParam) (fs.InodeEmbedder, error) { +func newRepositoryNodeFromSource(ctx context.Context, source types.RepositorySource, param *FSParam) (fs.InodeEmbedder, error) { if param.UseSymlinks { return &repositorySymlinkNode{ param: param, diff --git a/fstree/root.go b/fstree/root.go index c534e59..a30c42a 100644 --- a/fstree/root.go +++ b/fstree/root.go @@ -8,6 +8,7 @@ import ( "os/signal" "syscall" + "github.com/badjware/gitforgefs/types" "github.com/hanwen/go-fuse/v2/fs" "github.com/hanwen/go-fuse/v2/fuse" ) @@ -19,19 +20,14 @@ type staticNode interface { } type GitClient interface { - FetchLocalRepositoryPath(ctx context.Context, source RepositorySource) (string, error) -} - -type GitForge interface { - FetchRootGroupContent(ctx context.Context) (map[string]GroupSource, error) - FetchGroupContent(ctx context.Context, gid uint64) (map[string]GroupSource, map[string]RepositorySource, error) + FetchLocalRepositoryPath(ctx context.Context, source types.RepositorySource) (string, error) } type FSParam struct { UseSymlinks bool GitClient GitClient - GitForge GitForge + GitForge types.GitForge logger *slog.Logger } diff --git a/git/client.go b/git/client.go index 2532df0..35d291f 100644 --- a/git/client.go +++ b/git/client.go @@ -10,8 +10,8 @@ import ( "strconv" "github.com/badjware/gitforgefs/config" - "github.com/badjware/gitforgefs/fstree" "github.com/badjware/gitforgefs/queue" + "github.com/badjware/gitforgefs/types" "github.com/badjware/gitforgefs/utils" ) @@ -62,7 +62,7 @@ func NewClient(logger *slog.Logger, p config.GitClientConfig) (*gitClient, error return c, nil } -func (c *gitClient) FetchLocalRepositoryPath(ctx context.Context, source fstree.RepositorySource) (localRepoLoc string, err error) { +func (c *gitClient) FetchLocalRepositoryPath(ctx context.Context, source types.RepositorySource) (localRepoLoc string, err error) { rid := source.GetRepositoryID() cloneUrl := source.GetCloneURL() defaultBranch := source.GetDefaultBranch() diff --git a/main.go b/main.go index a66a81d..05c17b5 100644 --- a/main.go +++ b/main.go @@ -7,12 +7,14 @@ import ( "os" "strings" + "github.com/badjware/gitforgefs/cache" "github.com/badjware/gitforgefs/config" "github.com/badjware/gitforgefs/forges/gitea" "github.com/badjware/gitforgefs/forges/github" "github.com/badjware/gitforgefs/forges/gitlab" "github.com/badjware/gitforgefs/fstree" "github.com/badjware/gitforgefs/git" + "github.com/badjware/gitforgefs/types" ) func main() { @@ -74,7 +76,8 @@ func main() { } gitClient, _ := git.NewClient(logger, *gitClientParam) - var gitForgeClient fstree.GitForge + // setup backend + var gitForgeClient types.GitForge if loadedConfig.FS.Forge == config.ForgeGitlab { // Create the gitlab client gitlabClientConfig, err := config.MakeGitlabConfig(loadedConfig) @@ -101,6 +104,9 @@ func main() { gitForgeClient, _ = gitea.NewClient(logger, *giteaClientConfig) } + // setup cache + cache := cache.NewForgeCache(gitForgeClient) + // Start the filesystem err = fstree.Start( logger, @@ -109,7 +115,7 @@ func main() { &fstree.FSParam{ UseSymlinks: loadedConfig.FS.UseSymlinks, GitClient: gitClient, - GitForge: gitForgeClient, + GitForge: cache, }, *debug, ) diff --git a/types/types.go b/types/types.go new file mode 100644 index 0000000..320e571 --- /dev/null +++ b/types/types.go @@ -0,0 +1,27 @@ +package types + +import ( + "context" +) + +type GitForge interface { + FetchRootGroupContent(ctx context.Context) (map[string]GroupSource, error) + FetchGroupContent(ctx context.Context, source GroupSource) (GroupContent, error) +} + +type GroupSource interface { + GetGroupID() uint64 + GetGroupPath() string +} + +type RepositorySource interface { + GetRepositoryID() uint64 + GetRepositoryName() string + GetCloneURL() string + GetDefaultBranch() string +} + +type GroupContent struct { + Groups map[string]GroupSource + Repositories map[string]RepositorySource +} From b07a0324aff6c0911fa03d75cd3225155d2094d8 Mon Sep 17 00:00:00 2001 From: Massaki Archambault Date: Wed, 24 Dec 2025 14:20:36 -0500 Subject: [PATCH 2/8] fix gitlab repository fetch --- cache/cache.go | 12 +++++++++++- config/loader.go | 4 ++-- config/loader_test.go | 10 +++++----- forges/gitlab/client.go | 4 ++-- forges/gitlab/group.go | 4 ++-- forges/gitlab/project.go | 3 ++- forges/gitlab/user.go | 2 +- main.go | 2 +- 8 files changed, 26 insertions(+), 15 deletions(-) diff --git a/cache/cache.go b/cache/cache.go index b4001f3..210d3a8 100644 --- a/cache/cache.go +++ b/cache/cache.go @@ -2,6 +2,7 @@ package cache import ( "context" + "log/slog" "sync" "time" @@ -10,6 +11,7 @@ import ( type Cache struct { backend types.GitForge + logger *slog.Logger rootContentLock sync.RWMutex cachedRootContent map[string]types.GroupSource @@ -18,9 +20,10 @@ type Cache struct { cachedContent map[string]CachedContent } -func NewForgeCache(backend types.GitForge) types.GitForge { +func NewForgeCache(backend types.GitForge, logger *slog.Logger) types.GitForge { return &Cache{ backend: backend, + logger: logger, cachedContent: map[string]CachedContent{}, } @@ -43,6 +46,7 @@ func (c *Cache) FetchRootGroupContent(ctx context.Context) (map[string]types.Gro // check to make sure the data is still not there, // since RWMutex is not upgradeable and another thread may have grabbed the lock in the meantime if c.cachedRootContent == nil { + c.logger.Info("Fetching root content from backend") content, err := c.backend.FetchRootGroupContent(ctx) if err != nil { return nil, err @@ -56,10 +60,14 @@ func (c *Cache) FetchRootGroupContent(ctx context.Context) (map[string]types.Gro } func (c *Cache) FetchGroupContent(ctx context.Context, source types.GroupSource) (types.GroupContent, error) { + logger := c.logger.With("groupID", source.GetGroupID()).With("groupPath", source.GetGroupPath()) + c.contentLock.RLock() if cachedContent, found := c.cachedContent[source.GetGroupPath()]; !found { c.contentLock.RUnlock() + logger.Debug("Cache miss") + // acquire write lock c.contentLock.Lock() defer c.contentLock.Unlock() @@ -70,6 +78,7 @@ func (c *Cache) FetchGroupContent(ctx context.Context, source types.GroupSource) } // fetch content from backend and cache it + logger.Info("Fetching content from backend") content, err := c.backend.FetchGroupContent(ctx, source) if err != nil { return types.GroupContent{}, err @@ -81,6 +90,7 @@ func (c *Cache) FetchGroupContent(ctx context.Context, source types.GroupSource) return content, nil } else { c.contentLock.RUnlock() + logger.Debug("Cache hit") return cachedContent.GroupContent, nil } } diff --git a/config/loader.go b/config/loader.go index 02d0b6b..456e8d8 100644 --- a/config/loader.go +++ b/config/loader.go @@ -39,7 +39,7 @@ type ( URL string `yaml:"url,omitempty"` Token string `yaml:"token,omitempty"` - GroupIDs []uint64 `yaml:"group_ids,omitempty"` + GroupIDs []int `yaml:"group_ids,omitempty"` UserNames []string `yaml:"user_names,omitempty"` ArchivedProjectHandling string `yaml:"archived_project_handling,omitempty"` @@ -97,7 +97,7 @@ func LoadConfig(configPath string) (*Config, error) { URL: "https://gitlab.com", Token: "", PullMethod: "http", - GroupIDs: []uint64{9970}, + GroupIDs: []int{9970}, UserNames: []string{}, ArchivedProjectHandling: "hide", IncludeCurrentUser: true, diff --git a/config/loader_test.go b/config/loader_test.go index 0f4defa..cba0a03 100644 --- a/config/loader_test.go +++ b/config/loader_test.go @@ -24,7 +24,7 @@ func TestLoadConfig(t *testing.T) { URL: "https://example.com", Token: "12345", PullMethod: "ssh", - GroupIDs: []uint64{123}, + GroupIDs: []int{123}, UserNames: []string{"test-user"}, ArchivedProjectHandling: "hide", IncludeCurrentUser: true, @@ -147,7 +147,7 @@ func TestMakeGitlabConfig(t *testing.T) { URL: "https://gitlab.com", PullMethod: "http", Token: "", - GroupIDs: []uint64{9970}, + GroupIDs: []int{9970}, UserNames: []string{}, ArchivedProjectHandling: "hide", IncludeCurrentUser: true, @@ -157,7 +157,7 @@ func TestMakeGitlabConfig(t *testing.T) { URL: "https://gitlab.com", PullMethod: "http", Token: "", - GroupIDs: []uint64{9970}, + GroupIDs: []int{9970}, UserNames: []string{}, ArchivedProjectHandling: "hide", IncludeCurrentUser: true, @@ -172,7 +172,7 @@ func TestMakeGitlabConfig(t *testing.T) { URL: "https://gitlab.com", PullMethod: "invalid", Token: "", - GroupIDs: []uint64{9970}, + GroupIDs: []int{9970}, UserNames: []string{}, ArchivedProjectHandling: "hide", IncludeCurrentUser: true, @@ -186,7 +186,7 @@ func TestMakeGitlabConfig(t *testing.T) { URL: "https://gitlab.com", PullMethod: "http", Token: "", - GroupIDs: []uint64{9970}, + GroupIDs: []int{9970}, UserNames: []string{}, IncludeCurrentUser: true, ArchivedProjectHandling: "invalid", diff --git a/forges/gitlab/client.go b/forges/gitlab/client.go index 18e71e8..f9990a2 100644 --- a/forges/gitlab/client.go +++ b/forges/gitlab/client.go @@ -83,8 +83,8 @@ func (c *gitlabClient) FetchRootGroupContent(ctx context.Context) (map[string]ty func (c *gitlabClient) FetchGroupContent(ctx context.Context, source types.GroupSource) (types.GroupContent, error) { if _, found := c.users[source.GetGroupPath()]; found { - return c.fetchUserContent(ctx, source.GetGroupID()) + return c.fetchUserContent(ctx, int(source.GetGroupID())) } else { - return c.fetchGroupContent(ctx, source.GetGroupID()) + return c.fetchGroupContent(ctx, int(source.GetGroupID())) } } diff --git a/forges/gitlab/group.go b/forges/gitlab/group.go index 48b71c9..86d0ba4 100644 --- a/forges/gitlab/group.go +++ b/forges/gitlab/group.go @@ -34,7 +34,7 @@ func (c *gitlabClient) newGroupFromGitlabGroup(gitlabGroup *gitlab.Group) *Group } } -func (c *gitlabClient) fetchGroup(ctx context.Context, gid uint64) (*Group, error) { +func (c *gitlabClient) fetchGroup(ctx context.Context, gid int) (*Group, error) { gitlabGroup, _, err := c.client.Groups.GetGroup(gid, &gitlab.GetGroupOptions{}) if err != nil { return nil, fmt.Errorf("failed to fetch group with id %v: %v", gid, err) @@ -43,7 +43,7 @@ func (c *gitlabClient) fetchGroup(ctx context.Context, gid uint64) (*Group, erro return c.newGroupFromGitlabGroup(gitlabGroup), nil } -func (c *gitlabClient) fetchGroupContent(ctx context.Context, gid uint64) (types.GroupContent, error) { +func (c *gitlabClient) fetchGroupContent(ctx context.Context, gid int) (types.GroupContent, error) { childGroups := make(map[string]types.GroupSource) childProjects := make(map[string]types.RepositorySource) diff --git a/forges/gitlab/project.go b/forges/gitlab/project.go index 9dec86d..260db42 100644 --- a/forges/gitlab/project.go +++ b/forges/gitlab/project.go @@ -38,7 +38,8 @@ func (c *gitlabClient) newProjectFromGitlabProject(project *gitlab.Project) *Pro } p := Project{ ID: project.ID, - Path: project.Path, + Name: project.Name, + Path: project.PathWithNamespace, DefaultBranch: project.DefaultBranch, } if p.DefaultBranch == "" { diff --git a/forges/gitlab/user.go b/forges/gitlab/user.go index 0a26dcf..8cdfe9c 100644 --- a/forges/gitlab/user.go +++ b/forges/gitlab/user.go @@ -32,7 +32,7 @@ func (c *gitlabClient) fetchUser(ctx context.Context, uid int) (*User, error) { }, nil } -func (c *gitlabClient) fetchUserContent(ctx context.Context, uid uint64) (types.GroupContent, error) { +func (c *gitlabClient) fetchUserContent(ctx context.Context, uid int) (types.GroupContent, error) { childProjects := make(map[string]types.RepositorySource) // Fetch the user repositories diff --git a/main.go b/main.go index 05c17b5..bb50a47 100644 --- a/main.go +++ b/main.go @@ -105,7 +105,7 @@ func main() { } // setup cache - cache := cache.NewForgeCache(gitForgeClient) + cache := cache.NewForgeCache(gitForgeClient, logger) // Start the filesystem err = fstree.Start( From 08dccaeee2789eeacdbd526698c67933a83d6c10 Mon Sep 17 00:00:00 2001 From: Massaki Archambault Date: Wed, 24 Dec 2025 15:37:47 -0500 Subject: [PATCH 3/8] make group and repository interfaces expose the path and the name --- cache/cache.go | 21 +++++++++++---------- forges/gitea/client.go | 6 +++--- forges/gitea/organization.go | 14 +++++++++----- forges/gitea/repository.go | 9 ++++++++- forges/gitea/user.go | 16 ++++++++++------ forges/github/client.go | 6 +++--- forges/github/organization.go | 14 +++++++++----- forges/github/repository.go | 9 ++++++++- forges/github/user.go | 16 ++++++++++------ forges/gitlab/client.go | 6 +++--- forges/gitlab/group.go | 10 +++++----- forges/gitlab/project.go | 5 +++++ forges/gitlab/user.go | 12 ++++++++---- fstree/group.go | 4 ++-- fstree/refresh.go | 4 ++-- types/types.go | 16 +++++++++++----- 16 files changed, 107 insertions(+), 61 deletions(-) diff --git a/cache/cache.go b/cache/cache.go index 210d3a8..0b6f0f4 100644 --- a/cache/cache.go +++ b/cache/cache.go @@ -14,7 +14,7 @@ type Cache struct { logger *slog.Logger rootContentLock sync.RWMutex - cachedRootContent map[string]types.GroupSource + cachedRootContent map[string]types.RepositoryGroupSource contentLock sync.RWMutex cachedContent map[string]CachedContent @@ -30,11 +30,11 @@ func NewForgeCache(backend types.GitForge, logger *slog.Logger) types.GitForge { } type CachedContent struct { - types.GroupContent + types.RepositoryGroupContent creationTime time.Time } -func (c *Cache) FetchRootGroupContent(ctx context.Context) (map[string]types.GroupSource, error) { +func (c *Cache) FetchRootGroupContent(ctx context.Context) (map[string]types.RepositoryGroupSource, error) { c.rootContentLock.RLock() if c.cachedRootContent == nil { c.rootContentLock.RUnlock() @@ -59,7 +59,8 @@ func (c *Cache) FetchRootGroupContent(ctx context.Context) (map[string]types.Gro return c.cachedRootContent, nil } -func (c *Cache) FetchGroupContent(ctx context.Context, source types.GroupSource) (types.GroupContent, error) { +// TODO: improve locking strategy +func (c *Cache) FetchGroupContent(ctx context.Context, source types.RepositoryGroupSource) (types.RepositoryGroupContent, error) { logger := c.logger.With("groupID", source.GetGroupID()).With("groupPath", source.GetGroupPath()) c.contentLock.RLock() @@ -74,24 +75,24 @@ func (c *Cache) FetchGroupContent(ctx context.Context, source types.GroupSource) // read the map again to make sure the data is still not there if cachedContent, found := c.cachedContent[source.GetGroupPath()]; found { - return cachedContent.GroupContent, nil + return cachedContent.RepositoryGroupContent, nil } // fetch content from backend and cache it logger.Info("Fetching content from backend") content, err := c.backend.FetchGroupContent(ctx, source) if err != nil { - return types.GroupContent{}, err + return types.RepositoryGroupContent{}, err } - c.cachedContent[source.GetGroupPath()] = CachedContent{ - GroupContent: content, - creationTime: time.Now(), + c.cachedContent[source.GetGroupName()] = CachedContent{ + RepositoryGroupContent: content, + creationTime: time.Now(), } return content, nil } else { c.contentLock.RUnlock() logger.Debug("Cache hit") - return cachedContent.GroupContent, nil + return cachedContent.RepositoryGroupContent, nil } } diff --git a/forges/gitea/client.go b/forges/gitea/client.go index 1b40645..350b47e 100644 --- a/forges/gitea/client.go +++ b/forges/gitea/client.go @@ -46,8 +46,8 @@ func NewClient(logger *slog.Logger, config config.GiteaClientConfig) (*giteaClie return giteaClient, nil } -func (c *giteaClient) FetchRootGroupContent(ctx context.Context) (map[string]types.GroupSource, error) { - rootContent := make(map[string]types.GroupSource) +func (c *giteaClient) FetchRootGroupContent(ctx context.Context) (map[string]types.RepositoryGroupSource, error) { + rootContent := make(map[string]types.RepositoryGroupSource) for _, orgName := range c.GiteaClientConfig.OrgNames { org, err := c.fetchOrganization(ctx, orgName) @@ -71,7 +71,7 @@ func (c *giteaClient) FetchRootGroupContent(ctx context.Context) (map[string]typ return rootContent, nil } -func (c *giteaClient) FetchGroupContent(ctx context.Context, source types.GroupSource) (types.GroupContent, error) { +func (c *giteaClient) FetchGroupContent(ctx context.Context, source types.RepositoryGroupSource) (types.RepositoryGroupContent, error) { if _, found := c.users[source.GetGroupPath()]; found { return c.fetchUserContent(ctx, source.GetGroupPath()) } else { diff --git a/forges/gitea/organization.go b/forges/gitea/organization.go index 63ed10f..7649767 100644 --- a/forges/gitea/organization.go +++ b/forges/gitea/organization.go @@ -17,6 +17,10 @@ func (o *Organization) GetGroupID() uint64 { return uint64(o.ID) } +func (o *Organization) GetGroupName() string { + return o.Name +} + func (o *Organization) GetGroupPath() string { return o.Name } @@ -34,10 +38,10 @@ func (c *giteaClient) fetchOrganization(ctx context.Context, orgName string) (*O return &newOrg, nil } -func (c *giteaClient) fetchOrganizationContent(ctx context.Context, orgName string) (types.GroupContent, error) { +func (c *giteaClient) fetchOrganizationContent(ctx context.Context, orgName string) (types.RepositoryGroupContent, error) { org, err := c.fetchOrganization(ctx, orgName) if err != nil { - return types.GroupContent{}, err + return types.RepositoryGroupContent{}, err } repositories := make(map[string]types.RepositorySource) @@ -49,7 +53,7 @@ func (c *giteaClient) fetchOrganizationContent(ctx context.Context, orgName stri for { giteaRepositories, response, err := c.client.ListOrgRepos(org.Name, gitea.ListOrgReposOptions(listReposOptions)) if err != nil { - return types.GroupContent{}, fmt.Errorf("failed to fetch repository in gitea: %v", err) + return types.RepositoryGroupContent{}, fmt.Errorf("failed to fetch repository in gitea: %v", err) } for _, giteaRepository := range giteaRepositories { repository := c.newRepositoryFromGiteaRepository(giteaRepository) @@ -64,8 +68,8 @@ func (c *giteaClient) fetchOrganizationContent(ctx context.Context, orgName stri listReposOptions.Page = response.NextPage } - return types.GroupContent{ - Groups: make(map[string]types.GroupSource), + return types.RepositoryGroupContent{ + Groups: make(map[string]types.RepositoryGroupSource), Repositories: repositories, }, nil } diff --git a/forges/gitea/repository.go b/forges/gitea/repository.go index 790324f..2c9be2d 100644 --- a/forges/gitea/repository.go +++ b/forges/gitea/repository.go @@ -9,6 +9,7 @@ import ( type Repository struct { ID int64 + Name string Path string CloneURL string DefaultBranch string @@ -19,6 +20,10 @@ func (r *Repository) GetRepositoryID() uint64 { } func (r *Repository) GetRepositoryName() string { + return r.Name +} + +func (r *Repository) GetRepositoryPath() string { return r.Path } @@ -36,7 +41,8 @@ func (c *giteaClient) newRepositoryFromGiteaRepository(repository *gitea.Reposit } r := Repository{ ID: repository.ID, - Path: repository.Name, + Name: repository.Name, + Path: repository.FullName, DefaultBranch: repository.DefaultBranch, } if r.DefaultBranch == "" { @@ -48,6 +54,7 @@ func (c *giteaClient) newRepositoryFromGiteaRepository(repository *gitea.Reposit r.CloneURL = repository.CloneURL } if c.ArchivedRepoHandling == config.ArchivedProjectHide && repository.Archived { + r.Name = "." + r.Name r.Path = path.Join(path.Dir(r.Path), "."+path.Base(r.Path)) } return &r diff --git a/forges/gitea/user.go b/forges/gitea/user.go index dea98f3..9e46baf 100644 --- a/forges/gitea/user.go +++ b/forges/gitea/user.go @@ -17,6 +17,10 @@ func (u *User) GetGroupID() uint64 { return uint64(u.ID) } +func (u *User) GetGroupName() string { + return u.Name +} + func (u *User) GetGroupPath() string { return u.Name } @@ -34,10 +38,10 @@ func (c *giteaClient) fetchUser(ctx context.Context, userName string) (*User, er return &newUser, nil } -func (c *giteaClient) fetchUserContent(ctx context.Context, userName string) (types.GroupContent, error) { +func (c *giteaClient) fetchUserContent(ctx context.Context, userName string) (types.RepositoryGroupContent, error) { user, err := c.fetchUser(ctx, userName) if err != nil { - return types.GroupContent{}, err + return types.RepositoryGroupContent{}, err } repositories := make(map[string]types.RepositorySource) @@ -49,12 +53,12 @@ func (c *giteaClient) fetchUserContent(ctx context.Context, userName string) (ty for { giteaRepositories, response, err := c.client.ListUserRepos(user.Name, listReposOptions) if err != nil { - return types.GroupContent{}, fmt.Errorf("failed to fetch repository in gitea: %v", err) + return types.RepositoryGroupContent{}, fmt.Errorf("failed to fetch repository in gitea: %v", err) } for _, giteaRepository := range giteaRepositories { repository := c.newRepositoryFromGiteaRepository(giteaRepository) if repository != nil { - repositories[repository.Path] = repository + repositories[repository.GetRepositoryName()] = repository } } if response.NextPage == 0 { @@ -64,8 +68,8 @@ func (c *giteaClient) fetchUserContent(ctx context.Context, userName string) (ty listReposOptions.Page = response.NextPage } - return types.GroupContent{ - Groups: make(map[string]types.GroupSource), + return types.RepositoryGroupContent{ + Groups: make(map[string]types.RepositoryGroupSource), Repositories: repositories, }, nil } diff --git a/forges/github/client.go b/forges/github/client.go index 498967a..5f08ba4 100644 --- a/forges/github/client.go +++ b/forges/github/client.go @@ -45,8 +45,8 @@ func NewClient(logger *slog.Logger, config config.GithubClientConfig) (*githubCl return gitHubClient, nil } -func (c *githubClient) FetchRootGroupContent(ctx context.Context) (map[string]types.GroupSource, error) { - rootContent := make(map[string]types.GroupSource) +func (c *githubClient) FetchRootGroupContent(ctx context.Context) (map[string]types.RepositoryGroupSource, error) { + rootContent := make(map[string]types.RepositoryGroupSource) for _, orgName := range c.GithubClientConfig.OrgNames { org, err := c.fetchOrganization(ctx, orgName) @@ -70,7 +70,7 @@ func (c *githubClient) FetchRootGroupContent(ctx context.Context) (map[string]ty return rootContent, nil } -func (c *githubClient) FetchGroupContent(ctx context.Context, source types.GroupSource) (types.GroupContent, error) { +func (c *githubClient) FetchGroupContent(ctx context.Context, source types.RepositoryGroupSource) (types.RepositoryGroupContent, error) { if _, found := c.users[source.GetGroupPath()]; found { return c.fetchUserContent(ctx, source.GetGroupPath()) } else { diff --git a/forges/github/organization.go b/forges/github/organization.go index 59c2054..1af8588 100644 --- a/forges/github/organization.go +++ b/forges/github/organization.go @@ -17,6 +17,10 @@ func (o *Organization) GetGroupID() uint64 { return uint64(o.ID) } +func (o *Organization) GetGroupName() string { + return o.Name +} + func (o *Organization) GetGroupPath() string { return o.Name } @@ -32,10 +36,10 @@ func (c *githubClient) fetchOrganization(ctx context.Context, orgName string) (* }, nil } -func (c *githubClient) fetchOrganizationContent(ctx context.Context, orgName string) (types.GroupContent, error) { +func (c *githubClient) fetchOrganizationContent(ctx context.Context, orgName string) (types.RepositoryGroupContent, error) { org, err := c.fetchOrganization(ctx, orgName) if err != nil { - return types.GroupContent{}, err + return types.RepositoryGroupContent{}, err } repositories := make(map[string]types.RepositorySource) @@ -47,7 +51,7 @@ func (c *githubClient) fetchOrganizationContent(ctx context.Context, orgName str for { githubRepositories, response, err := c.client.Repositories.ListByOrg(ctx, org.Name, repositoryListOpt) if err != nil { - return types.GroupContent{}, fmt.Errorf("failed to fetch repository in github: %v", err) + return types.RepositoryGroupContent{}, fmt.Errorf("failed to fetch repository in github: %v", err) } for _, githubRepository := range githubRepositories { repository := c.newRepositoryFromGithubRepository(githubRepository) @@ -62,8 +66,8 @@ func (c *githubClient) fetchOrganizationContent(ctx context.Context, orgName str repositoryListOpt.Page = response.NextPage } - return types.GroupContent{ - Groups: make(map[string]types.GroupSource), + return types.RepositoryGroupContent{ + Groups: make(map[string]types.RepositoryGroupSource), Repositories: repositories, }, nil } diff --git a/forges/github/repository.go b/forges/github/repository.go index 8fc120d..61ef5a1 100644 --- a/forges/github/repository.go +++ b/forges/github/repository.go @@ -9,6 +9,7 @@ import ( type Repository struct { ID int64 + Name string Path string CloneURL string DefaultBranch string @@ -19,6 +20,10 @@ func (r *Repository) GetRepositoryID() uint64 { } func (r *Repository) GetRepositoryName() string { + return r.Name +} + +func (r *Repository) GetRepositoryPath() string { return r.Path } @@ -36,7 +41,8 @@ func (c *githubClient) newRepositoryFromGithubRepository(repository *github.Repo } r := Repository{ ID: *repository.ID, - Path: *repository.Name, + Name: *repository.Name, + Path: *repository.FullName, DefaultBranch: *repository.DefaultBranch, } if r.DefaultBranch == "" { @@ -48,6 +54,7 @@ func (c *githubClient) newRepositoryFromGithubRepository(repository *github.Repo r.CloneURL = *repository.CloneURL } if c.ArchivedRepoHandling == config.ArchivedProjectHide && *repository.Archived { + r.Name = "." + r.Name r.Path = path.Join(path.Dir(r.Path), "."+path.Base(r.Path)) } return &r diff --git a/forges/github/user.go b/forges/github/user.go index 9cfc16f..1910a51 100644 --- a/forges/github/user.go +++ b/forges/github/user.go @@ -17,6 +17,10 @@ func (u *User) GetGroupID() uint64 { return uint64(u.ID) } +func (u *User) GetGroupName() string { + return u.Name +} + func (u *User) GetGroupPath() string { return u.Name } @@ -32,10 +36,10 @@ func (c *githubClient) fetchUser(ctx context.Context, userName string) (*User, e }, nil } -func (c *githubClient) fetchUserContent(ctx context.Context, userName string) (types.GroupContent, error) { +func (c *githubClient) fetchUserContent(ctx context.Context, userName string) (types.RepositoryGroupContent, error) { user, err := c.fetchUser(ctx, userName) if err != nil { - return types.GroupContent{}, err + return types.RepositoryGroupContent{}, err } repositories := make(map[string]types.RepositorySource) @@ -47,12 +51,12 @@ func (c *githubClient) fetchUserContent(ctx context.Context, userName string) (t for { githubRepositories, response, err := c.client.Repositories.ListByUser(ctx, user.Name, repositoryListOpt) if err != nil { - return types.GroupContent{}, fmt.Errorf("failed to fetch repository in github: %v", err) + return types.RepositoryGroupContent{}, fmt.Errorf("failed to fetch repository in github: %v", err) } for _, githubRepository := range githubRepositories { repository := c.newRepositoryFromGithubRepository(githubRepository) if repository != nil { - repositories[repository.Path] = repository + repositories[repository.GetRepositoryName()] = repository } } if response.NextPage == 0 { @@ -62,8 +66,8 @@ func (c *githubClient) fetchUserContent(ctx context.Context, userName string) (t repositoryListOpt.Page = response.NextPage } - return types.GroupContent{ - Groups: make(map[string]types.GroupSource), + return types.RepositoryGroupContent{ + Groups: make(map[string]types.RepositoryGroupSource), Repositories: repositories, }, nil } diff --git a/forges/gitlab/client.go b/forges/gitlab/client.go index f9990a2..52e4eeb 100644 --- a/forges/gitlab/client.go +++ b/forges/gitlab/client.go @@ -59,8 +59,8 @@ func NewClient(logger *slog.Logger, config config.GitlabClientConfig) (*gitlabCl return gitlabClient, nil } -func (c *gitlabClient) FetchRootGroupContent(ctx context.Context) (map[string]types.GroupSource, error) { - rootContent := make(map[string]types.GroupSource) +func (c *gitlabClient) FetchRootGroupContent(ctx context.Context) (map[string]types.RepositoryGroupSource, error) { + rootContent := make(map[string]types.RepositoryGroupSource) // fetch root groups for _, gid := range c.GroupIDs { @@ -81,7 +81,7 @@ func (c *gitlabClient) FetchRootGroupContent(ctx context.Context) (map[string]ty return rootContent, nil } -func (c *gitlabClient) FetchGroupContent(ctx context.Context, source types.GroupSource) (types.GroupContent, error) { +func (c *gitlabClient) FetchGroupContent(ctx context.Context, source types.RepositoryGroupSource) (types.RepositoryGroupContent, error) { if _, found := c.users[source.GetGroupPath()]; found { return c.fetchUserContent(ctx, int(source.GetGroupID())) } else { diff --git a/forges/gitlab/group.go b/forges/gitlab/group.go index 86d0ba4..cb863b6 100644 --- a/forges/gitlab/group.go +++ b/forges/gitlab/group.go @@ -43,8 +43,8 @@ func (c *gitlabClient) fetchGroup(ctx context.Context, gid int) (*Group, error) return c.newGroupFromGitlabGroup(gitlabGroup), nil } -func (c *gitlabClient) fetchGroupContent(ctx context.Context, gid int) (types.GroupContent, error) { - childGroups := make(map[string]types.GroupSource) +func (c *gitlabClient) fetchGroupContent(ctx context.Context, gid int) (types.RepositoryGroupContent, error) { + childGroups := make(map[string]types.RepositoryGroupSource) childProjects := make(map[string]types.RepositorySource) // List subgroups in path @@ -58,7 +58,7 @@ func (c *gitlabClient) fetchGroupContent(ctx context.Context, gid int) (types.Gr for { gitlabGroups, response, err := c.client.Groups.ListSubGroups(gid, listGroupsOpt) if err != nil { - return types.GroupContent{}, fmt.Errorf("failed to fetch groups in gitlab: %v", err) + return types.RepositoryGroupContent{}, fmt.Errorf("failed to fetch groups in gitlab: %v", err) } for _, gitlabGroup := range gitlabGroups { group := c.newGroupFromGitlabGroup(gitlabGroup) @@ -82,7 +82,7 @@ func (c *gitlabClient) fetchGroupContent(ctx context.Context, gid int) (types.Gr for { gitlabProjects, response, err := c.client.Groups.ListGroupProjects(gid, listProjectOpt) if err != nil { - return types.GroupContent{}, fmt.Errorf("failed to fetch projects in gitlab: %v", err) + return types.RepositoryGroupContent{}, fmt.Errorf("failed to fetch projects in gitlab: %v", err) } for _, gitlabProject := range gitlabProjects { project := c.newProjectFromGitlabProject(gitlabProject) @@ -96,7 +96,7 @@ func (c *gitlabClient) fetchGroupContent(ctx context.Context, gid int) (types.Gr // Get the next page listProjectOpt.Page = response.NextPage } - return types.GroupContent{ + return types.RepositoryGroupContent{ Groups: childGroups, Repositories: childProjects, }, nil diff --git a/forges/gitlab/project.go b/forges/gitlab/project.go index 260db42..f0e2544 100644 --- a/forges/gitlab/project.go +++ b/forges/gitlab/project.go @@ -23,6 +23,10 @@ func (p *Project) GetRepositoryName() string { return p.Name } +func (p *Project) GetRepositoryPath() string { + return p.Path +} + func (p *Project) GetCloneURL() string { return p.CloneURL } @@ -51,6 +55,7 @@ func (c *gitlabClient) newProjectFromGitlabProject(project *gitlab.Project) *Pro p.CloneURL = project.HTTPURLToRepo } if c.ArchivedProjectHandling == config.ArchivedProjectHide && project.Archived { + p.Name = "." + p.Name p.Path = path.Join(path.Dir(p.Path), "."+path.Base(p.Path)) } return &p diff --git a/forges/gitlab/user.go b/forges/gitlab/user.go index 8cdfe9c..51e07f5 100644 --- a/forges/gitlab/user.go +++ b/forges/gitlab/user.go @@ -17,6 +17,10 @@ func (u *User) GetGroupID() uint64 { return uint64(u.ID) } +func (u *User) GetGroupName() string { + return u.Name +} + func (u *User) GetGroupPath() string { return u.Name } @@ -32,7 +36,7 @@ func (c *gitlabClient) fetchUser(ctx context.Context, uid int) (*User, error) { }, nil } -func (c *gitlabClient) fetchUserContent(ctx context.Context, uid int) (types.GroupContent, error) { +func (c *gitlabClient) fetchUserContent(ctx context.Context, uid int) (types.RepositoryGroupContent, error) { childProjects := make(map[string]types.RepositorySource) // Fetch the user repositories @@ -44,7 +48,7 @@ func (c *gitlabClient) fetchUserContent(ctx context.Context, uid int) (types.Gro for { gitlabProjects, response, err := c.client.Projects.ListUserProjects(uid, listProjectOpt) if err != nil { - return types.GroupContent{}, fmt.Errorf("failed to fetch projects in gitlab: %v", err) + return types.RepositoryGroupContent{}, fmt.Errorf("failed to fetch projects in gitlab: %v", err) } for _, gitlabProject := range gitlabProjects { project := c.newProjectFromGitlabProject(gitlabProject) @@ -58,8 +62,8 @@ func (c *gitlabClient) fetchUserContent(ctx context.Context, uid int) (types.Gro // Get the next page listProjectOpt.Page = response.NextPage } - return types.GroupContent{ - Groups: make(map[string]types.GroupSource), + return types.RepositoryGroupContent{ + Groups: make(map[string]types.RepositoryGroupSource), Repositories: childProjects, }, nil } diff --git a/fstree/group.go b/fstree/group.go index 81cc1fa..cfb5313 100644 --- a/fstree/group.go +++ b/fstree/group.go @@ -17,7 +17,7 @@ type groupNode struct { fs.Inode param *FSParam - source types.GroupSource + source types.RepositoryGroupSource staticNodes map[string]staticNode } @@ -27,7 +27,7 @@ var _ = (fs.NodeReaddirer)((*groupNode)(nil)) // Ensure we are implementing the NodeLookuper interface var _ = (fs.NodeLookuper)((*groupNode)(nil)) -func newGroupNodeFromSource(ctx context.Context, source types.GroupSource, param *FSParam) (fs.InodeEmbedder, error) { +func newGroupNodeFromSource(ctx context.Context, source types.RepositoryGroupSource, param *FSParam) (fs.InodeEmbedder, error) { node := &groupNode{ param: param, source: source, diff --git a/fstree/refresh.go b/fstree/refresh.go index e52ae4b..9f6468e 100644 --- a/fstree/refresh.go +++ b/fstree/refresh.go @@ -13,7 +13,7 @@ type refreshNode struct { fs.Inode ino uint64 - source types.GroupSource + source types.RepositoryGroupSource } // Ensure we are implementing the NodeSetattrer interface @@ -22,7 +22,7 @@ var _ = (fs.NodeSetattrer)((*refreshNode)(nil)) // Ensure we are implementing the NodeOpener interface var _ = (fs.NodeOpener)((*refreshNode)(nil)) -func newRefreshNode(source types.GroupSource, param *FSParam) *refreshNode { +func newRefreshNode(source types.RepositoryGroupSource, param *FSParam) *refreshNode { return &refreshNode{ ino: 0, source: source, diff --git a/types/types.go b/types/types.go index 320e571..a9aab03 100644 --- a/types/types.go +++ b/types/types.go @@ -5,23 +5,29 @@ import ( ) type GitForge interface { - FetchRootGroupContent(ctx context.Context) (map[string]GroupSource, error) - FetchGroupContent(ctx context.Context, source GroupSource) (GroupContent, error) + FetchRootGroupContent(ctx context.Context) (map[string]RepositoryGroupSource, error) + FetchGroupContent(ctx context.Context, source RepositoryGroupSource) (RepositoryGroupContent, error) } -type GroupSource interface { +type RepositoryGroupSource interface { GetGroupID() uint64 + GetGroupName() string GetGroupPath() string } type RepositorySource interface { GetRepositoryID() uint64 GetRepositoryName() string + GetRepositoryPath() string GetCloneURL() string GetDefaultBranch() string } -type GroupContent struct { - Groups map[string]GroupSource +type RepositoryGroupContent struct { + // a map of the subgroups contained in this group, keyed by group name + // must not be nil + Groups map[string]RepositoryGroupSource + // a map of the repositories contained in this group, keyed by repository name + // must not be nil Repositories map[string]RepositorySource } From c57372901f025d6970ca176c32b3feb673bb64de Mon Sep 17 00:00:00 2001 From: Massaki Archambault Date: Wed, 24 Dec 2025 16:09:34 -0500 Subject: [PATCH 4/8] add unit tests for cache --- cache/cache.go | 4 +- cache/cache_test.go | 142 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 144 insertions(+), 2 deletions(-) create mode 100644 cache/cache_test.go diff --git a/cache/cache.go b/cache/cache.go index 0b6f0f4..6fab6b2 100644 --- a/cache/cache.go +++ b/cache/cache.go @@ -20,7 +20,7 @@ type Cache struct { cachedContent map[string]CachedContent } -func NewForgeCache(backend types.GitForge, logger *slog.Logger) types.GitForge { +func NewForgeCache(backend types.GitForge, logger *slog.Logger) *Cache { return &Cache{ backend: backend, logger: logger, @@ -84,7 +84,7 @@ func (c *Cache) FetchGroupContent(ctx context.Context, source types.RepositoryGr if err != nil { return types.RepositoryGroupContent{}, err } - c.cachedContent[source.GetGroupName()] = CachedContent{ + c.cachedContent[source.GetGroupPath()] = CachedContent{ RepositoryGroupContent: content, creationTime: time.Now(), } diff --git a/cache/cache_test.go b/cache/cache_test.go new file mode 100644 index 0000000..7066091 --- /dev/null +++ b/cache/cache_test.go @@ -0,0 +1,142 @@ +package cache_test + +import ( + "context" + "io" + "log/slog" + "testing" + + "github.com/badjware/gitforgefs/cache" + "github.com/badjware/gitforgefs/types" +) + +type mockRepoGroupSource struct { + ID uint64 + Name string + Path string +} + +func (m mockRepoGroupSource) GetGroupID() uint64 { return m.ID } +func (m mockRepoGroupSource) GetGroupName() string { return m.Name } +func (m mockRepoGroupSource) GetGroupPath() string { return m.Path } + +type mockBackend struct { + RootCalls int + GroupCalls int + + rootContentValue map[string]types.RepositoryGroupSource + groupContentValue types.RepositoryGroupContent +} + +func (m *mockBackend) FetchRootGroupContent(ctx context.Context) (map[string]types.RepositoryGroupSource, error) { + m.RootCalls++ + return m.rootContentValue, nil +} + +func (m *mockBackend) FetchGroupContent(ctx context.Context, source types.RepositoryGroupSource) (types.RepositoryGroupContent, error) { + m.GroupCalls++ + return m.groupContentValue, nil +} + +func newLogger() *slog.Logger { + return slog.New(slog.NewTextHandler(io.Discard, &slog.HandlerOptions{})) +} + +func TestFetchRootGroupContent(t *testing.T) { + backend := &mockBackend{ + rootContentValue: map[string]types.RepositoryGroupSource{ + "g": mockRepoGroupSource{ID: 1, Name: "g", Path: "g"}, + }, + } + + logger := newLogger() + c := cache.NewForgeCache(backend, logger) + + ctx := context.Background() + + result1, err := c.FetchRootGroupContent(ctx) + if err != nil { + t.Fatalf("first FetchRootGroupContent failed: %v", err) + } + if content, ok := result1["g"]; !ok || content.GetGroupID() != 1 { + t.Fatalf("unexpected root content fetched: %v", result1) + } + + result2, err := c.FetchRootGroupContent(ctx) + if err != nil { + t.Fatalf("second FetchRootGroupContent failed: %v", err) + } + if content, ok := result2["g"]; !ok || content.GetGroupID() != 1 { + t.Fatalf("unexpected root content fetched: %v", result2) + } + + if backend.RootCalls != 1 { + t.Fatalf("expected backend.FetchRootGroupContent to be called once, got %d", backend.RootCalls) + } +} + +func TestFetchGroupContent(t *testing.T) { + backend := &mockBackend{ + groupContentValue: types.RepositoryGroupContent{ + Groups: map[string]types.RepositoryGroupSource{"group2": mockRepoGroupSource{ID: 2, Name: "group2", Path: "path/group1/group2"}}, + Repositories: map[string]types.RepositorySource{}, + }, + } + + logger := newLogger() + c := cache.NewForgeCache(backend, logger) + + src := mockRepoGroupSource{ID: 1, Name: "group1", Path: "path/group1"} + + ctx := context.Background() + + result1, err := c.FetchGroupContent(ctx, src) + if err != nil { + t.Fatalf("first FetchGroupContent failed: %v", err) + } + if content, ok := result1.Groups["group2"]; !ok || content.GetGroupID() != 2 { + t.Fatalf("unexpected group content fetched: %v", result1) + } + + result2, err := c.FetchGroupContent(ctx, src) + if err != nil { + t.Fatalf("second FetchGroupContent failed: %v", err) + } + if content, ok := result2.Groups["group2"]; !ok || content.GetGroupID() != 2 { + t.Fatalf("unexpected group content fetched: %v", result2) + } + + if backend.GroupCalls != 1 { + t.Fatalf("expected backend.FetchGroupContent to be called once, got %d", backend.GroupCalls) + } +} + +func TestInvalidateCache(t *testing.T) { + backend := &mockBackend{ + groupContentValue: types.RepositoryGroupContent{ + Groups: map[string]types.RepositoryGroupSource{"group2": mockRepoGroupSource{ID: 2, Name: "group2", Path: "path/group1/group2"}}, + Repositories: map[string]types.RepositorySource{}, + }, + } + + logger := newLogger() + c := cache.NewForgeCache(backend, logger) + + src := mockRepoGroupSource{ID: 1, Name: "group1", Path: "path/group1"} + + ctx := context.Background() + + if _, err := c.FetchGroupContent(ctx, src); err != nil { + t.Fatalf("first FetchGroupContent failed: %v", err) + } + + c.InvalidateCache(src.GetGroupPath()) + + if _, err := c.FetchGroupContent(ctx, src); err != nil { + t.Fatalf("second FetchGroupContent failed: %v", err) + } + + if backend.GroupCalls != 2 { + t.Fatalf("expected backend.FetchGroupContent to be called twice, got %d", backend.GroupCalls) + } +} From acb73790105343a873df24d52a66024b6ac8febe Mon Sep 17 00:00:00 2001 From: Massaki Archambault Date: Wed, 24 Dec 2025 16:15:31 -0500 Subject: [PATCH 5/8] refactor types --- cache/cache.go | 6 +++--- cache/cache_test.go | 2 +- fstree/group.go | 4 ++-- fstree/root.go | 10 +++------- git/client.go | 2 +- main.go | 2 +- types/types.go | 9 +++++++++ 7 files changed, 20 insertions(+), 15 deletions(-) diff --git a/cache/cache.go b/cache/cache.go index 6fab6b2..65a54b6 100644 --- a/cache/cache.go +++ b/cache/cache.go @@ -20,7 +20,7 @@ type Cache struct { cachedContent map[string]CachedContent } -func NewForgeCache(backend types.GitForge, logger *slog.Logger) *Cache { +func NewForgeCache(backend types.GitForge, logger *slog.Logger) types.GitForgeCacher { return &Cache{ backend: backend, logger: logger, @@ -96,9 +96,9 @@ func (c *Cache) FetchGroupContent(ctx context.Context, source types.RepositoryGr } } -func (c *Cache) InvalidateCache(path string) { +func (c *Cache) InvalidateCache(source types.RepositoryGroupSource) { c.contentLock.Lock() defer c.contentLock.Unlock() - delete(c.cachedContent, path) + delete(c.cachedContent, source.GetGroupPath()) } diff --git a/cache/cache_test.go b/cache/cache_test.go index 7066091..b35ab7a 100644 --- a/cache/cache_test.go +++ b/cache/cache_test.go @@ -130,7 +130,7 @@ func TestInvalidateCache(t *testing.T) { t.Fatalf("first FetchGroupContent failed: %v", err) } - c.InvalidateCache(src.GetGroupPath()) + c.InvalidateCache(src) if _, err := c.FetchGroupContent(ctx, src); err != nil { t.Fatalf("second FetchGroupContent failed: %v", err) diff --git a/fstree/group.go b/fstree/group.go index cfb5313..422509d 100644 --- a/fstree/group.go +++ b/fstree/group.go @@ -39,7 +39,7 @@ func newGroupNodeFromSource(ctx context.Context, source types.RepositoryGroupSou } func (n *groupNode) Readdir(ctx context.Context) (fs.DirStream, syscall.Errno) { - content, err := n.param.GitForge.FetchGroupContent(ctx, n.source) + content, err := n.param.Backend.FetchGroupContent(ctx, n.source) if err != nil { n.param.logger.Error(err.Error()) } @@ -70,7 +70,7 @@ func (n *groupNode) Readdir(ctx context.Context) (fs.DirStream, syscall.Errno) { } func (n *groupNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) (*fs.Inode, syscall.Errno) { - content, err := n.param.GitForge.FetchGroupContent(ctx, n.source) + content, err := n.param.Backend.FetchGroupContent(ctx, n.source) if err != nil { n.param.logger.Error(err.Error()) } else { diff --git a/fstree/root.go b/fstree/root.go index a30c42a..5ec1295 100644 --- a/fstree/root.go +++ b/fstree/root.go @@ -19,15 +19,11 @@ type staticNode interface { Mode() uint32 } -type GitClient interface { - FetchLocalRepositoryPath(ctx context.Context, source types.RepositorySource) (string, error) -} - type FSParam struct { UseSymlinks bool - GitClient GitClient - GitForge types.GitForge + GitClient types.GitClient + Backend types.GitForgeCacher logger *slog.Logger } @@ -67,7 +63,7 @@ func Start(logger *slog.Logger, mountpoint string, mountoptions []string, param } func (n *rootNode) OnAdd(ctx context.Context) { - rootGroups, err := n.param.GitForge.FetchRootGroupContent(ctx) + rootGroups, err := n.param.Backend.FetchRootGroupContent(ctx) if err != nil { panic(err) } diff --git a/git/client.go b/git/client.go index 35d291f..bf060d7 100644 --- a/git/client.go +++ b/git/client.go @@ -29,7 +29,7 @@ type gitClient struct { queue queue.TaskQueue } -func NewClient(logger *slog.Logger, p config.GitClientConfig) (*gitClient, error) { +func NewClient(logger *slog.Logger, p config.GitClientConfig) (types.GitClient, error) { // Create the client c := &gitClient{ GitClientConfig: p, diff --git a/main.go b/main.go index bb50a47..fc66cba 100644 --- a/main.go +++ b/main.go @@ -115,7 +115,7 @@ func main() { &fstree.FSParam{ UseSymlinks: loadedConfig.FS.UseSymlinks, GitClient: gitClient, - GitForge: cache, + Backend: cache, }, *debug, ) diff --git a/types/types.go b/types/types.go index a9aab03..aa3ff25 100644 --- a/types/types.go +++ b/types/types.go @@ -4,11 +4,20 @@ import ( "context" ) +type GitClient interface { + FetchLocalRepositoryPath(ctx context.Context, source RepositorySource) (string, error) +} + type GitForge interface { FetchRootGroupContent(ctx context.Context) (map[string]RepositoryGroupSource, error) FetchGroupContent(ctx context.Context, source RepositoryGroupSource) (RepositoryGroupContent, error) } +type GitForgeCacher interface { + GitForge + InvalidateCache(source RepositoryGroupSource) +} + type RepositoryGroupSource interface { GetGroupID() uint64 GetGroupName() string From ae75a500f7d564fa1276c45e1225c16e353176c4 Mon Sep 17 00:00:00 2001 From: Massaki Archambault Date: Wed, 24 Dec 2025 16:17:50 -0500 Subject: [PATCH 6/8] restore manual refresh --- fstree/refresh.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/fstree/refresh.go b/fstree/refresh.go index 9f6468e..d8fcb3a 100644 --- a/fstree/refresh.go +++ b/fstree/refresh.go @@ -11,7 +11,8 @@ import ( type refreshNode struct { fs.Inode - ino uint64 + ino uint64 + param *FSParam source types.RepositoryGroupSource } @@ -26,6 +27,7 @@ func newRefreshNode(source types.RepositoryGroupSource, param *FSParam) *refresh return &refreshNode{ ino: 0, source: source, + param: param, } } @@ -42,7 +44,6 @@ func (n *refreshNode) Setattr(ctx context.Context, fh fs.FileHandle, in *fuse.Se } func (n *refreshNode) Open(ctx context.Context, flags uint32) (fh fs.FileHandle, fuseFlags uint32, errno syscall.Errno) { - // FIXME - // n.source.InvalidateContentCache() + n.param.Backend.InvalidateCache(n.source) return nil, 0, 0 } From 1ccda455cdadf06a37b7b382d76ec709b88e9c60 Mon Sep 17 00:00:00 2001 From: Massaki Archambault Date: Wed, 24 Dec 2025 16:22:31 -0500 Subject: [PATCH 7/8] upgrade dependencies --- go.sum | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/go.sum b/go.sum index e634f76..a4ee6ef 100644 --- a/go.sum +++ b/go.sum @@ -28,8 +28,12 @@ github.com/hashicorp/go-retryablehttp v0.7.8 h1:ylXZWnqa7Lhqpk0L1P1LzDtGcCR0rPVU github.com/hashicorp/go-retryablehttp v0.7.8/go.mod h1:rjiScheydd+CxvumBsIrFKlx3iS0jrZ7LvzFGFmuKbw= github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/go-version v1.8.0 h1:KAkNb1HAiZd1ukkxDFGmokVZe1Xy9HG6NUp+bPle2i4= +github.com/hashicorp/go-version v1.8.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -42,31 +46,45 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/moby/sys/mountinfo v0.7.2 h1:1shs6aH5s4o5H2zQLn796ADW1wMrIwHsyJ2v9KouLrg= github.com/moby/sys/mountinfo v0.7.2/go.mod h1:1YOa8w8Ih7uW0wALDUgT1dTTSBrZ+HiBLGws92L2RU4= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= gitlab.com/gitlab-org/api/client-go v0.159.0 h1:ibKeribio/OCsrsUz7pkgIN4E7HWDyrw/lDR6P2R7lU= gitlab.com/gitlab-org/api/client-go v0.159.0/go.mod h1:D0DHF7ILUfFo/JcoGMAEndiKMm8SiP/WjyJ4OfXxCKw= +gitlab.com/gitlab-org/api/client-go v1.10.0 h1:VlB9gXQdG6w643lH53VduUHVnCWQG5Ty86VbXnyi70A= +gitlab.com/gitlab-org/api/client-go v1.10.0/go.mod h1:U3QKvjbT1J1FrgLsA7w/XlhoBIendUqB4o3/Ht3UhEQ= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04= golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0= +golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= +golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 h1:mgKeJMpvi0yx/sU5GsxQ7p6s2wtOnGAHZWCHUM4KGzY= golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546/go.mod h1:j/pmGrbnkbPtQfxEe5D0VQhZC6qKbfKifgD0oM7sR70= +golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 h1:fQsdNF2N+/YewlRZiricy4P1iimyPKZ/xwniHj8Q2a0= +golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93/go.mod h1:EPRbTFwzwjXj9NpYyyrvenVh9Y+GFeEvMNh7Xuz7xgU= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/oauth2 v0.33.0 h1:4Q+qn+E5z8gPRJfmRy7C2gGG3T4jIprK6aSYgTXGRpo= golang.org/x/oauth2 v0.33.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= +golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= +golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= +golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.36.0 h1:zMPR+aF8gfksFprF/Nc/rd1wRS1EI6nDBGyWAvDzx2Q= golang.org/x/term v0.36.0/go.mod h1:Qu394IJq6V6dCBRgwqshf3mPF85AqzYEzofzRdZkWss= +golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= @@ -76,6 +94,8 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= From c00a557061361d46b9163ad59a657c9b7d322785 Mon Sep 17 00:00:00 2001 From: Massaki Archambault Date: Wed, 24 Dec 2025 16:24:50 -0500 Subject: [PATCH 8/8] update changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d39848b..6c77db0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +# v1.1.1 +* Fix a crash when `archived_project_handling` is set to "hidden" in gitlab +* Refactored caching, laying the groundwork for automatic cache invalidation. + # v1.1.0 * Now default to using a loopback instead of symlinks. This should improve compatibility.