diff --git a/.vscode/launch.json b/.vscode/launch.json index 5500823..1e3f28e 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -10,8 +10,9 @@ "request": "launch", "mode": "debug", "program": "${workspaceRoot}", + "buildFlags": "-race", "args": [ - "-debug", + // "-debug", "-config", "${workspaceRoot}/config.yaml" ] diff --git a/CHANGELOG.md b/CHANGELOG.md index d39848b..6c77db0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +# v1.1.1 +* Fix a crash when `archived_project_handling` is set to "hidden" in gitlab +* Refactored caching, laying the groundwork for automatic cache invalidation. + # v1.1.0 * Now default to using a loopback instead of symlinks. This should improve compatibility. diff --git a/cache/cache.go b/cache/cache.go new file mode 100644 index 0000000..65a54b6 --- /dev/null +++ b/cache/cache.go @@ -0,0 +1,104 @@ +package cache + +import ( + "context" + "log/slog" + "sync" + "time" + + "github.com/badjware/gitforgefs/types" +) + +type Cache struct { + backend types.GitForge + logger *slog.Logger + + rootContentLock sync.RWMutex + cachedRootContent map[string]types.RepositoryGroupSource + + contentLock sync.RWMutex + cachedContent map[string]CachedContent +} + +func NewForgeCache(backend types.GitForge, logger *slog.Logger) types.GitForgeCacher { + return &Cache{ + backend: backend, + logger: logger, + + cachedContent: map[string]CachedContent{}, + } +} + +type CachedContent struct { + types.RepositoryGroupContent + creationTime time.Time +} + +func (c *Cache) FetchRootGroupContent(ctx context.Context) (map[string]types.RepositoryGroupSource, error) { + c.rootContentLock.RLock() + if c.cachedRootContent == nil { + c.rootContentLock.RUnlock() + + // acquire write lock + c.rootContentLock.Lock() + defer c.rootContentLock.Unlock() + + // check to make sure the data is still not there, + // since RWMutex is not upgradeable and another thread may have grabbed the lock in the meantime + if c.cachedRootContent == nil { + c.logger.Info("Fetching root content from backend") + content, err := c.backend.FetchRootGroupContent(ctx) + if err != nil { + return nil, err + } + c.cachedRootContent = content + } + return c.cachedRootContent, nil + } + c.rootContentLock.RUnlock() + return c.cachedRootContent, nil +} + +// TODO: improve locking strategy +func (c *Cache) FetchGroupContent(ctx context.Context, source types.RepositoryGroupSource) (types.RepositoryGroupContent, error) { + logger := c.logger.With("groupID", source.GetGroupID()).With("groupPath", source.GetGroupPath()) + + c.contentLock.RLock() + if cachedContent, found := c.cachedContent[source.GetGroupPath()]; !found { + c.contentLock.RUnlock() + + logger.Debug("Cache miss") + + // acquire write lock + c.contentLock.Lock() + defer c.contentLock.Unlock() + + // read the map again to make sure the data is still not there + if cachedContent, found := c.cachedContent[source.GetGroupPath()]; found { + return cachedContent.RepositoryGroupContent, nil + } + + // fetch content from backend and cache it + logger.Info("Fetching content from backend") + content, err := c.backend.FetchGroupContent(ctx, source) + if err != nil { + return types.RepositoryGroupContent{}, err + } + c.cachedContent[source.GetGroupPath()] = CachedContent{ + RepositoryGroupContent: content, + creationTime: time.Now(), + } + return content, nil + } else { + c.contentLock.RUnlock() + logger.Debug("Cache hit") + return cachedContent.RepositoryGroupContent, nil + } +} + +func (c *Cache) InvalidateCache(source types.RepositoryGroupSource) { + c.contentLock.Lock() + defer c.contentLock.Unlock() + + delete(c.cachedContent, source.GetGroupPath()) +} diff --git a/cache/cache_test.go b/cache/cache_test.go new file mode 100644 index 0000000..b35ab7a --- /dev/null +++ b/cache/cache_test.go @@ -0,0 +1,142 @@ +package cache_test + +import ( + "context" + "io" + "log/slog" + "testing" + + "github.com/badjware/gitforgefs/cache" + "github.com/badjware/gitforgefs/types" +) + +type mockRepoGroupSource struct { + ID uint64 + Name string + Path string +} + +func (m mockRepoGroupSource) GetGroupID() uint64 { return m.ID } +func (m mockRepoGroupSource) GetGroupName() string { return m.Name } +func (m mockRepoGroupSource) GetGroupPath() string { return m.Path } + +type mockBackend struct { + RootCalls int + GroupCalls int + + rootContentValue map[string]types.RepositoryGroupSource + groupContentValue types.RepositoryGroupContent +} + +func (m *mockBackend) FetchRootGroupContent(ctx context.Context) (map[string]types.RepositoryGroupSource, error) { + m.RootCalls++ + return m.rootContentValue, nil +} + +func (m *mockBackend) FetchGroupContent(ctx context.Context, source types.RepositoryGroupSource) (types.RepositoryGroupContent, error) { + m.GroupCalls++ + return m.groupContentValue, nil +} + +func newLogger() *slog.Logger { + return slog.New(slog.NewTextHandler(io.Discard, &slog.HandlerOptions{})) +} + +func TestFetchRootGroupContent(t *testing.T) { + backend := &mockBackend{ + rootContentValue: map[string]types.RepositoryGroupSource{ + "g": mockRepoGroupSource{ID: 1, Name: "g", Path: "g"}, + }, + } + + logger := newLogger() + c := cache.NewForgeCache(backend, logger) + + ctx := context.Background() + + result1, err := c.FetchRootGroupContent(ctx) + if err != nil { + t.Fatalf("first FetchRootGroupContent failed: %v", err) + } + if content, ok := result1["g"]; !ok || content.GetGroupID() != 1 { + t.Fatalf("unexpected root content fetched: %v", result1) + } + + result2, err := c.FetchRootGroupContent(ctx) + if err != nil { + t.Fatalf("second FetchRootGroupContent failed: %v", err) + } + if content, ok := result2["g"]; !ok || content.GetGroupID() != 1 { + t.Fatalf("unexpected root content fetched: %v", result2) + } + + if backend.RootCalls != 1 { + t.Fatalf("expected backend.FetchRootGroupContent to be called once, got %d", backend.RootCalls) + } +} + +func TestFetchGroupContent(t *testing.T) { + backend := &mockBackend{ + groupContentValue: types.RepositoryGroupContent{ + Groups: map[string]types.RepositoryGroupSource{"group2": mockRepoGroupSource{ID: 2, Name: "group2", Path: "path/group1/group2"}}, + Repositories: map[string]types.RepositorySource{}, + }, + } + + logger := newLogger() + c := cache.NewForgeCache(backend, logger) + + src := mockRepoGroupSource{ID: 1, Name: "group1", Path: "path/group1"} + + ctx := context.Background() + + result1, err := c.FetchGroupContent(ctx, src) + if err != nil { + t.Fatalf("first FetchGroupContent failed: %v", err) + } + if content, ok := result1.Groups["group2"]; !ok || content.GetGroupID() != 2 { + t.Fatalf("unexpected group content fetched: %v", result1) + } + + result2, err := c.FetchGroupContent(ctx, src) + if err != nil { + t.Fatalf("second FetchGroupContent failed: %v", err) + } + if content, ok := result2.Groups["group2"]; !ok || content.GetGroupID() != 2 { + t.Fatalf("unexpected group content fetched: %v", result2) + } + + if backend.GroupCalls != 1 { + t.Fatalf("expected backend.FetchGroupContent to be called once, got %d", backend.GroupCalls) + } +} + +func TestInvalidateCache(t *testing.T) { + backend := &mockBackend{ + groupContentValue: types.RepositoryGroupContent{ + Groups: map[string]types.RepositoryGroupSource{"group2": mockRepoGroupSource{ID: 2, Name: "group2", Path: "path/group1/group2"}}, + Repositories: map[string]types.RepositorySource{}, + }, + } + + logger := newLogger() + c := cache.NewForgeCache(backend, logger) + + src := mockRepoGroupSource{ID: 1, Name: "group1", Path: "path/group1"} + + ctx := context.Background() + + if _, err := c.FetchGroupContent(ctx, src); err != nil { + t.Fatalf("first FetchGroupContent failed: %v", err) + } + + c.InvalidateCache(src) + + if _, err := c.FetchGroupContent(ctx, src); err != nil { + t.Fatalf("second FetchGroupContent failed: %v", err) + } + + if backend.GroupCalls != 2 { + t.Fatalf("expected backend.FetchGroupContent to be called twice, got %d", backend.GroupCalls) + } +} diff --git a/forges/gitea/client.go b/forges/gitea/client.go index b568579..350b47e 100644 --- a/forges/gitea/client.go +++ b/forges/gitea/client.go @@ -4,11 +4,10 @@ import ( "context" "fmt" "log/slog" - "sync" "code.gitea.io/sdk/gitea" "github.com/badjware/gitforgefs/config" - "github.com/badjware/gitforgefs/fstree" + "github.com/badjware/gitforgefs/types" ) type giteaClient struct { @@ -17,15 +16,8 @@ type giteaClient struct { logger *slog.Logger - rootContent map[string]fstree.GroupSource - - // API response cache - organizationCacheMux sync.RWMutex - organizationNameToIDMap map[string]int64 - organizationCache map[int64]*Organization - userCacheMux sync.RWMutex - userNameToIDMap map[string]int64 - userCache map[int64]*User + // use a map without values for efficient lookups + users map[string]struct{} } func NewClient(logger *slog.Logger, config config.GiteaClientConfig) (*giteaClient, error) { @@ -40,12 +32,7 @@ func NewClient(logger *slog.Logger, config config.GiteaClientConfig) (*giteaClie logger: logger, - rootContent: nil, - - organizationNameToIDMap: map[string]int64{}, - organizationCache: map[int64]*Organization{}, - userNameToIDMap: map[string]int64{}, - userCache: map[int64]*User{}, + users: make(map[string]struct{}), } // Fetch current user and add it to the list @@ -53,45 +40,41 @@ func NewClient(logger *slog.Logger, config config.GiteaClientConfig) (*giteaClie if err != nil { logger.Warn("failed to fetch the current user:", "error", err.Error()) } else { - giteaClient.UserNames = append(giteaClient.UserNames, *¤tUser.UserName) + giteaClient.UserNames = append(giteaClient.UserNames, currentUser.UserName) } return giteaClient, nil } -func (c *giteaClient) FetchRootGroupContent(ctx context.Context) (map[string]fstree.GroupSource, error) { - if c.rootContent == nil { - rootContent := make(map[string]fstree.GroupSource) - - for _, orgName := range c.GiteaClientConfig.OrgNames { - org, err := c.fetchOrganization(ctx, orgName) - if err != nil { - c.logger.Warn(err.Error()) - } else { - rootContent[org.Name] = org - } - } +func (c *giteaClient) FetchRootGroupContent(ctx context.Context) (map[string]types.RepositoryGroupSource, error) { + rootContent := make(map[string]types.RepositoryGroupSource) - for _, userName := range c.GiteaClientConfig.UserNames { - user, err := c.fetchUser(ctx, userName) - if err != nil { - c.logger.Warn(err.Error()) - } else { - rootContent[user.Name] = user - } + for _, orgName := range c.GiteaClientConfig.OrgNames { + org, err := c.fetchOrganization(ctx, orgName) + if err != nil { + c.logger.Warn(err.Error()) + } else { + rootContent[org.Name] = org } + } - c.rootContent = rootContent + for _, userName := range c.GiteaClientConfig.UserNames { + user, err := c.fetchUser(ctx, userName) + if err != nil { + c.logger.Warn(err.Error()) + } else { + rootContent[user.Name] = user + c.users[user.Name] = struct{}{} + } } - return c.rootContent, nil + + return rootContent, nil } -func (c *giteaClient) FetchGroupContent(ctx context.Context, gid uint64) (map[string]fstree.GroupSource, map[string]fstree.RepositorySource, error) { - if org, found := c.organizationCache[int64(gid)]; found { - return c.fetchOrganizationContent(ctx, org) - } - if user, found := c.userCache[int64(gid)]; found { - return c.fetchUserContent(ctx, user) +func (c *giteaClient) FetchGroupContent(ctx context.Context, source types.RepositoryGroupSource) (types.RepositoryGroupContent, error) { + if _, found := c.users[source.GetGroupPath()]; found { + return c.fetchUserContent(ctx, source.GetGroupPath()) + } else { + return c.fetchOrganizationContent(ctx, source.GetGroupPath()) } - return nil, nil, fmt.Errorf("invalid gid: %v", gid) } diff --git a/forges/gitea/organization.go b/forges/gitea/organization.go index e0f5b65..7649767 100644 --- a/forges/gitea/organization.go +++ b/forges/gitea/organization.go @@ -3,51 +3,29 @@ package gitea import ( "context" "fmt" - "sync" "code.gitea.io/sdk/gitea" - "github.com/badjware/gitforgefs/fstree" + "github.com/badjware/gitforgefs/types" ) type Organization struct { ID int64 Name string - - mux sync.Mutex - - // hold org content - childRepositories map[string]fstree.RepositorySource } func (o *Organization) GetGroupID() uint64 { return uint64(o.ID) } -func (o *Organization) InvalidateContentCache() { - o.mux.Lock() - defer o.mux.Unlock() +func (o *Organization) GetGroupName() string { + return o.Name +} - // clear child repositories from cache - o.childRepositories = nil +func (o *Organization) GetGroupPath() string { + return o.Name } func (c *giteaClient) fetchOrganization(ctx context.Context, orgName string) (*Organization, error) { - c.organizationCacheMux.RLock() - cachedId, found := c.organizationNameToIDMap[orgName] - if found { - cachedOrg := c.organizationCache[cachedId] - c.organizationCacheMux.RUnlock() - - // if found in cache, return the cached reference - c.logger.Debug("Organization cache hit", "org_name", orgName) - return cachedOrg, nil - } else { - c.organizationCacheMux.RUnlock() - - c.logger.Debug("Organization cache miss", "org_name", orgName) - } - - // If not found in cache, fetch organization infos from API giteaOrg, _, err := c.client.GetOrg(orgName) if err != nil { return nil, fmt.Errorf("failed to fetch organization with name %v: %v", orgName, err) @@ -55,51 +33,43 @@ func (c *giteaClient) fetchOrganization(ctx context.Context, orgName string) (*O newOrg := Organization{ ID: giteaOrg.ID, Name: giteaOrg.UserName, - - childRepositories: nil, } - // save in cache - c.organizationCacheMux.Lock() - c.organizationCache[newOrg.ID] = &newOrg - c.organizationNameToIDMap[newOrg.Name] = newOrg.ID - c.organizationCacheMux.Unlock() - return &newOrg, nil } -func (c *giteaClient) fetchOrganizationContent(ctx context.Context, org *Organization) (map[string]fstree.GroupSource, map[string]fstree.RepositorySource, error) { - org.mux.Lock() - defer org.mux.Unlock() +func (c *giteaClient) fetchOrganizationContent(ctx context.Context, orgName string) (types.RepositoryGroupContent, error) { + org, err := c.fetchOrganization(ctx, orgName) + if err != nil { + return types.RepositoryGroupContent{}, err + } - // Get cached data if available - // TODO: cache cache invalidation? - if org.childRepositories == nil { - childRepositories := make(map[string]fstree.RepositorySource) + repositories := make(map[string]types.RepositorySource) - // Fetch the organization repositories - listReposOptions := gitea.ListReposOptions{ - ListOptions: gitea.ListOptions{PageSize: 100}, + // Fetch the organization repositories + listReposOptions := gitea.ListReposOptions{ + ListOptions: gitea.ListOptions{PageSize: 100}, + } + for { + giteaRepositories, response, err := c.client.ListOrgRepos(org.Name, gitea.ListOrgReposOptions(listReposOptions)) + if err != nil { + return types.RepositoryGroupContent{}, fmt.Errorf("failed to fetch repository in gitea: %v", err) } - for { - giteaRepositories, response, err := c.client.ListOrgRepos(org.Name, gitea.ListOrgReposOptions(listReposOptions)) - if err != nil { - return nil, nil, fmt.Errorf("failed to fetch repository in gitea: %v", err) - } - for _, giteaRepository := range giteaRepositories { - repository := c.newRepositoryFromGiteaRepository(ctx, giteaRepository) - if repository != nil { - childRepositories[repository.Path] = repository - } + for _, giteaRepository := range giteaRepositories { + repository := c.newRepositoryFromGiteaRepository(giteaRepository) + if repository != nil { + repositories[repository.GetRepositoryName()] = repository } - if response.NextPage == 0 { - break - } - // Get the next page - listReposOptions.Page = response.NextPage } - - org.childRepositories = childRepositories + if response.NextPage == 0 { + break + } + // Get the next page + listReposOptions.Page = response.NextPage } - return make(map[string]fstree.GroupSource), org.childRepositories, nil + + return types.RepositoryGroupContent{ + Groups: make(map[string]types.RepositoryGroupSource), + Repositories: repositories, + }, nil } diff --git a/forges/gitea/repository.go b/forges/gitea/repository.go index d2c6cec..2c9be2d 100644 --- a/forges/gitea/repository.go +++ b/forges/gitea/repository.go @@ -1,7 +1,6 @@ package gitea import ( - "context" "path" "code.gitea.io/sdk/gitea" @@ -10,6 +9,7 @@ import ( type Repository struct { ID int64 + Name string Path string CloneURL string DefaultBranch string @@ -19,6 +19,14 @@ func (r *Repository) GetRepositoryID() uint64 { return uint64(r.ID) } +func (r *Repository) GetRepositoryName() string { + return r.Name +} + +func (r *Repository) GetRepositoryPath() string { + return r.Path +} + func (r *Repository) GetCloneURL() string { return r.CloneURL } @@ -27,13 +35,14 @@ func (r *Repository) GetDefaultBranch() string { return r.DefaultBranch } -func (c *giteaClient) newRepositoryFromGiteaRepository(ctx context.Context, repository *gitea.Repository) *Repository { +func (c *giteaClient) newRepositoryFromGiteaRepository(repository *gitea.Repository) *Repository { if c.ArchivedRepoHandling == config.ArchivedProjectIgnore && repository.Archived { return nil } r := Repository{ ID: repository.ID, - Path: repository.Name, + Name: repository.Name, + Path: repository.FullName, DefaultBranch: repository.DefaultBranch, } if r.DefaultBranch == "" { @@ -45,6 +54,7 @@ func (c *giteaClient) newRepositoryFromGiteaRepository(ctx context.Context, repo r.CloneURL = repository.CloneURL } if c.ArchivedRepoHandling == config.ArchivedProjectHide && repository.Archived { + r.Name = "." + r.Name r.Path = path.Join(path.Dir(r.Path), "."+path.Base(r.Path)) } return &r diff --git a/forges/gitea/user.go b/forges/gitea/user.go index d392355..9e46baf 100644 --- a/forges/gitea/user.go +++ b/forges/gitea/user.go @@ -3,51 +3,29 @@ package gitea import ( "context" "fmt" - "sync" "code.gitea.io/sdk/gitea" - "github.com/badjware/gitforgefs/fstree" + "github.com/badjware/gitforgefs/types" ) type User struct { ID int64 Name string - - mux sync.Mutex - - // hold user content - childRepositories map[string]fstree.RepositorySource } func (u *User) GetGroupID() uint64 { return uint64(u.ID) } -func (u *User) InvalidateContentCache() { - u.mux.Lock() - defer u.mux.Unlock() +func (u *User) GetGroupName() string { + return u.Name +} - // clear child repositories from cache - u.childRepositories = nil +func (u *User) GetGroupPath() string { + return u.Name } func (c *giteaClient) fetchUser(ctx context.Context, userName string) (*User, error) { - c.userCacheMux.RLock() - cachedId, found := c.userNameToIDMap[userName] - if found { - cachedUser := c.userCache[cachedId] - c.userCacheMux.RUnlock() - - // if found in cache, return the cached reference - c.logger.Debug("User cache hit", "user_name", userName) - return cachedUser, nil - } else { - c.userCacheMux.RUnlock() - - c.logger.Debug("User cache miss", "user_name", userName) - } - - // If not found in cache, fetch user infos from API giteaUser, _, err := c.client.GetUserInfo(userName) if err != nil { return nil, fmt.Errorf("failed to fetch user with name %v: %v", userName, err) @@ -55,51 +33,43 @@ func (c *giteaClient) fetchUser(ctx context.Context, userName string) (*User, er newUser := User{ ID: giteaUser.ID, Name: giteaUser.UserName, - - childRepositories: nil, } - // save in cache - c.userCacheMux.Lock() - c.userCache[newUser.ID] = &newUser - c.userNameToIDMap[newUser.Name] = newUser.ID - c.userCacheMux.Unlock() - return &newUser, nil } -func (c *giteaClient) fetchUserContent(ctx context.Context, user *User) (map[string]fstree.GroupSource, map[string]fstree.RepositorySource, error) { - user.mux.Lock() - defer user.mux.Unlock() +func (c *giteaClient) fetchUserContent(ctx context.Context, userName string) (types.RepositoryGroupContent, error) { + user, err := c.fetchUser(ctx, userName) + if err != nil { + return types.RepositoryGroupContent{}, err + } - // Get cached data if available - // TODO: cache cache invalidation? - if user.childRepositories == nil { - childRepositories := make(map[string]fstree.RepositorySource) + repositories := make(map[string]types.RepositorySource) - // Fetch the user repositories - listReposOptions := gitea.ListReposOptions{ - ListOptions: gitea.ListOptions{PageSize: 100}, + // Fetch the user repositories + listReposOptions := gitea.ListReposOptions{ + ListOptions: gitea.ListOptions{PageSize: 100}, + } + for { + giteaRepositories, response, err := c.client.ListUserRepos(user.Name, listReposOptions) + if err != nil { + return types.RepositoryGroupContent{}, fmt.Errorf("failed to fetch repository in gitea: %v", err) } - for { - giteaRepositories, response, err := c.client.ListUserRepos(user.Name, listReposOptions) - if err != nil { - return nil, nil, fmt.Errorf("failed to fetch repository in gitea: %v", err) - } - for _, giteaRepository := range giteaRepositories { - repository := c.newRepositoryFromGiteaRepository(ctx, giteaRepository) - if repository != nil { - childRepositories[repository.Path] = repository - } + for _, giteaRepository := range giteaRepositories { + repository := c.newRepositoryFromGiteaRepository(giteaRepository) + if repository != nil { + repositories[repository.GetRepositoryName()] = repository } - if response.NextPage == 0 { - break - } - // Get the next page - listReposOptions.Page = response.NextPage } - - user.childRepositories = childRepositories + if response.NextPage == 0 { + break + } + // Get the next page + listReposOptions.Page = response.NextPage } - return make(map[string]fstree.GroupSource), user.childRepositories, nil + + return types.RepositoryGroupContent{ + Groups: make(map[string]types.RepositoryGroupSource), + Repositories: repositories, + }, nil } diff --git a/forges/github/client.go b/forges/github/client.go index a95827f..5f08ba4 100644 --- a/forges/github/client.go +++ b/forges/github/client.go @@ -2,12 +2,10 @@ package github import ( "context" - "fmt" "log/slog" - "sync" "github.com/badjware/gitforgefs/config" - "github.com/badjware/gitforgefs/fstree" + "github.com/badjware/gitforgefs/types" "github.com/google/go-github/v63/github" ) @@ -17,15 +15,8 @@ type githubClient struct { logger *slog.Logger - rootContent map[string]fstree.GroupSource - - // API response cache - organizationCacheMux sync.RWMutex - organizationNameToIDMap map[string]int64 - organizationCache map[int64]*Organization - userCacheMux sync.RWMutex - userNameToIDMap map[string]int64 - userCache map[int64]*User + // use a map without values for efficient lookups + users map[string]struct{} } func NewClient(logger *slog.Logger, config config.GithubClientConfig) (*githubClient, error) { @@ -40,12 +31,7 @@ func NewClient(logger *slog.Logger, config config.GithubClientConfig) (*githubCl logger: logger, - rootContent: nil, - - organizationNameToIDMap: map[string]int64{}, - organizationCache: map[int64]*Organization{}, - userNameToIDMap: map[string]int64{}, - userCache: map[int64]*User{}, + users: make(map[string]struct{}), } // Fetch current user and add it to the list @@ -59,39 +45,35 @@ func NewClient(logger *slog.Logger, config config.GithubClientConfig) (*githubCl return gitHubClient, nil } -func (c *githubClient) FetchRootGroupContent(ctx context.Context) (map[string]fstree.GroupSource, error) { - if c.rootContent == nil { - rootContent := make(map[string]fstree.GroupSource) - - for _, orgName := range c.GithubClientConfig.OrgNames { - org, err := c.fetchOrganization(ctx, orgName) - if err != nil { - c.logger.Warn(err.Error()) - } else { - rootContent[org.Name] = org - } - } +func (c *githubClient) FetchRootGroupContent(ctx context.Context) (map[string]types.RepositoryGroupSource, error) { + rootContent := make(map[string]types.RepositoryGroupSource) - for _, userName := range c.GithubClientConfig.UserNames { - user, err := c.fetchUser(ctx, userName) - if err != nil { - c.logger.Warn(err.Error()) - } else { - rootContent[user.Name] = user - } + for _, orgName := range c.GithubClientConfig.OrgNames { + org, err := c.fetchOrganization(ctx, orgName) + if err != nil { + c.logger.Warn(err.Error()) + } else { + rootContent[org.Name] = org } + } - c.rootContent = rootContent + for _, userName := range c.GithubClientConfig.UserNames { + user, err := c.fetchUser(ctx, userName) + if err != nil { + c.logger.Warn(err.Error()) + } else { + rootContent[user.Name] = user + c.users[user.Name] = struct{}{} + } } - return c.rootContent, nil + + return rootContent, nil } -func (c *githubClient) FetchGroupContent(ctx context.Context, gid uint64) (map[string]fstree.GroupSource, map[string]fstree.RepositorySource, error) { - if org, found := c.organizationCache[int64(gid)]; found { - return c.fetchOrganizationContent(ctx, org) - } - if user, found := c.userCache[int64(gid)]; found { - return c.fetchUserContent(ctx, user) +func (c *githubClient) FetchGroupContent(ctx context.Context, source types.RepositoryGroupSource) (types.RepositoryGroupContent, error) { + if _, found := c.users[source.GetGroupPath()]; found { + return c.fetchUserContent(ctx, source.GetGroupPath()) + } else { + return c.fetchOrganizationContent(ctx, source.GetGroupPath()) } - return nil, nil, fmt.Errorf("invalid gid: %v", gid) } diff --git a/forges/github/organization.go b/forges/github/organization.go index bef3ce2..1af8588 100644 --- a/forges/github/organization.go +++ b/forges/github/organization.go @@ -3,103 +3,71 @@ package github import ( "context" "fmt" - "sync" - "github.com/badjware/gitforgefs/fstree" + "github.com/badjware/gitforgefs/types" "github.com/google/go-github/v63/github" ) type Organization struct { ID int64 Name string - - mux sync.Mutex - - // hold org content - childRepositories map[string]fstree.RepositorySource } func (o *Organization) GetGroupID() uint64 { return uint64(o.ID) } -func (o *Organization) InvalidateContentCache() { - o.mux.Lock() - defer o.mux.Unlock() +func (o *Organization) GetGroupName() string { + return o.Name +} - // clear child repositories from cache - o.childRepositories = nil +func (o *Organization) GetGroupPath() string { + return o.Name } func (c *githubClient) fetchOrganization(ctx context.Context, orgName string) (*Organization, error) { - c.organizationCacheMux.RLock() - cachedId, found := c.organizationNameToIDMap[orgName] - if found { - cachedOrg := c.organizationCache[cachedId] - c.organizationCacheMux.RUnlock() - - // if found in cache, return the cached reference - c.logger.Debug("Organization cache hit", "org_name", orgName) - return cachedOrg, nil - } else { - c.organizationCacheMux.RUnlock() - - c.logger.Debug("Organization cache miss", "org_name", orgName) - } - - // If not found in cache, fetch organization infos from API githubOrg, _, err := c.client.Organizations.Get(ctx, orgName) if err != nil { return nil, fmt.Errorf("failed to fetch organization with name %v: %v", orgName, err) } - newOrg := Organization{ + return &Organization{ ID: *githubOrg.ID, Name: *githubOrg.Login, - - childRepositories: nil, - } - - // save in cache - c.organizationCacheMux.Lock() - c.organizationCache[newOrg.ID] = &newOrg - c.organizationNameToIDMap[newOrg.Name] = newOrg.ID - c.organizationCacheMux.Unlock() - - return &newOrg, nil + }, nil } -func (c *githubClient) fetchOrganizationContent(ctx context.Context, org *Organization) (map[string]fstree.GroupSource, map[string]fstree.RepositorySource, error) { - org.mux.Lock() - defer org.mux.Unlock() +func (c *githubClient) fetchOrganizationContent(ctx context.Context, orgName string) (types.RepositoryGroupContent, error) { + org, err := c.fetchOrganization(ctx, orgName) + if err != nil { + return types.RepositoryGroupContent{}, err + } - // Get cached data if available - // TODO: cache cache invalidation? - if org.childRepositories == nil { - childRepositories := make(map[string]fstree.RepositorySource) + repositories := make(map[string]types.RepositorySource) - // Fetch the organization repositories - repositoryListOpt := &github.RepositoryListByOrgOptions{ - ListOptions: github.ListOptions{PerPage: 100}, + // Fetch the organization repositories + repositoryListOpt := &github.RepositoryListByOrgOptions{ + ListOptions: github.ListOptions{PerPage: 100}, + } + for { + githubRepositories, response, err := c.client.Repositories.ListByOrg(ctx, org.Name, repositoryListOpt) + if err != nil { + return types.RepositoryGroupContent{}, fmt.Errorf("failed to fetch repository in github: %v", err) } - for { - githubRepositories, response, err := c.client.Repositories.ListByOrg(ctx, org.Name, repositoryListOpt) - if err != nil { - return nil, nil, fmt.Errorf("failed to fetch repository in github: %v", err) - } - for _, githubRepository := range githubRepositories { - repository := c.newRepositoryFromGithubRepository(ctx, githubRepository) - if repository != nil { - childRepositories[repository.Path] = repository - } + for _, githubRepository := range githubRepositories { + repository := c.newRepositoryFromGithubRepository(githubRepository) + if repository != nil { + repositories[repository.GetRepositoryName()] = repository } - if response.NextPage == 0 { - break - } - // Get the next page - repositoryListOpt.Page = response.NextPage } - - org.childRepositories = childRepositories + if response.NextPage == 0 { + break + } + // Get the next page + repositoryListOpt.Page = response.NextPage } - return make(map[string]fstree.GroupSource), org.childRepositories, nil + + return types.RepositoryGroupContent{ + Groups: make(map[string]types.RepositoryGroupSource), + Repositories: repositories, + }, nil } diff --git a/forges/github/repository.go b/forges/github/repository.go index ecfe720..61ef5a1 100644 --- a/forges/github/repository.go +++ b/forges/github/repository.go @@ -1,7 +1,6 @@ package github import ( - "context" "path" "github.com/badjware/gitforgefs/config" @@ -10,6 +9,7 @@ import ( type Repository struct { ID int64 + Name string Path string CloneURL string DefaultBranch string @@ -19,6 +19,14 @@ func (r *Repository) GetRepositoryID() uint64 { return uint64(r.ID) } +func (r *Repository) GetRepositoryName() string { + return r.Name +} + +func (r *Repository) GetRepositoryPath() string { + return r.Path +} + func (r *Repository) GetCloneURL() string { return r.CloneURL } @@ -27,13 +35,14 @@ func (r *Repository) GetDefaultBranch() string { return r.DefaultBranch } -func (c *githubClient) newRepositoryFromGithubRepository(ctx context.Context, repository *github.Repository) *Repository { +func (c *githubClient) newRepositoryFromGithubRepository(repository *github.Repository) *Repository { if c.ArchivedRepoHandling == config.ArchivedProjectIgnore && *repository.Archived { return nil } r := Repository{ ID: *repository.ID, - Path: *repository.Name, + Name: *repository.Name, + Path: *repository.FullName, DefaultBranch: *repository.DefaultBranch, } if r.DefaultBranch == "" { @@ -45,6 +54,7 @@ func (c *githubClient) newRepositoryFromGithubRepository(ctx context.Context, re r.CloneURL = *repository.CloneURL } if c.ArchivedRepoHandling == config.ArchivedProjectHide && *repository.Archived { + r.Name = "." + r.Name r.Path = path.Join(path.Dir(r.Path), "."+path.Base(r.Path)) } return &r diff --git a/forges/github/user.go b/forges/github/user.go index 5e16b90..1910a51 100644 --- a/forges/github/user.go +++ b/forges/github/user.go @@ -3,103 +3,71 @@ package github import ( "context" "fmt" - "sync" - "github.com/badjware/gitforgefs/fstree" + "github.com/badjware/gitforgefs/types" "github.com/google/go-github/v63/github" ) type User struct { ID int64 Name string - - mux sync.Mutex - - // hold user content - childRepositories map[string]fstree.RepositorySource } func (u *User) GetGroupID() uint64 { return uint64(u.ID) } -func (u *User) InvalidateContentCache() { - u.mux.Lock() - defer u.mux.Unlock() +func (u *User) GetGroupName() string { + return u.Name +} - // clear child repositories from cache - u.childRepositories = nil +func (u *User) GetGroupPath() string { + return u.Name } func (c *githubClient) fetchUser(ctx context.Context, userName string) (*User, error) { - c.userCacheMux.RLock() - cachedId, found := c.userNameToIDMap[userName] - if found { - cachedUser := c.userCache[cachedId] - c.userCacheMux.RUnlock() - - // if found in cache, return the cached reference - c.logger.Debug("User cache hit", "user_name", userName) - return cachedUser, nil - } else { - c.userCacheMux.RUnlock() - - c.logger.Debug("User cache miss", "user_name", userName) - } - - // If not found in cache, fetch user infos from API githubUser, _, err := c.client.Users.Get(ctx, userName) if err != nil { return nil, fmt.Errorf("failed to fetch user with name %v: %v", userName, err) } - newUser := User{ + return &User{ ID: *githubUser.ID, Name: *githubUser.Login, - - childRepositories: nil, - } - - // save in cache - c.userCacheMux.Lock() - c.userCache[newUser.ID] = &newUser - c.userNameToIDMap[newUser.Name] = newUser.ID - c.userCacheMux.Unlock() - - return &newUser, nil + }, nil } -func (c *githubClient) fetchUserContent(ctx context.Context, user *User) (map[string]fstree.GroupSource, map[string]fstree.RepositorySource, error) { - user.mux.Lock() - defer user.mux.Unlock() +func (c *githubClient) fetchUserContent(ctx context.Context, userName string) (types.RepositoryGroupContent, error) { + user, err := c.fetchUser(ctx, userName) + if err != nil { + return types.RepositoryGroupContent{}, err + } - // Get cached data if available - // TODO: cache cache invalidation? - if user.childRepositories == nil { - childRepositories := make(map[string]fstree.RepositorySource) + repositories := make(map[string]types.RepositorySource) - // Fetch the user repositories - repositoryListOpt := &github.RepositoryListByUserOptions{ - ListOptions: github.ListOptions{PerPage: 100}, + // Fetch the user repositories + repositoryListOpt := &github.RepositoryListByUserOptions{ + ListOptions: github.ListOptions{PerPage: 100}, + } + for { + githubRepositories, response, err := c.client.Repositories.ListByUser(ctx, user.Name, repositoryListOpt) + if err != nil { + return types.RepositoryGroupContent{}, fmt.Errorf("failed to fetch repository in github: %v", err) } - for { - githubRepositories, response, err := c.client.Repositories.ListByUser(ctx, user.Name, repositoryListOpt) - if err != nil { - return nil, nil, fmt.Errorf("failed to fetch repository in github: %v", err) - } - for _, githubRepository := range githubRepositories { - repository := c.newRepositoryFromGithubRepository(ctx, githubRepository) - if repository != nil { - childRepositories[repository.Path] = repository - } + for _, githubRepository := range githubRepositories { + repository := c.newRepositoryFromGithubRepository(githubRepository) + if repository != nil { + repositories[repository.GetRepositoryName()] = repository } - if response.NextPage == 0 { - break - } - // Get the next page - repositoryListOpt.Page = response.NextPage } - - user.childRepositories = childRepositories + if response.NextPage == 0 { + break + } + // Get the next page + repositoryListOpt.Page = response.NextPage } - return make(map[string]fstree.GroupSource), user.childRepositories, nil + + return types.RepositoryGroupContent{ + Groups: make(map[string]types.RepositoryGroupSource), + Repositories: repositories, + }, nil } diff --git a/forges/gitlab/client.go b/forges/gitlab/client.go index 1fb42e9..52e4eeb 100644 --- a/forges/gitlab/client.go +++ b/forges/gitlab/client.go @@ -4,11 +4,9 @@ import ( "context" "fmt" "log/slog" - "slices" - "sync" "github.com/badjware/gitforgefs/config" - "github.com/badjware/gitforgefs/fstree" + "github.com/badjware/gitforgefs/types" gitlab "gitlab.com/gitlab-org/api/client-go" ) @@ -18,15 +16,8 @@ type gitlabClient struct { logger *slog.Logger - rootContent map[string]fstree.GroupSource - - userIDs []int - - // API response cache - groupCacheMux sync.RWMutex - groupCache map[int]*Group - userCacheMux sync.RWMutex - userCache map[int]*User + // use a map without values for efficient lookups + users map[string]int } func NewClient(logger *slog.Logger, config config.GitlabClientConfig) (*gitlabClient, error) { @@ -44,12 +35,7 @@ func NewClient(logger *slog.Logger, config config.GitlabClientConfig) (*gitlabCl logger: logger, - rootContent: nil, - - userIDs: []int{}, - - groupCache: map[int]*Group{}, - userCache: map[int]*User{}, + users: make(map[string]int), } // Fetch current user and add it to the list @@ -57,7 +43,7 @@ func NewClient(logger *slog.Logger, config config.GitlabClientConfig) (*gitlabCl if err != nil { logger.Warn("failed to fetch the current user:", "error", err.Error()) } else { - gitlabClient.userIDs = append(gitlabClient.userIDs, currentUser.ID) + gitlabClient.users[currentUser.Username] = currentUser.ID } // Fetch the configured users and add them to the list @@ -66,54 +52,39 @@ func NewClient(logger *slog.Logger, config config.GitlabClientConfig) (*gitlabCl if err != nil || len(user) != 1 { logger.Warn("failed to fetch the user", "userName", userName, "error", err.Error()) } else { - gitlabClient.userIDs = append(gitlabClient.userIDs, user[0].ID) + gitlabClient.users[userName] = user[0].ID } } return gitlabClient, nil } -func (c *gitlabClient) FetchRootGroupContent(ctx context.Context) (map[string]fstree.GroupSource, error) { - // use cached values if available - if c.rootContent == nil { - rootGroupCache := make(map[string]fstree.GroupSource) +func (c *gitlabClient) FetchRootGroupContent(ctx context.Context) (map[string]types.RepositoryGroupSource, error) { + rootContent := make(map[string]types.RepositoryGroupSource) - // fetch root groups - for _, gid := range c.GroupIDs { - group, err := c.fetchGroup(ctx, gid) - if err != nil { - return nil, err - } - rootGroupCache[group.Name] = group + // fetch root groups + for _, gid := range c.GroupIDs { + group, err := c.fetchGroup(ctx, gid) + if err != nil { + return nil, err } - // fetch users - for _, uid := range c.userIDs { - user, err := c.fetchUser(ctx, uid) - if err != nil { - return nil, err - } - rootGroupCache[user.Name] = user + rootContent[group.Name] = group + } + // fetch users + for _, uid := range c.users { + user, err := c.fetchUser(ctx, uid) + if err != nil { + return nil, err } - - c.rootContent = rootGroupCache + rootContent[user.Name] = user } - return c.rootContent, nil + return rootContent, nil } -func (c *gitlabClient) FetchGroupContent(ctx context.Context, gid uint64) (map[string]fstree.GroupSource, map[string]fstree.RepositorySource, error) { - if slices.Contains[[]int, int](c.userIDs, int(gid)) { - // gid is a user - user, err := c.fetchUser(ctx, int(gid)) - if err != nil { - return nil, nil, err - } - return c.fetchUserContent(ctx, user) +func (c *gitlabClient) FetchGroupContent(ctx context.Context, source types.RepositoryGroupSource) (types.RepositoryGroupContent, error) { + if _, found := c.users[source.GetGroupPath()]; found { + return c.fetchUserContent(ctx, int(source.GetGroupID())) } else { - // gid is a group - group, err := c.fetchGroup(ctx, int(gid)) - if err != nil { - return nil, nil, err - } - return c.fetchGroupContent(ctx, group) + return c.fetchGroupContent(ctx, int(source.GetGroupID())) } } diff --git a/forges/gitlab/group.go b/forges/gitlab/group.go index 9ee48d5..cb863b6 100644 --- a/forges/gitlab/group.go +++ b/forges/gitlab/group.go @@ -3,180 +3,101 @@ package gitlab import ( "context" "fmt" - "sync" - "github.com/badjware/gitforgefs/fstree" + "github.com/badjware/gitforgefs/types" gitlab "gitlab.com/gitlab-org/api/client-go" ) type Group struct { ID int Name string - - gitlabClient *gitlabClient - - mux sync.Mutex - - // hold group content - childGroups map[string]fstree.GroupSource - childProjects map[string]fstree.RepositorySource + Path string } func (g *Group) GetGroupID() uint64 { return uint64(g.ID) } -func (g *Group) InvalidateContentCache() { - g.mux.Lock() - defer g.mux.Unlock() - - // clear child group from cache - g.gitlabClient.groupCacheMux.Lock() - for _, childGroup := range g.childGroups { - gid := int(childGroup.GetGroupID()) - delete(g.gitlabClient.groupCache, gid) - } - g.gitlabClient.groupCacheMux.Unlock() - g.childGroups = nil +func (g *Group) GetGroupName() string { + return g.Name +} - // clear child repositories from cache - g.childGroups = nil +func (g *Group) GetGroupPath() string { + return g.Path } -func (c *gitlabClient) fetchGroup(ctx context.Context, gid int) (*Group, error) { - // start by searching the cache - // TODO: cache invalidation? - c.groupCacheMux.RLock() - group, found := c.groupCache[gid] - c.groupCacheMux.RUnlock() - if found { - c.logger.Debug("Group cache hit", "gid", gid) - return group, nil - } else { - c.logger.Debug("Group cache miss; fetching group", "gid", gid) +func (c *gitlabClient) newGroupFromGitlabGroup(gitlabGroup *gitlab.Group) *Group { + return &Group{ + ID: gitlabGroup.ID, + Name: gitlabGroup.Path, + Path: gitlabGroup.FullPath, } +} - // If not in cache, fetch group infos from API +func (c *gitlabClient) fetchGroup(ctx context.Context, gid int) (*Group, error) { gitlabGroup, _, err := c.client.Groups.GetGroup(gid, &gitlab.GetGroupOptions{}) if err != nil { return nil, fmt.Errorf("failed to fetch group with id %v: %v", gid, err) } c.logger.Debug("Fetched group", "gid", gid) - newGroup := Group{ - ID: gitlabGroup.ID, - Name: gitlabGroup.Path, - - gitlabClient: c, - - childGroups: nil, - childProjects: nil, - } - - // save in cache - c.groupCacheMux.Lock() - c.groupCache[gid] = &newGroup - c.groupCacheMux.Unlock() - - return &newGroup, nil + return c.newGroupFromGitlabGroup(gitlabGroup), nil } -func (c *gitlabClient) newGroupFromGitlabGroup(gitlabGroup *gitlab.Group) (*Group, error) { - gid := gitlabGroup.ID - - // start by searching the cache - c.groupCacheMux.RLock() - group, found := c.groupCache[gid] - c.groupCacheMux.RUnlock() - if found { - // if found in cache, return the cached reference - c.logger.Debug("Group cache hit", "gid", gid) - return group, nil - } else { - c.logger.Debug("Group cache miss; registering group", "gid", gid) - } - - // if not found in cache, convert and save to cache now - newGroup := Group{ - ID: gitlabGroup.ID, - Name: gitlabGroup.Path, - - gitlabClient: c, - - childGroups: nil, - childProjects: nil, +func (c *gitlabClient) fetchGroupContent(ctx context.Context, gid int) (types.RepositoryGroupContent, error) { + childGroups := make(map[string]types.RepositoryGroupSource) + childProjects := make(map[string]types.RepositorySource) + + // List subgroups in path + listGroupsOpt := &gitlab.ListSubGroupsOptions{ + ListOptions: gitlab.ListOptions{ + Page: 1, + PerPage: 100, + }, + AllAvailable: gitlab.Ptr(true), } - - // save in cache - c.groupCacheMux.Lock() - c.groupCache[gid] = &newGroup - c.groupCacheMux.Unlock() - - return &newGroup, nil -} - -func (c *gitlabClient) fetchGroupContent(ctx context.Context, group *Group) (map[string]fstree.GroupSource, map[string]fstree.RepositorySource, error) { - // Only a single routine can fetch the group content at the time. - // We lock for the whole duration of the function to avoid fetching the same data from the API - // multiple times if concurrent calls where to occur. - group.mux.Lock() - defer group.mux.Unlock() - - // Get cached data if available - // TODO: cache cache invalidation? - if group.childGroups == nil || group.childProjects == nil { - childGroups := make(map[string]fstree.GroupSource) - childProjects := make(map[string]fstree.RepositorySource) - - // List subgroups in path - listGroupsOpt := &gitlab.ListSubGroupsOptions{ - ListOptions: gitlab.ListOptions{ - Page: 1, - PerPage: 100, - }, - AllAvailable: gitlab.Ptr(true), + for { + gitlabGroups, response, err := c.client.Groups.ListSubGroups(gid, listGroupsOpt) + if err != nil { + return types.RepositoryGroupContent{}, fmt.Errorf("failed to fetch groups in gitlab: %v", err) } - for { - gitlabGroups, response, err := c.client.Groups.ListSubGroups(group.ID, listGroupsOpt) - if err != nil { - return nil, nil, fmt.Errorf("failed to fetch groups in gitlab: %v", err) - } - for _, gitlabGroup := range gitlabGroups { - group, _ := c.newGroupFromGitlabGroup(gitlabGroup) - childGroups[group.Name] = group - } - if response.CurrentPage >= response.TotalPages { - break + for _, gitlabGroup := range gitlabGroups { + group := c.newGroupFromGitlabGroup(gitlabGroup) + if group != nil { + childGroups[group.GetGroupName()] = group } - // Get the next page - listGroupsOpt.Page = response.NextPage } + if response.CurrentPage >= response.TotalPages { + break + } + // Get the next page + listGroupsOpt.Page = response.NextPage + } - // List projects in path - listProjectOpt := &gitlab.ListGroupProjectsOptions{ - ListOptions: gitlab.ListOptions{ - Page: 1, - PerPage: 100, - }} - for { - gitlabProjects, response, err := c.client.Groups.ListGroupProjects(group.ID, listProjectOpt) - if err != nil { - return nil, nil, fmt.Errorf("failed to fetch projects in gitlab: %v", err) - } - for _, gitlabProject := range gitlabProjects { - project := c.newProjectFromGitlabProject(ctx, gitlabProject) - if project != nil { - childProjects[project.Path] = project - } - } - if response.CurrentPage >= response.TotalPages { - break + // List projects in path + listProjectOpt := &gitlab.ListGroupProjectsOptions{ + ListOptions: gitlab.ListOptions{ + Page: 1, + PerPage: 100, + }} + for { + gitlabProjects, response, err := c.client.Groups.ListGroupProjects(gid, listProjectOpt) + if err != nil { + return types.RepositoryGroupContent{}, fmt.Errorf("failed to fetch projects in gitlab: %v", err) + } + for _, gitlabProject := range gitlabProjects { + project := c.newProjectFromGitlabProject(gitlabProject) + if project != nil { + childProjects[project.GetRepositoryName()] = project } - // Get the next page - listProjectOpt.Page = response.NextPage } - - group.childGroups = childGroups - group.childProjects = childProjects + if response.CurrentPage >= response.TotalPages { + break + } + // Get the next page + listProjectOpt.Page = response.NextPage } - return group.childGroups, group.childProjects, nil + return types.RepositoryGroupContent{ + Groups: childGroups, + Repositories: childProjects, + }, nil } diff --git a/forges/gitlab/project.go b/forges/gitlab/project.go index 2587265..f0e2544 100644 --- a/forges/gitlab/project.go +++ b/forges/gitlab/project.go @@ -1,7 +1,6 @@ package gitlab import ( - "context" "path" "github.com/badjware/gitforgefs/config" @@ -10,6 +9,7 @@ import ( type Project struct { ID int + Name string Path string CloneURL string DefaultBranch string @@ -19,6 +19,14 @@ func (p *Project) GetRepositoryID() uint64 { return uint64(p.ID) } +func (p *Project) GetRepositoryName() string { + return p.Name +} + +func (p *Project) GetRepositoryPath() string { + return p.Path +} + func (p *Project) GetCloneURL() string { return p.CloneURL } @@ -27,14 +35,15 @@ func (p *Project) GetDefaultBranch() string { return p.DefaultBranch } -func (c *gitlabClient) newProjectFromGitlabProject(ctx context.Context, project *gitlab.Project) *Project { +func (c *gitlabClient) newProjectFromGitlabProject(project *gitlab.Project) *Project { // https://godoc.org/github.com/xanzy/go-gitlab#Project if c.ArchivedProjectHandling == config.ArchivedProjectIgnore && project.Archived { return nil } p := Project{ ID: project.ID, - Path: project.Path, + Name: project.Name, + Path: project.PathWithNamespace, DefaultBranch: project.DefaultBranch, } if p.DefaultBranch == "" { @@ -46,6 +55,7 @@ func (c *gitlabClient) newProjectFromGitlabProject(ctx context.Context, project p.CloneURL = project.HTTPURLToRepo } if c.ArchivedProjectHandling == config.ArchivedProjectHide && project.Archived { + p.Name = "." + p.Name p.Path = path.Join(path.Dir(p.Path), "."+path.Base(p.Path)) } return &p diff --git a/forges/gitlab/user.go b/forges/gitlab/user.go index 726e96b..51e07f5 100644 --- a/forges/gitlab/user.go +++ b/forges/gitlab/user.go @@ -3,105 +3,67 @@ package gitlab import ( "context" "fmt" - "sync" - "github.com/badjware/gitforgefs/fstree" + "github.com/badjware/gitforgefs/types" gitlab "gitlab.com/gitlab-org/api/client-go" ) type User struct { ID int Name string - - mux sync.Mutex - - // hold user content - childProjects map[string]fstree.RepositorySource } func (u *User) GetGroupID() uint64 { return uint64(u.ID) } -func (u *User) InvalidateContentCache() { - u.mux.Lock() - defer u.mux.Unlock() +func (u *User) GetGroupName() string { + return u.Name +} - // clear child repositories from cache - u.childProjects = nil +func (u *User) GetGroupPath() string { + return u.Name } func (c *gitlabClient) fetchUser(ctx context.Context, uid int) (*User, error) { - // start by searching the cache - // TODO: cache invalidation? - c.userCacheMux.RLock() - user, found := c.userCache[uid] - c.userCacheMux.RUnlock() - if found { - // if found in cache, return the cached reference - c.logger.Debug("User cache hit", "uid", uid) - return user, nil - } else { - c.logger.Debug("User cache miss", "uid", uid) - } - - // If not found in cache, fetch group infos from API gitlabUser, _, err := c.client.Users.GetUser(uid, gitlab.GetUsersOptions{}) if err != nil { return nil, fmt.Errorf("failed to fetch user with id %v: %v", uid, err) } - newUser := User{ + return &User{ ID: gitlabUser.ID, Name: gitlabUser.Username, - - childProjects: nil, - } - - // save in cache - c.userCacheMux.Lock() - c.userCache[uid] = &newUser - c.userCacheMux.Unlock() - - return &newUser, nil + }, nil } -func (c *gitlabClient) fetchUserContent(ctx context.Context, user *User) (map[string]fstree.GroupSource, map[string]fstree.RepositorySource, error) { - // Only a single routine can fetch the user content at the time. - // We lock for the whole duration of the function to avoid fetching the same data from the API - // multiple times if concurrent calls where to occur. - user.mux.Lock() - defer user.mux.Unlock() - - // Get cached data if available - // TODO: cache cache invalidation? - if user.childProjects == nil { - childProjects := make(map[string]fstree.RepositorySource) - - // Fetch the user repositories - listProjectOpt := &gitlab.ListProjectsOptions{ - ListOptions: gitlab.ListOptions{ - Page: 1, - PerPage: 100, - }} - for { - gitlabProjects, response, err := c.client.Projects.ListUserProjects(user.ID, listProjectOpt) - if err != nil { - return nil, nil, fmt.Errorf("failed to fetch projects in gitlab: %v", err) - } - for _, gitlabProject := range gitlabProjects { - project := c.newProjectFromGitlabProject(ctx, gitlabProject) - if project != nil { - childProjects[project.Path] = project - } - } - if response.CurrentPage >= response.TotalPages { - break +func (c *gitlabClient) fetchUserContent(ctx context.Context, uid int) (types.RepositoryGroupContent, error) { + childProjects := make(map[string]types.RepositorySource) + + // Fetch the user repositories + listProjectOpt := &gitlab.ListProjectsOptions{ + ListOptions: gitlab.ListOptions{ + Page: 1, + PerPage: 100, + }} + for { + gitlabProjects, response, err := c.client.Projects.ListUserProjects(uid, listProjectOpt) + if err != nil { + return types.RepositoryGroupContent{}, fmt.Errorf("failed to fetch projects in gitlab: %v", err) + } + for _, gitlabProject := range gitlabProjects { + project := c.newProjectFromGitlabProject(gitlabProject) + if project != nil { + childProjects[project.GetRepositoryName()] = project } - // Get the next page - listProjectOpt.Page = response.NextPage } - - user.childProjects = childProjects + if response.CurrentPage >= response.TotalPages { + break + } + // Get the next page + listProjectOpt.Page = response.NextPage } - return make(map[string]fstree.GroupSource), user.childProjects, nil + return types.RepositoryGroupContent{ + Groups: make(map[string]types.RepositoryGroupSource), + Repositories: childProjects, + }, nil } diff --git a/fstree/group.go b/fstree/group.go index b58d874..422509d 100644 --- a/fstree/group.go +++ b/fstree/group.go @@ -4,6 +4,7 @@ import ( "context" "syscall" + "github.com/badjware/gitforgefs/types" "github.com/hanwen/go-fuse/v2/fs" "github.com/hanwen/go-fuse/v2/fuse" ) @@ -16,22 +17,17 @@ type groupNode struct { fs.Inode param *FSParam - source GroupSource + source types.RepositoryGroupSource staticNodes map[string]staticNode } -type GroupSource interface { - GetGroupID() uint64 - InvalidateContentCache() -} - // Ensure we are implementing the NodeReaddirer interface var _ = (fs.NodeReaddirer)((*groupNode)(nil)) // Ensure we are implementing the NodeLookuper interface var _ = (fs.NodeLookuper)((*groupNode)(nil)) -func newGroupNodeFromSource(ctx context.Context, source GroupSource, param *FSParam) (fs.InodeEmbedder, error) { +func newGroupNodeFromSource(ctx context.Context, source types.RepositoryGroupSource, param *FSParam) (fs.InodeEmbedder, error) { node := &groupNode{ param: param, source: source, @@ -43,20 +39,20 @@ func newGroupNodeFromSource(ctx context.Context, source GroupSource, param *FSPa } func (n *groupNode) Readdir(ctx context.Context) (fs.DirStream, syscall.Errno) { - groups, repositories, err := n.param.GitForge.FetchGroupContent(ctx, n.source.GetGroupID()) + content, err := n.param.Backend.FetchGroupContent(ctx, n.source) if err != nil { n.param.logger.Error(err.Error()) } - entries := make([]fuse.DirEntry, 0, len(groups)+len(repositories)+len(n.staticNodes)) - for groupName, group := range groups { + entries := make([]fuse.DirEntry, 0, len(content.Groups)+len(content.Repositories)+len(n.staticNodes)) + for groupName, group := range content.Groups { entries = append(entries, fuse.DirEntry{ Name: groupName, Ino: group.GetGroupID() + groupBaseInode, Mode: fuse.S_IFDIR, }) } - for repositoryName, repository := range repositories { + for repositoryName, repository := range content.Repositories { entries = append(entries, fuse.DirEntry{ Name: repositoryName, Ino: repository.GetRepositoryID() + repositoryBaseInode, @@ -74,12 +70,12 @@ func (n *groupNode) Readdir(ctx context.Context) (fs.DirStream, syscall.Errno) { } func (n *groupNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) (*fs.Inode, syscall.Errno) { - groups, repositories, err := n.param.GitForge.FetchGroupContent(ctx, n.source.GetGroupID()) + content, err := n.param.Backend.FetchGroupContent(ctx, n.source) if err != nil { n.param.logger.Error(err.Error()) } else { // Check if the map of groups contains it - group, found := groups[name] + group, found := content.Groups[name] if found { attrs := fs.StableAttr{ Ino: group.GetGroupID() + groupBaseInode, @@ -90,7 +86,7 @@ func (n *groupNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) } // Check if the map of projects contains it - repository, found := repositories[name] + repository, found := content.Repositories[name] if found { attrs := fs.StableAttr{ Ino: repository.GetRepositoryID() + repositoryBaseInode, diff --git a/fstree/refresh.go b/fstree/refresh.go index 25671e9..d8fcb3a 100644 --- a/fstree/refresh.go +++ b/fstree/refresh.go @@ -4,15 +4,17 @@ import ( "context" "syscall" + "github.com/badjware/gitforgefs/types" "github.com/hanwen/go-fuse/v2/fs" "github.com/hanwen/go-fuse/v2/fuse" ) type refreshNode struct { fs.Inode - ino uint64 + ino uint64 + param *FSParam - source GroupSource + source types.RepositoryGroupSource } // Ensure we are implementing the NodeSetattrer interface @@ -21,10 +23,11 @@ var _ = (fs.NodeSetattrer)((*refreshNode)(nil)) // Ensure we are implementing the NodeOpener interface var _ = (fs.NodeOpener)((*refreshNode)(nil)) -func newRefreshNode(source GroupSource, param *FSParam) *refreshNode { +func newRefreshNode(source types.RepositoryGroupSource, param *FSParam) *refreshNode { return &refreshNode{ ino: 0, source: source, + param: param, } } @@ -41,6 +44,6 @@ func (n *refreshNode) Setattr(ctx context.Context, fh fs.FileHandle, in *fuse.Se } func (n *refreshNode) Open(ctx context.Context, flags uint32) (fh fs.FileHandle, fuseFlags uint32, errno syscall.Errno) { - n.source.InvalidateContentCache() + n.param.Backend.InvalidateCache(n.source) return nil, 0, 0 } diff --git a/fstree/repository.go b/fstree/repository.go index ce1f49f..192f548 100644 --- a/fstree/repository.go +++ b/fstree/repository.go @@ -8,6 +8,7 @@ import ( "syscall" "time" + "github.com/badjware/gitforgefs/types" "github.com/hanwen/go-fuse/v2/fs" ) @@ -19,20 +20,13 @@ type repositorySymlinkNode struct { fs.Inode param *FSParam - source RepositorySource -} - -type RepositorySource interface { - // GetName() string - GetRepositoryID() uint64 - GetCloneURL() string - GetDefaultBranch() string + source types.RepositorySource } // Ensure we are implementing the NodeReaddirer interface var _ = (fs.NodeReadlinker)((*repositorySymlinkNode)(nil)) -func newRepositoryNodeFromSource(ctx context.Context, source RepositorySource, param *FSParam) (fs.InodeEmbedder, error) { +func newRepositoryNodeFromSource(ctx context.Context, source types.RepositorySource, param *FSParam) (fs.InodeEmbedder, error) { if param.UseSymlinks { return &repositorySymlinkNode{ param: param, diff --git a/fstree/root.go b/fstree/root.go index c534e59..5ec1295 100644 --- a/fstree/root.go +++ b/fstree/root.go @@ -8,6 +8,7 @@ import ( "os/signal" "syscall" + "github.com/badjware/gitforgefs/types" "github.com/hanwen/go-fuse/v2/fs" "github.com/hanwen/go-fuse/v2/fuse" ) @@ -18,20 +19,11 @@ type staticNode interface { Mode() uint32 } -type GitClient interface { - FetchLocalRepositoryPath(ctx context.Context, source RepositorySource) (string, error) -} - -type GitForge interface { - FetchRootGroupContent(ctx context.Context) (map[string]GroupSource, error) - FetchGroupContent(ctx context.Context, gid uint64) (map[string]GroupSource, map[string]RepositorySource, error) -} - type FSParam struct { UseSymlinks bool - GitClient GitClient - GitForge GitForge + GitClient types.GitClient + Backend types.GitForgeCacher logger *slog.Logger } @@ -71,7 +63,7 @@ func Start(logger *slog.Logger, mountpoint string, mountoptions []string, param } func (n *rootNode) OnAdd(ctx context.Context) { - rootGroups, err := n.param.GitForge.FetchRootGroupContent(ctx) + rootGroups, err := n.param.Backend.FetchRootGroupContent(ctx) if err != nil { panic(err) } diff --git a/git/client.go b/git/client.go index 2532df0..bf060d7 100644 --- a/git/client.go +++ b/git/client.go @@ -10,8 +10,8 @@ import ( "strconv" "github.com/badjware/gitforgefs/config" - "github.com/badjware/gitforgefs/fstree" "github.com/badjware/gitforgefs/queue" + "github.com/badjware/gitforgefs/types" "github.com/badjware/gitforgefs/utils" ) @@ -29,7 +29,7 @@ type gitClient struct { queue queue.TaskQueue } -func NewClient(logger *slog.Logger, p config.GitClientConfig) (*gitClient, error) { +func NewClient(logger *slog.Logger, p config.GitClientConfig) (types.GitClient, error) { // Create the client c := &gitClient{ GitClientConfig: p, @@ -62,7 +62,7 @@ func NewClient(logger *slog.Logger, p config.GitClientConfig) (*gitClient, error return c, nil } -func (c *gitClient) FetchLocalRepositoryPath(ctx context.Context, source fstree.RepositorySource) (localRepoLoc string, err error) { +func (c *gitClient) FetchLocalRepositoryPath(ctx context.Context, source types.RepositorySource) (localRepoLoc string, err error) { rid := source.GetRepositoryID() cloneUrl := source.GetCloneURL() defaultBranch := source.GetDefaultBranch() diff --git a/go.sum b/go.sum index e634f76..a4ee6ef 100644 --- a/go.sum +++ b/go.sum @@ -28,8 +28,12 @@ github.com/hashicorp/go-retryablehttp v0.7.8 h1:ylXZWnqa7Lhqpk0L1P1LzDtGcCR0rPVU github.com/hashicorp/go-retryablehttp v0.7.8/go.mod h1:rjiScheydd+CxvumBsIrFKlx3iS0jrZ7LvzFGFmuKbw= github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/go-version v1.8.0 h1:KAkNb1HAiZd1ukkxDFGmokVZe1Xy9HG6NUp+bPle2i4= +github.com/hashicorp/go-version v1.8.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -42,31 +46,45 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/moby/sys/mountinfo v0.7.2 h1:1shs6aH5s4o5H2zQLn796ADW1wMrIwHsyJ2v9KouLrg= github.com/moby/sys/mountinfo v0.7.2/go.mod h1:1YOa8w8Ih7uW0wALDUgT1dTTSBrZ+HiBLGws92L2RU4= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= gitlab.com/gitlab-org/api/client-go v0.159.0 h1:ibKeribio/OCsrsUz7pkgIN4E7HWDyrw/lDR6P2R7lU= gitlab.com/gitlab-org/api/client-go v0.159.0/go.mod h1:D0DHF7ILUfFo/JcoGMAEndiKMm8SiP/WjyJ4OfXxCKw= +gitlab.com/gitlab-org/api/client-go v1.10.0 h1:VlB9gXQdG6w643lH53VduUHVnCWQG5Ty86VbXnyi70A= +gitlab.com/gitlab-org/api/client-go v1.10.0/go.mod h1:U3QKvjbT1J1FrgLsA7w/XlhoBIendUqB4o3/Ht3UhEQ= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04= golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0= +golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= +golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 h1:mgKeJMpvi0yx/sU5GsxQ7p6s2wtOnGAHZWCHUM4KGzY= golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546/go.mod h1:j/pmGrbnkbPtQfxEe5D0VQhZC6qKbfKifgD0oM7sR70= +golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 h1:fQsdNF2N+/YewlRZiricy4P1iimyPKZ/xwniHj8Q2a0= +golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93/go.mod h1:EPRbTFwzwjXj9NpYyyrvenVh9Y+GFeEvMNh7Xuz7xgU= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/oauth2 v0.33.0 h1:4Q+qn+E5z8gPRJfmRy7C2gGG3T4jIprK6aSYgTXGRpo= golang.org/x/oauth2 v0.33.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= +golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= +golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= +golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.36.0 h1:zMPR+aF8gfksFprF/Nc/rd1wRS1EI6nDBGyWAvDzx2Q= golang.org/x/term v0.36.0/go.mod h1:Qu394IJq6V6dCBRgwqshf3mPF85AqzYEzofzRdZkWss= +golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= @@ -76,6 +94,8 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/main.go b/main.go index a66a81d..fc66cba 100644 --- a/main.go +++ b/main.go @@ -7,12 +7,14 @@ import ( "os" "strings" + "github.com/badjware/gitforgefs/cache" "github.com/badjware/gitforgefs/config" "github.com/badjware/gitforgefs/forges/gitea" "github.com/badjware/gitforgefs/forges/github" "github.com/badjware/gitforgefs/forges/gitlab" "github.com/badjware/gitforgefs/fstree" "github.com/badjware/gitforgefs/git" + "github.com/badjware/gitforgefs/types" ) func main() { @@ -74,7 +76,8 @@ func main() { } gitClient, _ := git.NewClient(logger, *gitClientParam) - var gitForgeClient fstree.GitForge + // setup backend + var gitForgeClient types.GitForge if loadedConfig.FS.Forge == config.ForgeGitlab { // Create the gitlab client gitlabClientConfig, err := config.MakeGitlabConfig(loadedConfig) @@ -101,6 +104,9 @@ func main() { gitForgeClient, _ = gitea.NewClient(logger, *giteaClientConfig) } + // setup cache + cache := cache.NewForgeCache(gitForgeClient, logger) + // Start the filesystem err = fstree.Start( logger, @@ -109,7 +115,7 @@ func main() { &fstree.FSParam{ UseSymlinks: loadedConfig.FS.UseSymlinks, GitClient: gitClient, - GitForge: gitForgeClient, + Backend: cache, }, *debug, ) diff --git a/types/types.go b/types/types.go new file mode 100644 index 0000000..aa3ff25 --- /dev/null +++ b/types/types.go @@ -0,0 +1,42 @@ +package types + +import ( + "context" +) + +type GitClient interface { + FetchLocalRepositoryPath(ctx context.Context, source RepositorySource) (string, error) +} + +type GitForge interface { + FetchRootGroupContent(ctx context.Context) (map[string]RepositoryGroupSource, error) + FetchGroupContent(ctx context.Context, source RepositoryGroupSource) (RepositoryGroupContent, error) +} + +type GitForgeCacher interface { + GitForge + InvalidateCache(source RepositoryGroupSource) +} + +type RepositoryGroupSource interface { + GetGroupID() uint64 + GetGroupName() string + GetGroupPath() string +} + +type RepositorySource interface { + GetRepositoryID() uint64 + GetRepositoryName() string + GetRepositoryPath() string + GetCloneURL() string + GetDefaultBranch() string +} + +type RepositoryGroupContent struct { + // a map of the subgroups contained in this group, keyed by group name + // must not be nil + Groups map[string]RepositoryGroupSource + // a map of the repositories contained in this group, keyed by repository name + // must not be nil + Repositories map[string]RepositorySource +}