diff --git a/.gitignore b/.gitignore index 8410399..8a9438c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ # generate job temporary files .tmp + +db/vimcolorschemes.db diff --git a/cli/generate.go b/cli/generate.go index 4f5e93e..6f375fd 100644 --- a/cli/generate.go +++ b/cli/generate.go @@ -1,21 +1,35 @@ package cli import ( + "context" "encoding/json" "fmt" "log" "os" "os/exec" "strings" - "time" "github.com/vimcolorschemes/worker/internal/database" file "github.com/vimcolorschemes/worker/internal/file" - repoHelper "github.com/vimcolorschemes/worker/internal/repository" - - "go.mongodb.org/mongo-driver/bson" + "github.com/vimcolorschemes/worker/internal/store" ) +type ColorschemeData struct { + Light []store.ColorschemeGroupDefinition + Dark []store.ColorschemeGroupDefinition +} + +type ColorschemeGroupDefinition struct { + Name string + HexCode string +} + +var colorschemeVariantStore *store.ColorschemeVariantStore + +func init() { + colorschemeVariantStore = store.NewColorschemeVariantStore(database.Connect()) +} + var tmpDirectoryPath string var packDirectoryPath string var vimrcPath string @@ -24,7 +38,7 @@ var colorDataFilePath string var debugMode bool // Generate vim color scheme data for all valid repositories -func Generate(force bool, debug bool, repoKey string) bson.M { +func Generate(force bool, debug bool, repoKey string) database.JSONB { debugMode = debug initVimFiles() @@ -33,83 +47,78 @@ func Generate(force bool, debug bool, repoKey string) bson.M { fmt.Println() - var repositories []repoHelper.Repository + var repositories []store.Repository if repoKey != "" { - repository, err := database.GetRepository(repoKey) + repository, err := repositoryStore.GetByKey(context.TODO(), repoKey) if err != nil { log.Panic(err) } - repositories = []repoHelper.Repository{repository} - } else if force || debug { - repositories = database.GetRepositories() + repositories = []store.Repository{*repository} } else { - repositories = database.GetRepositoriesToGenerate() + repositories = repositoryStore.GetAll() } log.Printf("Generating vim preview for %d repositories", len(repositories)) for index, repository := range repositories { - log.Print("\nGenerating vim previews for ", repository.Owner.Name, "/", repository.Name, " (", index+1, "/", len(repositories), ")") + log.Print("\nGenerating vim previews for ", repository.Owner, "/", repository.Name, " (", index+1, "/", len(repositories), ")") - key := fmt.Sprintf("%s__%s", repository.Owner.Name, repository.Name) + key := fmt.Sprintf("%s__%s", repository.Owner, repository.Name) err := installPlugin(repository.GithubURL, key) if err != nil { log.Printf("Error installing plugin: %s", err) - repository.GenerateValid = false - updateRepositoryAfterGenerate(repository) + // repository.GenerateValid = false + // updateRepositoryAfterGenerate(repository) continue } - var data, dataError = getVimColorSchemeColorData() + var dataMap, dataError = getColorschemeDataMap() err = deletePlugin(key) if err != nil { log.Printf("Error deleting plugin: %s", err) } if dataError != nil { log.Printf("Error getting color data: %s", dataError) - repository.GenerateValid = false - updateRepositoryAfterGenerate(repository) + // repository.GenerateValid = false + // updateRepositoryAfterGenerate(repository) continue } - var vimColorSchemes []repoHelper.VimColorScheme - - for name := range data { + for name := range dataMap { if name == "default" || name == "module-injection" || name == "tick_tock" { continue } - var backgrounds []repoHelper.VimBackgroundValue - if data[name].Light != nil { - backgrounds = append(backgrounds, repoHelper.LightBackground) - } - if data[name].Dark != nil { - backgrounds = append(backgrounds, repoHelper.DarkBackground) + if dataMap[name].Light != nil { + err = colorschemeVariantStore.UpsertColorschemeVariant(context.TODO(), &store.ColorschemeVariant{ + Name: name, + RepositoryID: repository.ID, + Background: store.BackgroundLight, + ColorData: dataMap[name].Light, + }) + if err != nil { + log.Printf("Error upserting colorscheme variant: %s", err) + } } - vimColorSchemes = append( - vimColorSchemes, - repoHelper.VimColorScheme{ - Name: name, - Data: data[name], - Backgrounds: backgrounds, + if dataMap[name].Dark != nil { + err = colorschemeVariantStore.UpsertColorschemeVariant(context.TODO(), &store.ColorschemeVariant{ + Name: name, + RepositoryID: repository.ID, + Background: store.BackgroundDark, + ColorData: dataMap[name].Dark, }) + if err != nil { + log.Printf("Error upserting colorscheme variant: %s", err) + } + } } - - repository.VimColorSchemes = vimColorSchemes - repository.GenerateValid = len(repository.VimColorSchemes) > 0 - updateRepositoryAfterGenerate(repository) } cleanUp() - return bson.M{"repositoryCount": len(repositories)} -} - -func updateRepositoryAfterGenerate(repository repoHelper.Repository) { - log.Printf("Generate valid: %v", repository.GenerateValid) - generateObject := getGenerateRepositoryObject(repository) - database.UpsertRepository(repository.ID, generateObject) + log.Printf("Generated %d repositories", len(repositories)) + return database.JSONB{"count": len(repositories)} } // Initializes a temporary directory for vim configuration files @@ -259,22 +268,23 @@ func deletePlugin(key string) error { return err } -// Gathers the colorscheme data from vimcolorschemes/extractor.nvim -func getVimColorSchemeColorData() (map[string]repoHelper.VimColorSchemeData, error) { +// Gathers the colorscheme data from vimcolorschemes/extractor.nvim mapped by +// colorscheme name +func getColorschemeDataMap() (map[string]ColorschemeData, error) { err := executePreviewGenerator() if err != nil { log.Printf("Error executing nvim: %s", err) return nil, err } - vimColorSchemeOutput, err := file.GetLocalFileContent(colorDataFilePath) + colorschemeOutput, err := file.GetLocalFileContent(colorDataFilePath) if err != nil { log.Printf("Error getting local file content from \"%s\": %s", colorDataFilePath, err) return nil, err } - var data map[string]repoHelper.VimColorSchemeData - err = json.Unmarshal([]byte(vimColorSchemeOutput), &data) + var data map[string]ColorschemeData + err = json.Unmarshal([]byte(colorschemeOutput), &data) if err != nil { return nil, err } @@ -325,11 +335,3 @@ func cleanUp() { log.Panic(err) } } - -func getGenerateRepositoryObject(repository repoHelper.Repository) bson.M { - return bson.M{ - "vimColorSchemes": repository.VimColorSchemes, - "generateValid": repository.GenerateValid, - "generatedAt": time.Now(), - } -} diff --git a/cli/import.go b/cli/import.go index fbcfad6..e39d307 100644 --- a/cli/import.go +++ b/cli/import.go @@ -1,6 +1,7 @@ package cli import ( + "context" "log" "math" "strings" @@ -8,14 +9,14 @@ import ( "github.com/vimcolorschemes/worker/internal/database" "github.com/vimcolorschemes/worker/internal/dotenv" "github.com/vimcolorschemes/worker/internal/github" - - "go.mongodb.org/mongo-driver/bson" + "github.com/vimcolorschemes/worker/internal/store" gogithub "github.com/google/go-github/v68/github" ) var repositoryCountLimit int var repositoryCountLimitPerPage int +var repositoryStore *store.RepositoryStore var queries = []string{ "vim theme", @@ -31,6 +32,8 @@ var queries = []string{ } func init() { + repositoryStore = store.NewRepositoryStore(database.Connect()) + repositoryCountLimitValue, err := dotenv.GetInt("GITHUB_REPOSITORY_COUNT_LIMIT") if err != nil { repositoryCountLimitValue = 100 @@ -41,7 +44,7 @@ func init() { } // Import potential vim color scheme repositories from Github -func Import(_force bool, _debug bool, repoKey string) bson.M { +func Import(_force bool, _debug bool, repoKey string) database.JSONB { log.Printf("Repository limit: %d", repositoryCountLimit) var repositories []*gogithub.Repository @@ -62,22 +65,20 @@ func Import(_force bool, _debug bool, repoKey string) bson.M { log.Print("Upserting ", len(repositories), " repositories") for _, repository := range repositories { log.Print("Upserting ", *repository.Name) - repositoryUpdateObject := getImportRepositoryObject(repository) - database.UpsertRepository(*repository.ID, repositoryUpdateObject) + err := repositoryStore.Upsert(context.TODO(), store.Repository{ + ID: *repository.ID, + Name: *repository.Name, + Owner: *repository.Owner.Login, + Description: *repository.Description, + CreatedAt: *repository.CreatedAt.GetTime(), + UpdatedAt: *repository.PushedAt.GetTime(), + GithubURL: *repository.HTMLURL, + }) + if err != nil { + log.Println("Error upserting repository:", err) + } } - return bson.M{"repositoryCount": len(repositories)} -} - -func getImportRepositoryObject(repository *gogithub.Repository) bson.M { - return bson.M{ - "_id": repository.GetID(), - "owner.name": repository.GetOwner().GetLogin(), - "owner.avatarURL": repository.GetOwner().GetAvatarURL(), - "name": repository.GetName(), - "description": repository.GetDescription(), - "githubURL": repository.GetHTMLURL(), - "githubCreatedAt": repository.GetCreatedAt().Time, - "pushedAt": repository.GetPushedAt().Time, - } + log.Printf("Imported %d repositories", len(repositories)) + return database.JSONB{"count": len(repositories)} } diff --git a/cli/update.go b/cli/update.go index b170162..7fd6096 100644 --- a/cli/update.go +++ b/cli/update.go @@ -1,86 +1,66 @@ package cli import ( - "fmt" + "context" "log" - "time" "github.com/vimcolorschemes/worker/internal/database" "github.com/vimcolorschemes/worker/internal/github" - repoHelper "github.com/vimcolorschemes/worker/internal/repository" - - "go.mongodb.org/mongo-driver/bson" + "github.com/vimcolorschemes/worker/internal/store" ) +var repositoryStargazersCountSnapshotStore *store.RepositoryStargarzersCountSnapshotStore + +func init() { + repositoryStargazersCountSnapshotStore = store.NewRepositoryStargazersCountSnapshotStore(database.Connect()) +} + // Update the imported repositories with all kinds of useful information -func Update(_force bool, _debug bool, repoKey string) bson.M { - var repositories []repoHelper.Repository +func Update(_force bool, _debug bool, repoKey string) database.JSONB { + var repositories []store.Repository if repoKey != "" { - repository, err := database.GetRepository(repoKey) + repository, err := repositoryStore.GetByKey(context.TODO(), repoKey) if err != nil { log.Panic(err) } - repositories = []repoHelper.Repository{repository} + repositories = []store.Repository{*repository} } else { - repositories = database.GetRepositories() + repositories = repositoryStore.GetAll() } log.Print(len(repositories), " repositories to update") for index, repository := range repositories { - fmt.Println() - - log.Print("Updating ", index, " of ", len(repositories), ": ", repository.Owner.Name, "/", repository.Name) - - updatedRepository := updateRepository(repository) - - updateObject := getUpdateRepositoryObject(updatedRepository) + log.Print("Updating ", index, " of ", len(repositories), ": ", repository.Owner, "/", repository.Name) - database.UpsertRepository(repository.ID, updateObject) - } - - return bson.M{"repositoryCount": len(repositories)} -} + githubRepository, err := github.GetRepository(repository.Owner, repository.Name) + if err != nil { + log.Print("Error fetching ", repository.Owner, "/", repository.Name) + continue + } -func updateRepository(repository repoHelper.Repository) repoHelper.Repository { - githubRepository, err := github.GetRepository(repository.Owner.Name, repository.Name) - if err != nil { - log.Print("Error fetching ", repository.Owner.Name, "/", repository.Name) - repository.UpdateValid = false - return repository - } + err = repositoryStore.Upsert(context.TODO(), store.Repository{ + ID: *githubRepository.ID, + Name: *githubRepository.Name, + Owner: *githubRepository.Owner.Login, + Description: *githubRepository.Description, + CreatedAt: *githubRepository.CreatedAt.GetTime(), + UpdatedAt: *githubRepository.PushedAt.GetTime(), + GithubURL: *githubRepository.HTMLURL, + }) + if err != nil { + log.Println("Error upserting repository:", err) + } - if githubRepository.PushedAt == nil { - log.Print("No commits on ", repository.Owner.Name, "/", repository.Name) - repository.UpdateValid = false - return repository + err = repositoryStargazersCountSnapshotStore.Insert(context.TODO(), store.RepositoryStargazersCountSnapshot{ + RepositoryID: *githubRepository.ID, + StargazersCount: *githubRepository.StargazersCount, + }) + if err != nil { + log.Println("Error inserting stargazers count:", err) + } } - repository.PushedAt = githubRepository.PushedAt.Time - - log.Print("Gathering basic infos") - repository.StargazersCount = *githubRepository.StargazersCount - - log.Print("Building stargazers count history") - repository.StargazersCountHistory = repository.AppendToStargazersCountHistory() - - log.Print("Computing week stargazers count") - repository.WeekStargazersCount = repository.ComputeTrendingStargazersCount(7) - - log.Print("Checking if ", repository.Owner.Name, "/", repository.Name, " is valid") - repository.UpdateValid = repository.IsValidAfterUpdate() - log.Printf("Update valid: %v", repository.UpdateValid) - - return repository -} - -func getUpdateRepositoryObject(repository repoHelper.Repository) bson.M { - return bson.M{ - "pushedAt": repository.PushedAt, - "stargazersCount": repository.StargazersCount, - "stargazersCountHistory": repository.StargazersCountHistory, - "weekStargazersCount": repository.WeekStargazersCount, - "updateValid": repository.UpdateValid, - "updatedAt": time.Now(), - } + log.Printf("Updated %d repositories", len(repositories)) + return database.JSONB{"count": len(repositories)} } diff --git a/cmd/worker/main.go b/cmd/worker/main.go index 2934ae6..ec2ae3b 100644 --- a/cmd/worker/main.go +++ b/cmd/worker/main.go @@ -1,6 +1,7 @@ package main import ( + "context" "errors" "fmt" "log" @@ -8,19 +9,21 @@ import ( "strings" "time" - "go.mongodb.org/mongo-driver/bson" - "github.com/vimcolorschemes/worker/cli" "github.com/vimcolorschemes/worker/internal/database" + "github.com/vimcolorschemes/worker/internal/store" ) -var jobRunnerMap = map[string]interface{}{ +var jobRunnerMap = map[string]func(force bool, debug bool, repoKey string) database.JSONB{ "import": cli.Import, "update": cli.Update, "generate": cli.Generate, } func main() { + database := database.Connect() + jobReportStore := *store.NewJobReportStore(database) + job, force, debug, repoKey, err := getJobArgs(os.Args) log.Printf("Running %s", job) @@ -38,8 +41,8 @@ func main() { os.Exit(1) } - runner := jobRunnerMap[job] - if runner == nil { + runner, ok := jobRunnerMap[job] + if !ok { log.Print(job, " is not a valid job") os.Exit(1) } @@ -48,10 +51,19 @@ func main() { fmt.Println() - data := runner.(func(force bool, debug bool, repoKey string) bson.M)(force, debug, repoKey) + data := runner(force, debug, repoKey) elapsedTime := time.Since(startTime) - database.CreateReport(job, elapsedTime.Seconds(), data) + + err = jobReportStore.Create(context.TODO(), store.JobReport{ + Job: store.Job(job), + ReportData: data, + ElapsedTimeInSeconds: int64(elapsedTime.Seconds()), + CreatedAt: time.Now(), + }) + if err != nil { + log.Panic(err) + } fmt.Println() log.Printf("Elapsed time: %s\n", elapsedTime) @@ -71,19 +83,16 @@ func getJobArgs(osArgs []string) (string, bool, bool, string, error) { args := osArgs[2:] - forceIndex := getArgIndex(args, "--force") - force := forceIndex != -1 - - debugIndex := getArgIndex(args, "--debug") - debug := debugIndex != -1 + force := getArgIndex(args, "--force") != -1 + debug := getArgIndex(args, "--debug") != -1 repoIndex := getArgIndex(args, "--repo") if repoIndex == -1 || len(args) < repoIndex+1 { - return osArgs[1], force, debug, "", nil + return job, force, debug, "", nil } repoKey := strings.ToLower(args[repoIndex+1]) - return osArgs[1], force, debug, repoKey, nil + return job, force, debug, repoKey, nil } func getArgIndex(args []string, target string) int { diff --git a/db/create-migration b/db/create-migration new file mode 100755 index 0000000..13b4087 --- /dev/null +++ b/db/create-migration @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +file="db/migrations/$(date +%Y%m%d%H%M%S)_$1.sql" + +echo "Creating migration file: $file" + +touch $file diff --git a/db/migrations/20251005184928_repositories.sql b/db/migrations/20251005184928_repositories.sql new file mode 100644 index 0000000..aff4d3f --- /dev/null +++ b/db/migrations/20251005184928_repositories.sql @@ -0,0 +1,11 @@ +CREATE TABLE repositories ( + id INTEGER PRIMARY KEY, + name TEXT NOT NULL, + owner TEXT NOT NULL, + description TEXT, + created_at TIMESTAMP NOT NULL, + updated_at TIMESTAMP NOT NULL, + github_url TEXT NOT NULL UNIQUE, + + UNIQUE(owner, name) +); diff --git a/db/migrations/20251005190841_repositories_stargazers_count.sql b/db/migrations/20251005190841_repositories_stargazers_count.sql new file mode 100644 index 0000000..83b6127 --- /dev/null +++ b/db/migrations/20251005190841_repositories_stargazers_count.sql @@ -0,0 +1,52 @@ +ALTER TABLE repositories ADD COLUMN cached_stargazers_count INTEGER DEFAULT 0; +CREATE INDEX idx_repositories_cached_stargazers_count ON repositories(cached_stargazers_count); + +ALTER TABLE repositories ADD COLUMN cached_weekly_stargazers_count INTEGER DEFAULT 0; +CREATE INDEX idx_repositories_cached_weekly_stargazers_count ON repositories(cached_weekly_stargazers_count); + +CREATE TABLE repository_stargazers_count_snapshots ( + repository_id INTEGER NOT NULL REFERENCES repositories(id) ON DELETE CASCADE, + snapshot_date TEXT NOT NULL, + stargazers_count INTEGER NOT NULL, + + PRIMARY KEY (repository_id, snapshot_date) +); + +CREATE TRIGGER update_cached_stargazers_count +AFTER INSERT ON repository_stargazers_count_snapshots +FOR EACH ROW +BEGIN + UPDATE repositories + SET cached_stargazers_count = ( + SELECT stargazers_count + FROM repository_stargazers_count_snapshots + WHERE repository_id = NEW.repository_id + ORDER BY snapshot_date DESC, stargazers_count DESC + LIMIT 1 + ) + WHERE id = NEW.repository_id; +END; + +CREATE TRIGGER update_cached_weekly_stargazers_count +AFTER INSERT ON repository_stargazers_count_snapshots +FOR EACH ROW +BEGIN + UPDATE repositories + SET cached_weekly_stargazers_count = + NEW.stargazers_count - + ( + SELECT COALESCE( + ( + SELECT stargazers_count + FROM repository_stargazers_count_snapshots + WHERE repository_id = NEW.repository_id + AND snapshot_date >= date(NEW.snapshot_date, '-7 days') + ORDER BY snapshot_date ASC, stargazers_count DESC + LIMIT 1 + ), + 0 + ) + ) + WHERE id = NEW.repository_id; +END; + diff --git a/db/migrations/20251005195216_colorscheme_variants.sql b/db/migrations/20251005195216_colorscheme_variants.sql new file mode 100644 index 0000000..2adafc4 --- /dev/null +++ b/db/migrations/20251005195216_colorscheme_variants.sql @@ -0,0 +1,9 @@ +CREATE TABLE colorscheme_variants ( + colorscheme_name TEXT NOT NULL, + repository_id INTEGER NOT NULL REFERENCES repositories(id) ON DELETE CASCADE, + background TEXT NOT NULL CHECK (background IN ('light', 'dark')), + color_data JSON NOT NULL, + + PRIMARY KEY (colorscheme_name, repository_id, background) + FOREIGN KEY (colorscheme_name, repository_id) REFERENCES colorschemes(name, repository_id) ON DELETE CASCADE +) diff --git a/db/migrations/20251005200040_job_reports.sql b/db/migrations/20251005200040_job_reports.sql new file mode 100644 index 0000000..ffacfa7 --- /dev/null +++ b/db/migrations/20251005200040_job_reports.sql @@ -0,0 +1,8 @@ +CREATE TABLE job_reports ( + job TEXT NOT NULL CHECK (job IN ('import', 'update', 'generate')), + report_data JSONB NOT NULL, + elapsed_time_in_seconds INTEGER NOT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + + PRIMARY KEY (job, created_at) +); diff --git a/db/migrations/20251005200455_repository_job_reports.sql b/db/migrations/20251005200455_repository_job_reports.sql new file mode 100644 index 0000000..7d54551 --- /dev/null +++ b/db/migrations/20251005200455_repository_job_reports.sql @@ -0,0 +1,9 @@ +CREATE TABLE repository_job_reports ( + job TEXT NOT NULL CHECK (job IN ('import', 'update', 'generate')), + repository_id INTEGER NOT NULL REFERENCES repositories(id) ON DELETE CASCADE, + success BOOLEAN NOT NULL, + error TEXT, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + + PRIMARY KEY (job, repository_id, created_at) +); diff --git a/db/reset b/db/reset new file mode 100755 index 0000000..d828b50 --- /dev/null +++ b/db/reset @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +if ! command -v sqlite3 &> /dev/null +then + echo "sqlite3 could not be found, please install it first." + exit +fi + +DB_FILE="db/vimcolorschemes.db" + +if [ -f "$DB_FILE" ]; then + rm "$DB_FILE" + echo "Database '$DB_FILE' has been deleted." +else + echo "Database '$DB_FILE' does not exist." +fi + +db/setup diff --git a/db/run-migrations b/db/run-migrations new file mode 100755 index 0000000..c694826 --- /dev/null +++ b/db/run-migrations @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +if ! command -v sqlite3 &> /dev/null +then + echo "sqlite3 could not be found, please install it first." + exit +fi + +DB_FILE="db/vimcolorschemes.db" +MIGRATIONS_DIR="db/migrations" + +for migration in "$MIGRATIONS_DIR"/*.sql; do + MIGRATION_ID=$(basename "$migration") + + # Skip if already applied + if sqlite3 "$DB_FILE" -batch -bail "SELECT 1 FROM migrations WHERE id = '$MIGRATION_ID' LIMIT 1;" | grep -q 1; then + continue + fi + + echo "Applying migration: $MIGRATION_ID" + sqlite3 "$DB_FILE" -batch -bail < /dev/null +then + echo "sqlite3 could not be found, please install it first." + exit +fi + +DB_FILE="db/vimcolorschemes.db" +MIGRATIONS_DIR="db/migrations" + +if [ ! -f "$DB_FILE" ]; then + sqlite3 "$DB_FILE" <