From 0c29b0eb76b445811a38c3042fd194cf7a89bbb9 Mon Sep 17 00:00:00 2001 From: Robin Gagnon Date: Sun, 5 Oct 2025 18:54:06 -0500 Subject: [PATCH 01/17] build: Add DB setup script and migrations --- .gitignore | 2 ++ db/create-migration | 7 +++++ db/migrations/20251005184928_repositories.sql | 9 ++++++ db/setup | 31 +++++++++++++++++++ 4 files changed, 49 insertions(+) create mode 100755 db/create-migration create mode 100644 db/migrations/20251005184928_repositories.sql create mode 100755 db/setup diff --git a/.gitignore b/.gitignore index 8410399..8a9438c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ # generate job temporary files .tmp + +db/vimcolorschemes.db diff --git a/db/create-migration b/db/create-migration new file mode 100755 index 0000000..13b4087 --- /dev/null +++ b/db/create-migration @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +file="db/migrations/$(date +%Y%m%d%H%M%S)_$1.sql" + +echo "Creating migration file: $file" + +touch $file diff --git a/db/migrations/20251005184928_repositories.sql b/db/migrations/20251005184928_repositories.sql new file mode 100644 index 0000000..30c215e --- /dev/null +++ b/db/migrations/20251005184928_repositories.sql @@ -0,0 +1,9 @@ +CREATE TABLE repositories ( + id INTEGER PRIMARY KEY, + name TEXT NOT NULL, + owner TEXT NOT NULL, + description TEXT, + created_at TIMESTAMP NOT NULL, + updated_at TIMESTAMP NOT NULL, + UNIQUE(owner, name) +); diff --git a/db/setup b/db/setup new file mode 100755 index 0000000..dfcb6f6 --- /dev/null +++ b/db/setup @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +if ! command -v sqlite3 &> /dev/null +then + echo "sqlite3 could not be found, please install it first." + exit +fi + +DB_FILE="db/vimcolorschemes.db" +MIGRATIONS_DIR="db/migrations" + +if [ ! -f "$DB_FILE" ]; then + sqlite3 "$DB_FILE" < Date: Sun, 5 Oct 2025 18:56:49 -0500 Subject: [PATCH 02/17] build: Add DB reset script --- db/reset | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100755 db/reset diff --git a/db/reset b/db/reset new file mode 100755 index 0000000..d828b50 --- /dev/null +++ b/db/reset @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +if ! command -v sqlite3 &> /dev/null +then + echo "sqlite3 could not be found, please install it first." + exit +fi + +DB_FILE="db/vimcolorschemes.db" + +if [ -f "$DB_FILE" ]; then + rm "$DB_FILE" + echo "Database '$DB_FILE' has been deleted." +else + echo "Database '$DB_FILE' does not exist." +fi + +db/setup From 7fe7ca2ab5e2b992b76384c0b2f6b4ec6b16aee1 Mon Sep 17 00:00:00 2001 From: Robin Gagnon Date: Sun, 5 Oct 2025 19:43:46 -0500 Subject: [PATCH 03/17] build: Add stargazers count columns and triggers --- ...05190841_repositories_stargazers_count.sql | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 db/migrations/20251005190841_repositories_stargazers_count.sql diff --git a/db/migrations/20251005190841_repositories_stargazers_count.sql b/db/migrations/20251005190841_repositories_stargazers_count.sql new file mode 100644 index 0000000..658e735 --- /dev/null +++ b/db/migrations/20251005190841_repositories_stargazers_count.sql @@ -0,0 +1,51 @@ +ALTER TABLE repositories ADD COLUMN cached_stargazers_count INTEGER DEFAULT 0; +CREATE INDEX idx_repositories_cached_stargazers_count ON repositories(cached_stargazers_count); + +ALTER TABLE repositories ADD COLUMN cached_weekly_stargazers_count INTEGER DEFAULT 0; +CREATE INDEX idx_repositories_cached_weekly_stargazers_count ON repositories(cached_weekly_stargazers_count); + +CREATE TABLE repository_stargazers_count_snapshots ( + repository_id INTEGER NOT NULL REFERENCES repositories(id) ON DELETE CASCADE, + snapshot_date TEXT NOT NULL, + stargazers_count INTEGER NOT NULL, + PRIMARY KEY (repository_id, snapshot_date) +); + +CREATE TRIGGER update_cached_stargazers_count +AFTER INSERT ON repository_stargazers_count_snapshots +FOR EACH ROW +BEGIN + UPDATE repositories + SET cached_stargazers_count = ( + SELECT stargazers_count + FROM repository_stargazers_count_snapshots + WHERE repository_id = NEW.repository_id + ORDER BY snapshot_date DESC, stargazers_count DESC + LIMIT 1 + ) + WHERE id = NEW.repository_id; +END; + +CREATE TRIGGER update_cached_weekly_stargazers_count +AFTER INSERT ON repository_stargazers_count_snapshots +FOR EACH ROW +BEGIN + UPDATE repositories + SET cached_weekly_stargazers_count = + NEW.stargazers_count - + ( + SELECT COALESCE( + ( + SELECT stargazers_count + FROM repository_stargazers_count_snapshots + WHERE repository_id = NEW.repository_id + AND snapshot_date >= date(NEW.snapshot_date, '-7 days') + ORDER BY snapshot_date ASC, stargazers_count DESC + LIMIT 1 + ), + 0 + ) + ) + WHERE id = NEW.repository_id; +END; + From 0907deeb7866076b5f8b2312cab3fe2b0a1d1815 Mon Sep 17 00:00:00 2001 From: Robin Gagnon Date: Sun, 5 Oct 2025 19:59:04 -0500 Subject: [PATCH 04/17] build: Add colorscheme data tables --- db/migrations/20251005184928_repositories.sql | 1 + ...251005190841_repositories_stargazers_count.sql | 1 + db/migrations/20251005195216_colorschemes.sql | 15 +++++++++++++++ 3 files changed, 17 insertions(+) create mode 100644 db/migrations/20251005195216_colorschemes.sql diff --git a/db/migrations/20251005184928_repositories.sql b/db/migrations/20251005184928_repositories.sql index 30c215e..200eabb 100644 --- a/db/migrations/20251005184928_repositories.sql +++ b/db/migrations/20251005184928_repositories.sql @@ -5,5 +5,6 @@ CREATE TABLE repositories ( description TEXT, created_at TIMESTAMP NOT NULL, updated_at TIMESTAMP NOT NULL, + UNIQUE(owner, name) ); diff --git a/db/migrations/20251005190841_repositories_stargazers_count.sql b/db/migrations/20251005190841_repositories_stargazers_count.sql index 658e735..83b6127 100644 --- a/db/migrations/20251005190841_repositories_stargazers_count.sql +++ b/db/migrations/20251005190841_repositories_stargazers_count.sql @@ -8,6 +8,7 @@ CREATE TABLE repository_stargazers_count_snapshots ( repository_id INTEGER NOT NULL REFERENCES repositories(id) ON DELETE CASCADE, snapshot_date TEXT NOT NULL, stargazers_count INTEGER NOT NULL, + PRIMARY KEY (repository_id, snapshot_date) ); diff --git a/db/migrations/20251005195216_colorschemes.sql b/db/migrations/20251005195216_colorschemes.sql new file mode 100644 index 0000000..aa08a61 --- /dev/null +++ b/db/migrations/20251005195216_colorschemes.sql @@ -0,0 +1,15 @@ +CREATE TABLE colorschemes ( + name TEXT NOT NULL, + repository_id INTEGER NOT NULL REFERENCES repositories(id) ON DELETE CASCADE, + + PRIMARY KEY (name, repository_id) +); + +CREATE TABLE colorscheme_color_data ( + colorscheme_name TEXT NOT NULL, + repository_id INTEGER NOT NULL, + background TEXT NOT NULL CHECK (background IN ('light', 'dark')), + color_data TEXT NOT NULL, + + PRIMARY KEY (colorscheme_name, repository_id, background) +) From 680dcce4fd315d5645a26012ef25304e826883d5 Mon Sep 17 00:00:00 2001 From: Robin Gagnon Date: Sun, 5 Oct 2025 20:04:26 -0500 Subject: [PATCH 05/17] build: Add job_reports table --- db/migrations/20251005200040_job_reports.sql | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 db/migrations/20251005200040_job_reports.sql diff --git a/db/migrations/20251005200040_job_reports.sql b/db/migrations/20251005200040_job_reports.sql new file mode 100644 index 0000000..0f6c006 --- /dev/null +++ b/db/migrations/20251005200040_job_reports.sql @@ -0,0 +1,7 @@ +CREATE TABLE job_reports ( + id SERIAL PRIMARY KEY, + job TEXT NOT NULL CHECK (job IN ('import', 'update', 'generate')), + report_data JSONB NOT NULL, + elapsed_time_in_seconds INTEGER NOT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); From b640a138b2df0a33584fad7bb7f5bacdc13a1cce Mon Sep 17 00:00:00 2001 From: Robin Gagnon Date: Sun, 5 Oct 2025 20:21:37 -0500 Subject: [PATCH 06/17] build: Add repository job reports table --- db/migrations/20251005200455_repository_job_reports.sql | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 db/migrations/20251005200455_repository_job_reports.sql diff --git a/db/migrations/20251005200455_repository_job_reports.sql b/db/migrations/20251005200455_repository_job_reports.sql new file mode 100644 index 0000000..7d54551 --- /dev/null +++ b/db/migrations/20251005200455_repository_job_reports.sql @@ -0,0 +1,9 @@ +CREATE TABLE repository_job_reports ( + job TEXT NOT NULL CHECK (job IN ('import', 'update', 'generate')), + repository_id INTEGER NOT NULL REFERENCES repositories(id) ON DELETE CASCADE, + success BOOLEAN NOT NULL, + error TEXT, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + + PRIMARY KEY (job, repository_id, created_at) +); From c84d518ea779bd09eb2faeda721bf437ed7fd718 Mon Sep 17 00:00:00 2001 From: Robin Gagnon Date: Sun, 5 Oct 2025 20:21:53 -0500 Subject: [PATCH 07/17] build: Add error handling to migrations setup --- db/setup | 40 ++++++++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/db/setup b/db/setup index dfcb6f6..d522598 100755 --- a/db/setup +++ b/db/setup @@ -2,30 +2,46 @@ if ! command -v sqlite3 &> /dev/null then - echo "sqlite3 could not be found, please install it first." - exit + echo "sqlite3 could not be found, please install it first." + exit fi DB_FILE="db/vimcolorschemes.db" MIGRATIONS_DIR="db/migrations" if [ ! -f "$DB_FILE" ]; then - sqlite3 "$DB_FILE" < Date: Fri, 10 Oct 2025 21:01:30 -0500 Subject: [PATCH 08/17] feat: Connect import script to sqlite --- cli/generate.go | 23 +++--- cli/import.go | 37 +++++---- cli/update.go | 24 +++--- cmd/worker/main.go | 24 +++--- go.mod | 17 ++-- go.sum | 58 +++----------- internal/database/database.go | 144 ++++++---------------------------- internal/store/repository.go | 70 +++++++++++++++++ 8 files changed, 161 insertions(+), 236 deletions(-) create mode 100644 internal/store/repository.go diff --git a/cli/generate.go b/cli/generate.go index 4f5e93e..4a7c09b 100644 --- a/cli/generate.go +++ b/cli/generate.go @@ -9,7 +9,6 @@ import ( "strings" "time" - "github.com/vimcolorschemes/worker/internal/database" file "github.com/vimcolorschemes/worker/internal/file" repoHelper "github.com/vimcolorschemes/worker/internal/repository" @@ -24,7 +23,7 @@ var colorDataFilePath string var debugMode bool // Generate vim color scheme data for all valid repositories -func Generate(force bool, debug bool, repoKey string) bson.M { +func Generate(force bool, debug bool, repoKey string) int { debugMode = debug initVimFiles() @@ -35,15 +34,15 @@ func Generate(force bool, debug bool, repoKey string) bson.M { var repositories []repoHelper.Repository if repoKey != "" { - repository, err := database.GetRepository(repoKey) - if err != nil { - log.Panic(err) - } - repositories = []repoHelper.Repository{repository} + // repository, err := database.GetRepository(repoKey) + // if err != nil { + // log.Panic(err) + // } + // repositories = []repoHelper.Repository{repository} } else if force || debug { - repositories = database.GetRepositories() + // repositories = database.GetRepositories() } else { - repositories = database.GetRepositoriesToGenerate() + // repositories = database.GetRepositoriesToGenerate() } log.Printf("Generating vim preview for %d repositories", len(repositories)) @@ -103,13 +102,13 @@ func Generate(force bool, debug bool, repoKey string) bson.M { cleanUp() - return bson.M{"repositoryCount": len(repositories)} + return len(repositories) } func updateRepositoryAfterGenerate(repository repoHelper.Repository) { log.Printf("Generate valid: %v", repository.GenerateValid) - generateObject := getGenerateRepositoryObject(repository) - database.UpsertRepository(repository.ID, generateObject) + // generateObject := getGenerateRepositoryObject(repository) + // database.UpsertRepository(repository.ID, generateObject) } // Initializes a temporary directory for vim configuration files diff --git a/cli/import.go b/cli/import.go index fbcfad6..7acd289 100644 --- a/cli/import.go +++ b/cli/import.go @@ -1,6 +1,7 @@ package cli import ( + "context" "log" "math" "strings" @@ -8,14 +9,14 @@ import ( "github.com/vimcolorschemes/worker/internal/database" "github.com/vimcolorschemes/worker/internal/dotenv" "github.com/vimcolorschemes/worker/internal/github" - - "go.mongodb.org/mongo-driver/bson" + "github.com/vimcolorschemes/worker/internal/store" gogithub "github.com/google/go-github/v68/github" ) var repositoryCountLimit int var repositoryCountLimitPerPage int +var repositoryStore *store.RepositoryStore var queries = []string{ "vim theme", @@ -31,6 +32,8 @@ var queries = []string{ } func init() { + repositoryStore = store.NewRepositoryStore(database.Connect()) + repositoryCountLimitValue, err := dotenv.GetInt("GITHUB_REPOSITORY_COUNT_LIMIT") if err != nil { repositoryCountLimitValue = 100 @@ -41,7 +44,7 @@ func init() { } // Import potential vim color scheme repositories from Github -func Import(_force bool, _debug bool, repoKey string) bson.M { +func Import(_force bool, _debug bool, repoKey string) int { log.Printf("Repository limit: %d", repositoryCountLimit) var repositories []*gogithub.Repository @@ -62,22 +65,18 @@ func Import(_force bool, _debug bool, repoKey string) bson.M { log.Print("Upserting ", len(repositories), " repositories") for _, repository := range repositories { log.Print("Upserting ", *repository.Name) - repositoryUpdateObject := getImportRepositoryObject(repository) - database.UpsertRepository(*repository.ID, repositoryUpdateObject) + err := repositoryStore.Upsert(context.TODO(), store.Repository{ + ID: *repository.ID, + Name: *repository.Name, + Owner: *repository.Owner.Login, + Description: *repository.Description, + CreatedAt: *repository.CreatedAt.GetTime(), + UpdatedAt: *repository.PushedAt.GetTime(), + }) + if err != nil { + log.Println("Error upserting repository:", err) + } } - return bson.M{"repositoryCount": len(repositories)} -} - -func getImportRepositoryObject(repository *gogithub.Repository) bson.M { - return bson.M{ - "_id": repository.GetID(), - "owner.name": repository.GetOwner().GetLogin(), - "owner.avatarURL": repository.GetOwner().GetAvatarURL(), - "name": repository.GetName(), - "description": repository.GetDescription(), - "githubURL": repository.GetHTMLURL(), - "githubCreatedAt": repository.GetCreatedAt().Time, - "pushedAt": repository.GetPushedAt().Time, - } + return len(repositories) } diff --git a/cli/update.go b/cli/update.go index b170162..9623b46 100644 --- a/cli/update.go +++ b/cli/update.go @@ -5,7 +5,7 @@ import ( "log" "time" - "github.com/vimcolorschemes/worker/internal/database" + // "github.com/vimcolorschemes/worker/internal/database" "github.com/vimcolorschemes/worker/internal/github" repoHelper "github.com/vimcolorschemes/worker/internal/repository" @@ -13,16 +13,16 @@ import ( ) // Update the imported repositories with all kinds of useful information -func Update(_force bool, _debug bool, repoKey string) bson.M { +func Update(_force bool, _debug bool, repoKey string) int { var repositories []repoHelper.Repository if repoKey != "" { - repository, err := database.GetRepository(repoKey) - if err != nil { - log.Panic(err) - } - repositories = []repoHelper.Repository{repository} + // repository, err := database.GetRepository(repoKey) + // if err != nil { + // log.Panic(err) + // } + // repositories = []repoHelper.Repository{repository} } else { - repositories = database.GetRepositories() + // repositories = database.GetRepositories() } log.Print(len(repositories), " repositories to update") @@ -32,14 +32,14 @@ func Update(_force bool, _debug bool, repoKey string) bson.M { log.Print("Updating ", index, " of ", len(repositories), ": ", repository.Owner.Name, "/", repository.Name) - updatedRepository := updateRepository(repository) + // updatedRepository := updateRepository(repository) - updateObject := getUpdateRepositoryObject(updatedRepository) + // updateObject := getUpdateRepositoryObject(updatedRepository) - database.UpsertRepository(repository.ID, updateObject) + // database.UpsertRepository(repository.ID, updateObject) } - return bson.M{"repositoryCount": len(repositories)} + return len(repositories) } func updateRepository(repository repoHelper.Repository) repoHelper.Repository { diff --git a/cmd/worker/main.go b/cmd/worker/main.go index 2934ae6..c9f821a 100644 --- a/cmd/worker/main.go +++ b/cmd/worker/main.go @@ -8,13 +8,10 @@ import ( "strings" "time" - "go.mongodb.org/mongo-driver/bson" - "github.com/vimcolorschemes/worker/cli" - "github.com/vimcolorschemes/worker/internal/database" ) -var jobRunnerMap = map[string]interface{}{ +var jobRunnerMap = map[string]func(force bool, debug bool, repoKey string) int{ "import": cli.Import, "update": cli.Update, "generate": cli.Generate, @@ -38,8 +35,8 @@ func main() { os.Exit(1) } - runner := jobRunnerMap[job] - if runner == nil { + runner, ok := jobRunnerMap[job] + if !ok { log.Print(job, " is not a valid job") os.Exit(1) } @@ -48,10 +45,10 @@ func main() { fmt.Println() - data := runner.(func(force bool, debug bool, repoKey string) bson.M)(force, debug, repoKey) + data := runner(force, debug, repoKey) + log.Printf("Processed %d items", data) elapsedTime := time.Since(startTime) - database.CreateReport(job, elapsedTime.Seconds(), data) fmt.Println() log.Printf("Elapsed time: %s\n", elapsedTime) @@ -71,19 +68,16 @@ func getJobArgs(osArgs []string) (string, bool, bool, string, error) { args := osArgs[2:] - forceIndex := getArgIndex(args, "--force") - force := forceIndex != -1 - - debugIndex := getArgIndex(args, "--debug") - debug := debugIndex != -1 + force := getArgIndex(args, "--force") != -1 + debug := getArgIndex(args, "--debug") != -1 repoIndex := getArgIndex(args, "--repo") if repoIndex == -1 || len(args) < repoIndex+1 { - return osArgs[1], force, debug, "", nil + return job, force, debug, "", nil } repoKey := strings.ToLower(args[repoIndex+1]) - return osArgs[1], force, debug, repoKey, nil + return job, force, debug, repoKey, nil } func getArgIndex(args []string, target string) int { diff --git a/go.mod b/go.mod index a71d44c..e3a8ea7 100644 --- a/go.mod +++ b/go.mod @@ -5,20 +5,15 @@ go 1.23 require ( github.com/google/go-github/v68 v68.0.0 github.com/joho/godotenv v1.5.1 - go.mongodb.org/mongo-driver v1.17.2 + github.com/mattn/go-sqlite3 v1.14.32 + github.com/tursodatabase/libsql-client-go v0.0.0-20240902231107-85af5b9d094d + go.mongodb.org/mongo-driver v1.17.4 golang.org/x/oauth2 v0.25.0 ) require ( - github.com/golang/snappy v0.0.4 // indirect + github.com/antlr4-go/antlr/v4 v4.13.0 // indirect + github.com/coder/websocket v1.8.12 // indirect github.com/google/go-querystring v1.1.0 // indirect - github.com/klauspost/compress v1.17.11 // indirect - github.com/montanaflynn/stats v0.7.1 // indirect - github.com/xdg-go/pbkdf2 v1.0.0 // indirect - github.com/xdg-go/scram v1.1.2 // indirect - github.com/xdg-go/stringprep v1.0.4 // indirect - github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect - golang.org/x/crypto v0.32.0 // indirect - golang.org/x/sync v0.10.0 // indirect - golang.org/x/text v0.21.0 // indirect + golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8 // indirect ) diff --git a/go.sum b/go.sum index 03b52af..8ddd6b6 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,9 @@ +github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= +github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= +github.com/coder/websocket v1.8.12 h1:5bUXkEPPIbewrnkU8LTCLVaxi4N4J8ahufH2vlo4NAo= +github.com/coder/websocket v1.8.12/go.mod h1:LNVeNrXQZfe5qhS9ALED3uA+l5pPqvwXg3CKoDBB2gs= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= -github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -11,50 +13,14 @@ github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= -github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= -github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= -github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE= -github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= -github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= -github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= -github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= -github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= -github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= -github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= -github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM= -github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.mongodb.org/mongo-driver v1.17.2 h1:gvZyk8352qSfzyZ2UMWcpDpMSGEr1eqE4T793SqyhzM= -go.mongodb.org/mongo-driver v1.17.2/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc= -golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +github.com/mattn/go-sqlite3 v1.14.32 h1:JD12Ag3oLy1zQA+BNn74xRgaBbdhbNIDYvQUEuuErjs= +github.com/mattn/go-sqlite3 v1.14.32/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/tursodatabase/libsql-client-go v0.0.0-20240902231107-85af5b9d094d h1:dOMI4+zEbDI37KGb0TI44GUAwxHF9cMsIoDTJ7UmgfU= +github.com/tursodatabase/libsql-client-go v0.0.0-20240902231107-85af5b9d094d/go.mod h1:l8xTsYB90uaVdMHXMCxKKLSgw5wLYBwBKKefNIUnm9s= +go.mongodb.org/mongo-driver v1.17.4 h1:jUorfmVzljjr0FLzYQsGP8cgN/qzzxlY9Vh0C9KFXVw= +go.mongodb.org/mongo-driver v1.17.4/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ= +golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8 h1:aAcj0Da7eBAtrTp03QXWvm88pSyOt+UgdZw2BFZ+lEw= +golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8/go.mod h1:CQ1k9gNrJ50XIzaKCRR2hssIjF07kZFEiieALBM/ARQ= golang.org/x/oauth2 v0.25.0 h1:CY4y7XT9v0cRI9oupztF8AgiIu99L/ksR/Xp/6jrZ70= golang.org/x/oauth2 v0.25.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= -golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= -golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/internal/database/database.go b/internal/database/database.go index 1a12b54..f2f54fc 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -1,139 +1,41 @@ package database import ( - "context" - "errors" + "database/sql" "log" "os" "strings" - "time" - "github.com/vimcolorschemes/worker/internal/dotenv" - "github.com/vimcolorschemes/worker/internal/repository" - - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" + _ "github.com/mattn/go-sqlite3" + _ "github.com/tursodatabase/libsql-client-go/libsql" ) -var ctx = context.TODO() -var repositoriesCollection *mongo.Collection -var reportsCollection *mongo.Collection +func Connect() *sql.DB { + url := os.Getenv("TURSO_DATABASE_URL") + token := os.Getenv("TURSO_AUTH_TOKEN") -func init() { - if strings.HasSuffix(os.Args[0], ".test") { - // Running in test mode - return - } + var driver, dsn string - connectionString, exists := dotenv.Get("MONGODB_CONNECTION_STRING") - if !exists { - log.Panic("Database connection string not found in env") + if strings.HasPrefix(url, "libsql://") { + driver = "libsql" + sep := "?" + if strings.Contains(url, "?") { + sep = "&" + } + dsn = url + if token != "" { + dsn = dsn + sep + "authToken=" + token + } + } else { + driver, dsn = "sqlite3", "./db/vimcolorschemes.db" } - clientOptions := options.Client().ApplyURI(connectionString) - - databaseUsername, usernameExists := dotenv.Get("MONGODB_USERNAME") - databasePassword, passwordExists := dotenv.Get("MONGODB_PASSWORD") - if usernameExists && databaseUsername != "" && passwordExists && databasePassword != "" { - credentials := options.Credential{Username: databaseUsername, Password: databasePassword} - clientOptions.SetAuth(credentials) - } + log.Printf("Connecting to database with driver %s and dsn %s", driver, dsn) - client, err := mongo.Connect(ctx, clientOptions) + db, err := sql.Open(driver, dsn) if err != nil { - panic(err) + log.Fatal(err) } - err = client.Ping(ctx, nil) - if err != nil { - panic(err) - } - - databaseName, databaseNameExists := dotenv.Get("MONGODB_DATABASE") - if !databaseNameExists { - databaseName = "vimcolorschemes" - } - - database := client.Database(databaseName) - repositoriesCollection = database.Collection("repositories") - reportsCollection = database.Collection("reports") -} - -// GetRepositories gets all repositories stored in the database -func GetRepositories() []repository.Repository { - return getRepositories(bson.M{}) -} - -// GetRepository gets the repository matching the repository key -func GetRepository(repoKey string) (repository.Repository, error) { - matches := strings.Split(repoKey, "/") - - if len(matches) < 2 { - return repository.Repository{}, errors.New("key not valid") - } - - var repo repository.Repository - - ownerName := bson.M{"$regex": matches[0], "$options": "i"} - name := bson.M{"$regex": matches[1], "$options": "i"} - err := repositoriesCollection.FindOne(ctx, bson.M{"owner.name": ownerName, "name": name}).Decode(&repo) - if err != nil { - return repository.Repository{}, err - } - - return repo, nil -} - -// GetRepositoriesToGenerate gets all repositories that are due for a preview -// generate. -func GetRepositoriesToGenerate() []repository.Repository { - return getRepositories(bson.M{"updateValid": true, "$expr": bson.M{"$gt": []string{"$pushedAt", "$generatedAt"}}}) -} - -func getRepositories(filter bson.M) []repository.Repository { - var repositories []repository.Repository - cursor, err := repositoriesCollection.Find(ctx, filter) - if err != nil { - log.Print("here") - panic(err) - } - if err = cursor.All(ctx, &repositories); err != nil { - log.Print("or here") - panic(err) - } - return repositories -} - -// UpsertRepository updates the repository if it exists, inserts it if not -func UpsertRepository(id int64, updateObject bson.M) { - filter := bson.M{"_id": id} - - update := bson.M{"$set": updateObject} - delete(updateObject, "_id") - - upsertOptions := options.Update().SetUpsert(true) - - _, err := repositoriesCollection.UpdateOne(ctx, filter, update, upsertOptions) - - if err != nil { - log.Printf("Error upserting repository: %s", err) - panic(err) - } -} - -// CreateReport stores a job report in the database -func CreateReport(job string, elapsedTime float64, data bson.M) { - object := bson.M{ - "date": time.Now(), - "job": job, - "elapsedTime": elapsedTime, - "data": data, - } - _, err := reportsCollection.InsertOne(ctx, object, &options.InsertOneOptions{}) - - if err != nil { - log.Printf("Error creating report: %s", err) - panic(err) - } + return db } diff --git a/internal/store/repository.go b/internal/store/repository.go new file mode 100644 index 0000000..4e5a82b --- /dev/null +++ b/internal/store/repository.go @@ -0,0 +1,70 @@ +package store + +import ( + "context" + "database/sql" + "errors" + "time" +) + +type Repository struct { + ID int64 `db:"id"` + Name string `db:"name"` + Owner string `db:"owner"` + Description string `db:"description"` + CreatedAt time.Time `db:"created_at"` + UpdatedAt time.Time `db:"updated_at"` +} + +type RepositoryStore struct { + database *sql.DB +} + +func NewRepositoryStore(database *sql.DB) *RepositoryStore { + return &RepositoryStore{database: database} +} + +func (store *RepositoryStore) GetByKey(ctx context.Context, owner, name string) (*Repository, error) { + var r Repository + err := store.database.QueryRowContext(ctx, ` + SELECT id, owner, name, description, created_at, updated_at + FROM repositories + WHERE owner = ? AND name = ? + `, owner, name).Scan( + &r.ID, + &r.Owner, + &r.Name, + &r.Description, + &r.CreatedAt, + &r.UpdatedAt, + ) + + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } + return nil, err + } + + return &r, nil +} + +func (store *RepositoryStore) Upsert(ctx context.Context, r Repository) error { + _, err := store.database.ExecContext(ctx, ` + INSERT INTO repositories (id, owner, name, description, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT(id) DO UPDATE SET + owner = excluded.owner, + name = excluded.name, + description = excluded.description, + updated_at = excluded.updated_at; + `, + r.ID, + r.Owner, + r.Name, + r.Description, + r.CreatedAt, + r.UpdatedAt, + ) + return err +} From 5408608a9afa3d0c1ce0a150a9ae65d3dc14288e Mon Sep 17 00:00:00 2001 From: Robin Gagnon Date: Sun, 12 Oct 2025 19:55:02 -0500 Subject: [PATCH 09/17] build: Split migrations and DB setup scripts --- db/run-migrations | 33 +++++++++++++++++++++++++++++++++ db/setup | 25 ------------------------- 2 files changed, 33 insertions(+), 25 deletions(-) create mode 100755 db/run-migrations diff --git a/db/run-migrations b/db/run-migrations new file mode 100755 index 0000000..c694826 --- /dev/null +++ b/db/run-migrations @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +if ! command -v sqlite3 &> /dev/null +then + echo "sqlite3 could not be found, please install it first." + exit +fi + +DB_FILE="db/vimcolorschemes.db" +MIGRATIONS_DIR="db/migrations" + +for migration in "$MIGRATIONS_DIR"/*.sql; do + MIGRATION_ID=$(basename "$migration") + + # Skip if already applied + if sqlite3 "$DB_FILE" -batch -bail "SELECT 1 FROM migrations WHERE id = '$MIGRATION_ID' LIMIT 1;" | grep -q 1; then + continue + fi + + echo "Applying migration: $MIGRATION_ID" + sqlite3 "$DB_FILE" -batch -bail < Date: Sun, 12 Oct 2025 20:44:28 -0500 Subject: [PATCH 10/17] feat(update): Insert stargazers count snapshots on update --- cli/update.go | 108 +++++++----------- internal/store/repository.go | 43 ++++++- internal/store/repository_stargazers_count.go | 36 ++++++ 3 files changed, 121 insertions(+), 66 deletions(-) create mode 100644 internal/store/repository_stargazers_count.go diff --git a/cli/update.go b/cli/update.go index 9623b46..6601f56 100644 --- a/cli/update.go +++ b/cli/update.go @@ -1,86 +1,64 @@ package cli import ( - "fmt" + "context" "log" - "time" - // "github.com/vimcolorschemes/worker/internal/database" + "github.com/vimcolorschemes/worker/internal/database" "github.com/vimcolorschemes/worker/internal/github" - repoHelper "github.com/vimcolorschemes/worker/internal/repository" - - "go.mongodb.org/mongo-driver/bson" + "github.com/vimcolorschemes/worker/internal/store" ) +var repositoryStargazersCountStore *store.RepositoryStargarzersCountStore + +func init() { + repositoryStargazersCountStore = store.NewRepositoryStargazersCountStore(database.Connect()) +} + // Update the imported repositories with all kinds of useful information func Update(_force bool, _debug bool, repoKey string) int { - var repositories []repoHelper.Repository + var repositories []store.Repository if repoKey != "" { - // repository, err := database.GetRepository(repoKey) - // if err != nil { - // log.Panic(err) - // } - // repositories = []repoHelper.Repository{repository} + repository, err := repositoryStore.GetByKey(context.TODO(), repoKey) + if err != nil { + log.Panic(err) + } + repositories = []store.Repository{*repository} } else { - // repositories = database.GetRepositories() + repositories = repositoryStore.GetAll() } log.Print(len(repositories), " repositories to update") for index, repository := range repositories { - fmt.Println() - - log.Print("Updating ", index, " of ", len(repositories), ": ", repository.Owner.Name, "/", repository.Name) - - // updatedRepository := updateRepository(repository) - - // updateObject := getUpdateRepositoryObject(updatedRepository) - - // database.UpsertRepository(repository.ID, updateObject) + log.Print("Updating ", index, " of ", len(repositories), ": ", repository.Owner, "/", repository.Name) + + githubRepository, err := github.GetRepository(repository.Owner, repository.Name) + if err != nil { + log.Print("Error fetching ", repository.Owner, "/", repository.Name) + continue + } + + err = repositoryStore.Upsert(context.TODO(), store.Repository{ + ID: *githubRepository.ID, + Name: *githubRepository.Name, + Owner: *githubRepository.Owner.Login, + Description: *githubRepository.Description, + CreatedAt: *githubRepository.CreatedAt.GetTime(), + UpdatedAt: *githubRepository.PushedAt.GetTime(), + }) + if err != nil { + log.Println("Error upserting repository:", err) + } + + err = repositoryStargazersCountStore.Insert(context.TODO(), store.RepositoryStargazersCount{ + RepositoryID: *githubRepository.ID, + StargazersCount: *githubRepository.StargazersCount, + }) + if err != nil { + log.Println("Error inserting stargazers count:", err) + } } return len(repositories) } - -func updateRepository(repository repoHelper.Repository) repoHelper.Repository { - githubRepository, err := github.GetRepository(repository.Owner.Name, repository.Name) - if err != nil { - log.Print("Error fetching ", repository.Owner.Name, "/", repository.Name) - repository.UpdateValid = false - return repository - } - - if githubRepository.PushedAt == nil { - log.Print("No commits on ", repository.Owner.Name, "/", repository.Name) - repository.UpdateValid = false - return repository - } - - repository.PushedAt = githubRepository.PushedAt.Time - - log.Print("Gathering basic infos") - repository.StargazersCount = *githubRepository.StargazersCount - - log.Print("Building stargazers count history") - repository.StargazersCountHistory = repository.AppendToStargazersCountHistory() - - log.Print("Computing week stargazers count") - repository.WeekStargazersCount = repository.ComputeTrendingStargazersCount(7) - - log.Print("Checking if ", repository.Owner.Name, "/", repository.Name, " is valid") - repository.UpdateValid = repository.IsValidAfterUpdate() - log.Printf("Update valid: %v", repository.UpdateValid) - - return repository -} - -func getUpdateRepositoryObject(repository repoHelper.Repository) bson.M { - return bson.M{ - "pushedAt": repository.PushedAt, - "stargazersCount": repository.StargazersCount, - "stargazersCountHistory": repository.StargazersCountHistory, - "weekStargazersCount": repository.WeekStargazersCount, - "updateValid": repository.UpdateValid, - "updatedAt": time.Now(), - } -} diff --git a/internal/store/repository.go b/internal/store/repository.go index 4e5a82b..3057241 100644 --- a/internal/store/repository.go +++ b/internal/store/repository.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "errors" + "strings" "time" ) @@ -24,7 +25,14 @@ func NewRepositoryStore(database *sql.DB) *RepositoryStore { return &RepositoryStore{database: database} } -func (store *RepositoryStore) GetByKey(ctx context.Context, owner, name string) (*Repository, error) { +func (store *RepositoryStore) GetByKey(ctx context.Context, key string) (*Repository, error) { + parts := strings.SplitN(key, "/", 2) + if len(parts) != 2 { + return nil, errors.New("invalid repository key") + } + owner := parts[0] + name := parts[1] + var r Repository err := store.database.QueryRowContext(ctx, ` SELECT id, owner, name, description, created_at, updated_at @@ -68,3 +76,36 @@ func (store *RepositoryStore) Upsert(ctx context.Context, r Repository) error { ) return err } + +func (store *RepositoryStore) GetAll() []Repository { + rows, err := store.database.Query(` + SELECT id, owner, name, description, created_at, updated_at + FROM repositories + `) + if err != nil { + panic(err) + } + defer rows.Close() + + var repositories []Repository + for rows.Next() { + var r Repository + if err := rows.Scan( + &r.ID, + &r.Owner, + &r.Name, + &r.Description, + &r.CreatedAt, + &r.UpdatedAt, + ); err != nil { + panic(err) + } + repositories = append(repositories, r) + } + + if err := rows.Err(); err != nil { + panic(err) + } + + return repositories +} diff --git a/internal/store/repository_stargazers_count.go b/internal/store/repository_stargazers_count.go new file mode 100644 index 0000000..89cd413 --- /dev/null +++ b/internal/store/repository_stargazers_count.go @@ -0,0 +1,36 @@ +package store + +import ( + "context" + "database/sql" + "time" +) + +type RepositoryStargazersCount struct { + RepositoryID int64 `db:"repository_id"` + SnapshotDate time.Time `db:"snapshot_date"` + StargazersCount int `db:"stargazers_count"` +} + +type RepositoryStargarzersCountStore struct { + database *sql.DB +} + +func NewRepositoryStargazersCountStore(database *sql.DB) *RepositoryStargarzersCountStore { + return &RepositoryStargarzersCountStore{database: database} +} + +func (store *RepositoryStargarzersCountStore) Insert(ctx context.Context, c RepositoryStargazersCount) error { + today := time.Now().UTC().Truncate(24 * time.Hour) + + _, err := store.database.ExecContext(ctx, ` + INSERT INTO repository_stargazers_count_snapshots (repository_id, snapshot_date, stargazers_count) + VALUES (?, ?, ?) + ON CONFLICT(repository_id, snapshot_date) DO UPDATE SET stargazers_count = excluded.stargazers_count + `, + c.RepositoryID, + today, + c.StargazersCount, + ) + return err +} From 864d6b915d5b7ad1abfa282c330269c429c5b213 Mon Sep 17 00:00:00 2001 From: Robin Gagnon Date: Sun, 12 Oct 2025 20:59:38 -0500 Subject: [PATCH 11/17] refactor(update): Use sqlite date function for stargazers count snapshot --- cli/update.go | 6 ++-- internal/store/repository_stargazers_count.go | 36 ------------------- .../repository_stargazers_count_snapshot.go | 31 ++++++++++++++++ 3 files changed, 34 insertions(+), 39 deletions(-) delete mode 100644 internal/store/repository_stargazers_count.go create mode 100644 internal/store/repository_stargazers_count_snapshot.go diff --git a/cli/update.go b/cli/update.go index 6601f56..b2c4ed3 100644 --- a/cli/update.go +++ b/cli/update.go @@ -9,10 +9,10 @@ import ( "github.com/vimcolorschemes/worker/internal/store" ) -var repositoryStargazersCountStore *store.RepositoryStargarzersCountStore +var repositoryStargazersCountSnapshotStore *store.RepositoryStargarzersCountSnapshotStore func init() { - repositoryStargazersCountStore = store.NewRepositoryStargazersCountStore(database.Connect()) + repositoryStargazersCountSnapshotStore = store.NewRepositoryStargazersCountSnapshotStore(database.Connect()) } // Update the imported repositories with all kinds of useful information @@ -51,7 +51,7 @@ func Update(_force bool, _debug bool, repoKey string) int { log.Println("Error upserting repository:", err) } - err = repositoryStargazersCountStore.Insert(context.TODO(), store.RepositoryStargazersCount{ + err = repositoryStargazersCountSnapshotStore.Insert(context.TODO(), store.RepositoryStargazersCountSnapshot{ RepositoryID: *githubRepository.ID, StargazersCount: *githubRepository.StargazersCount, }) diff --git a/internal/store/repository_stargazers_count.go b/internal/store/repository_stargazers_count.go deleted file mode 100644 index 89cd413..0000000 --- a/internal/store/repository_stargazers_count.go +++ /dev/null @@ -1,36 +0,0 @@ -package store - -import ( - "context" - "database/sql" - "time" -) - -type RepositoryStargazersCount struct { - RepositoryID int64 `db:"repository_id"` - SnapshotDate time.Time `db:"snapshot_date"` - StargazersCount int `db:"stargazers_count"` -} - -type RepositoryStargarzersCountStore struct { - database *sql.DB -} - -func NewRepositoryStargazersCountStore(database *sql.DB) *RepositoryStargarzersCountStore { - return &RepositoryStargarzersCountStore{database: database} -} - -func (store *RepositoryStargarzersCountStore) Insert(ctx context.Context, c RepositoryStargazersCount) error { - today := time.Now().UTC().Truncate(24 * time.Hour) - - _, err := store.database.ExecContext(ctx, ` - INSERT INTO repository_stargazers_count_snapshots (repository_id, snapshot_date, stargazers_count) - VALUES (?, ?, ?) - ON CONFLICT(repository_id, snapshot_date) DO UPDATE SET stargazers_count = excluded.stargazers_count - `, - c.RepositoryID, - today, - c.StargazersCount, - ) - return err -} diff --git a/internal/store/repository_stargazers_count_snapshot.go b/internal/store/repository_stargazers_count_snapshot.go new file mode 100644 index 0000000..4ee30ec --- /dev/null +++ b/internal/store/repository_stargazers_count_snapshot.go @@ -0,0 +1,31 @@ +package store + +import ( + "context" + "database/sql" +) + +type RepositoryStargazersCountSnapshot struct { + RepositoryID int64 `db:"repository_id"` + StargazersCount int `db:"stargazers_count"` +} + +type RepositoryStargarzersCountSnapshotStore struct { + database *sql.DB +} + +func NewRepositoryStargazersCountSnapshotStore(database *sql.DB) *RepositoryStargarzersCountSnapshotStore { + return &RepositoryStargarzersCountSnapshotStore{database: database} +} + +func (store *RepositoryStargarzersCountSnapshotStore) Insert(ctx context.Context, c RepositoryStargazersCountSnapshot) error { + _, err := store.database.ExecContext(ctx, ` + INSERT INTO repository_stargazers_count_snapshots (repository_id, snapshot_date, stargazers_count) + VALUES (?, date('now'), ?) + ON CONFLICT(repository_id, snapshot_date) DO UPDATE SET stargazers_count = excluded.stargazers_count + `, + c.RepositoryID, + c.StargazersCount, + ) + return err +} From 0adeda0ae56ca6e32383677fac4a4098b0c14b10 Mon Sep 17 00:00:00 2001 From: Robin Gagnon Date: Sun, 12 Oct 2025 21:00:07 -0500 Subject: [PATCH 12/17] refactor(db): Use JSON format and add missing FK to colorscheme data --- db/migrations/20251005195216_colorschemes.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db/migrations/20251005195216_colorschemes.sql b/db/migrations/20251005195216_colorschemes.sql index aa08a61..1809dcc 100644 --- a/db/migrations/20251005195216_colorschemes.sql +++ b/db/migrations/20251005195216_colorschemes.sql @@ -7,9 +7,9 @@ CREATE TABLE colorschemes ( CREATE TABLE colorscheme_color_data ( colorscheme_name TEXT NOT NULL, - repository_id INTEGER NOT NULL, + repository_id INTEGER NOT NULL REFERENCES repositories(id) ON DELETE CASCADE, background TEXT NOT NULL CHECK (background IN ('light', 'dark')), - color_data TEXT NOT NULL, + color_data JSON NOT NULL, PRIMARY KEY (colorscheme_name, repository_id, background) ) From 307ea611d6f96b6ac2e95f9d607aa7baa93c0203 Mon Sep 17 00:00:00 2001 From: Robin Gagnon Date: Mon, 13 Oct 2025 19:39:43 -0500 Subject: [PATCH 13/17] feat: Add Github URL property on repositories --- cli/import.go | 1 + cli/update.go | 1 + db/migrations/20251005184928_repositories.sql | 1 + internal/store/repository.go | 15 ++++++++++----- 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/cli/import.go b/cli/import.go index 7acd289..c7550da 100644 --- a/cli/import.go +++ b/cli/import.go @@ -72,6 +72,7 @@ func Import(_force bool, _debug bool, repoKey string) int { Description: *repository.Description, CreatedAt: *repository.CreatedAt.GetTime(), UpdatedAt: *repository.PushedAt.GetTime(), + GithubURL: *repository.HTMLURL, }) if err != nil { log.Println("Error upserting repository:", err) diff --git a/cli/update.go b/cli/update.go index b2c4ed3..daeb5f5 100644 --- a/cli/update.go +++ b/cli/update.go @@ -46,6 +46,7 @@ func Update(_force bool, _debug bool, repoKey string) int { Description: *githubRepository.Description, CreatedAt: *githubRepository.CreatedAt.GetTime(), UpdatedAt: *githubRepository.PushedAt.GetTime(), + GithubURL: *githubRepository.HTMLURL, }) if err != nil { log.Println("Error upserting repository:", err) diff --git a/db/migrations/20251005184928_repositories.sql b/db/migrations/20251005184928_repositories.sql index 200eabb..4f073f8 100644 --- a/db/migrations/20251005184928_repositories.sql +++ b/db/migrations/20251005184928_repositories.sql @@ -5,6 +5,7 @@ CREATE TABLE repositories ( description TEXT, created_at TIMESTAMP NOT NULL, updated_at TIMESTAMP NOT NULL, + github_url TEXT NOT NULL UNIQUE CHECK (github_url LIKE 'https?://github.com/%/%'), UNIQUE(owner, name) ); diff --git a/internal/store/repository.go b/internal/store/repository.go index 3057241..8ddc4f1 100644 --- a/internal/store/repository.go +++ b/internal/store/repository.go @@ -15,6 +15,7 @@ type Repository struct { Description string `db:"description"` CreatedAt time.Time `db:"created_at"` UpdatedAt time.Time `db:"updated_at"` + GithubURL string `db:"github_url"` } type RepositoryStore struct { @@ -35,7 +36,7 @@ func (store *RepositoryStore) GetByKey(ctx context.Context, key string) (*Reposi var r Repository err := store.database.QueryRowContext(ctx, ` - SELECT id, owner, name, description, created_at, updated_at + SELECT id, owner, name, description, created_at, updated_at, github_url FROM repositories WHERE owner = ? AND name = ? `, owner, name).Scan( @@ -45,6 +46,7 @@ func (store *RepositoryStore) GetByKey(ctx context.Context, key string) (*Reposi &r.Description, &r.CreatedAt, &r.UpdatedAt, + &r.GithubURL, ) if err != nil { @@ -59,13 +61,14 @@ func (store *RepositoryStore) GetByKey(ctx context.Context, key string) (*Reposi func (store *RepositoryStore) Upsert(ctx context.Context, r Repository) error { _, err := store.database.ExecContext(ctx, ` - INSERT INTO repositories (id, owner, name, description, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?) + INSERT INTO repositories (id, owner, name, description, created_at, updated_at, github_url) + VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT(id) DO UPDATE SET owner = excluded.owner, name = excluded.name, description = excluded.description, - updated_at = excluded.updated_at; + updated_at = excluded.updated_at, + github_url = excluded.github_url; `, r.ID, r.Owner, @@ -73,13 +76,14 @@ func (store *RepositoryStore) Upsert(ctx context.Context, r Repository) error { r.Description, r.CreatedAt, r.UpdatedAt, + r.GithubURL, ) return err } func (store *RepositoryStore) GetAll() []Repository { rows, err := store.database.Query(` - SELECT id, owner, name, description, created_at, updated_at + SELECT id, owner, name, description, created_at, updated_at, github_url FROM repositories `) if err != nil { @@ -97,6 +101,7 @@ func (store *RepositoryStore) GetAll() []Repository { &r.Description, &r.CreatedAt, &r.UpdatedAt, + &r.GithubURL, ); err != nil { panic(err) } From b0eafcf0a76bba31c9e40d88cb5efc46ca9746b2 Mon Sep 17 00:00:00 2001 From: Robin Gagnon Date: Mon, 13 Oct 2025 19:40:38 -0500 Subject: [PATCH 14/17] refactor: Use single table for colorscheme variants --- ...> 20251005195216_colorscheme_variants.sql} | 10 +--- internal/store/colorscheme_variant.go | 53 +++++++++++++++++++ 2 files changed, 55 insertions(+), 8 deletions(-) rename db/migrations/{20251005195216_colorschemes.sql => 20251005195216_colorscheme_variants.sql} (56%) create mode 100644 internal/store/colorscheme_variant.go diff --git a/db/migrations/20251005195216_colorschemes.sql b/db/migrations/20251005195216_colorscheme_variants.sql similarity index 56% rename from db/migrations/20251005195216_colorschemes.sql rename to db/migrations/20251005195216_colorscheme_variants.sql index 1809dcc..2adafc4 100644 --- a/db/migrations/20251005195216_colorschemes.sql +++ b/db/migrations/20251005195216_colorscheme_variants.sql @@ -1,15 +1,9 @@ -CREATE TABLE colorschemes ( - name TEXT NOT NULL, - repository_id INTEGER NOT NULL REFERENCES repositories(id) ON DELETE CASCADE, - - PRIMARY KEY (name, repository_id) -); - -CREATE TABLE colorscheme_color_data ( +CREATE TABLE colorscheme_variants ( colorscheme_name TEXT NOT NULL, repository_id INTEGER NOT NULL REFERENCES repositories(id) ON DELETE CASCADE, background TEXT NOT NULL CHECK (background IN ('light', 'dark')), color_data JSON NOT NULL, PRIMARY KEY (colorscheme_name, repository_id, background) + FOREIGN KEY (colorscheme_name, repository_id) REFERENCES colorschemes(name, repository_id) ON DELETE CASCADE ) diff --git a/internal/store/colorscheme_variant.go b/internal/store/colorscheme_variant.go new file mode 100644 index 0000000..50c90b6 --- /dev/null +++ b/internal/store/colorscheme_variant.go @@ -0,0 +1,53 @@ +package store + +import ( + "context" + "database/sql" + "encoding/json" + "log" +) + +type Background string + +const ( + BackgroundLight Background = "light" + BackgroundDark Background = "dark" +) + +type ColorschemeVariant struct { + Name string `db:"colorscheme_name"` + RepositoryID int64 `db:"repository_id"` + Background Background `db:"background"` + ColorData []ColorschemeGroupDefinition `db:"color_data"` +} + +type ColorschemeGroupDefinition struct { + Name string + HexCode string +} + +type ColorschemeVariantStore struct { + database *sql.DB +} + +func NewColorschemeVariantStore(database *sql.DB) *ColorschemeVariantStore { + return &ColorschemeVariantStore{database: database} +} + +func (store *ColorschemeVariantStore) UpsertColorschemeVariant(ctx context.Context, colorschemeVariant *ColorschemeVariant) error { + colorData, err := json.Marshal(colorschemeVariant.ColorData) + if err != nil { + return err + } + + log.Printf("Upserting colorscheme variant %s: %v", colorschemeVariant.Name, colorData) + + _, err = store.database.ExecContext(ctx, ` + INSERT INTO colorscheme_variants (colorscheme_name, repository_id, background, color_data) + VALUES (?, ?, ?, ?) + ON CONFLICT(colorscheme_name, repository_id, background) + DO UPDATE SET color_data = excluded.color_data + `, colorschemeVariant.Name, colorschemeVariant.RepositoryID, colorschemeVariant.Background, string(colorData)) + + return err +} From 06cce85f2768ca4e07190cbc1e29ebbd8a29ba81 Mon Sep 17 00:00:00 2001 From: Robin Gagnon Date: Mon, 13 Oct 2025 20:29:54 -0500 Subject: [PATCH 15/17] fix: Remove URL constraint on repository github URL --- db/migrations/20251005184928_repositories.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/migrations/20251005184928_repositories.sql b/db/migrations/20251005184928_repositories.sql index 4f073f8..aff4d3f 100644 --- a/db/migrations/20251005184928_repositories.sql +++ b/db/migrations/20251005184928_repositories.sql @@ -5,7 +5,7 @@ CREATE TABLE repositories ( description TEXT, created_at TIMESTAMP NOT NULL, updated_at TIMESTAMP NOT NULL, - github_url TEXT NOT NULL UNIQUE CHECK (github_url LIKE 'https?://github.com/%/%'), + github_url TEXT NOT NULL UNIQUE, UNIQUE(owner, name) ); From 75e78dd63a105c6524e1b0ce2b6fb80dc0e7529f Mon Sep 17 00:00:00 2001 From: Robin Gagnon Date: Mon, 13 Oct 2025 20:30:12 -0500 Subject: [PATCH 16/17] feat(generate): Store colorscheme variants from data --- cli/generate.go | 118 ++++++++++++++++++++++++------------------------ 1 file changed, 60 insertions(+), 58 deletions(-) diff --git a/cli/generate.go b/cli/generate.go index 4a7c09b..579604b 100644 --- a/cli/generate.go +++ b/cli/generate.go @@ -1,20 +1,35 @@ package cli import ( + "context" "encoding/json" "fmt" "log" "os" "os/exec" "strings" - "time" + "github.com/vimcolorschemes/worker/internal/database" file "github.com/vimcolorschemes/worker/internal/file" - repoHelper "github.com/vimcolorschemes/worker/internal/repository" - - "go.mongodb.org/mongo-driver/bson" + "github.com/vimcolorschemes/worker/internal/store" ) +type ColorschemeData struct { + Light []store.ColorschemeGroupDefinition + Dark []store.ColorschemeGroupDefinition +} + +type ColorschemeGroupDefinition struct { + Name string + HexCode string +} + +var colorschemeVariantStore *store.ColorschemeVariantStore + +func init() { + colorschemeVariantStore = store.NewColorschemeVariantStore(database.Connect()) +} + var tmpDirectoryPath string var packDirectoryPath string var vimrcPath string @@ -32,72 +47,72 @@ func Generate(force bool, debug bool, repoKey string) int { fmt.Println() - var repositories []repoHelper.Repository + var repositories []store.Repository if repoKey != "" { - // repository, err := database.GetRepository(repoKey) - // if err != nil { - // log.Panic(err) - // } - // repositories = []repoHelper.Repository{repository} - } else if force || debug { - // repositories = database.GetRepositories() + repository, err := repositoryStore.GetByKey(context.TODO(), repoKey) + if err != nil { + log.Panic(err) + } + repositories = []store.Repository{*repository} } else { - // repositories = database.GetRepositoriesToGenerate() + repositories = repositoryStore.GetAll() } log.Printf("Generating vim preview for %d repositories", len(repositories)) for index, repository := range repositories { - log.Print("\nGenerating vim previews for ", repository.Owner.Name, "/", repository.Name, " (", index+1, "/", len(repositories), ")") + log.Print("\nGenerating vim previews for ", repository.Owner, "/", repository.Name, " (", index+1, "/", len(repositories), ")") - key := fmt.Sprintf("%s__%s", repository.Owner.Name, repository.Name) + key := fmt.Sprintf("%s__%s", repository.Owner, repository.Name) err := installPlugin(repository.GithubURL, key) if err != nil { log.Printf("Error installing plugin: %s", err) - repository.GenerateValid = false - updateRepositoryAfterGenerate(repository) + // repository.GenerateValid = false + // updateRepositoryAfterGenerate(repository) continue } - var data, dataError = getVimColorSchemeColorData() + var dataMap, dataError = getColorschemeDataMap() err = deletePlugin(key) if err != nil { log.Printf("Error deleting plugin: %s", err) } if dataError != nil { log.Printf("Error getting color data: %s", dataError) - repository.GenerateValid = false - updateRepositoryAfterGenerate(repository) + // repository.GenerateValid = false + // updateRepositoryAfterGenerate(repository) continue } - var vimColorSchemes []repoHelper.VimColorScheme - - for name := range data { + for name := range dataMap { if name == "default" || name == "module-injection" || name == "tick_tock" { continue } - var backgrounds []repoHelper.VimBackgroundValue - if data[name].Light != nil { - backgrounds = append(backgrounds, repoHelper.LightBackground) - } - if data[name].Dark != nil { - backgrounds = append(backgrounds, repoHelper.DarkBackground) + if dataMap[name].Light != nil { + err = colorschemeVariantStore.UpsertColorschemeVariant(context.TODO(), &store.ColorschemeVariant{ + Name: name, + RepositoryID: repository.ID, + Background: store.BackgroundLight, + ColorData: dataMap[name].Light, + }) + if err != nil { + log.Printf("Error upserting colorscheme variant: %s", err) + } } - vimColorSchemes = append( - vimColorSchemes, - repoHelper.VimColorScheme{ - Name: name, - Data: data[name], - Backgrounds: backgrounds, + if dataMap[name].Dark != nil { + err = colorschemeVariantStore.UpsertColorschemeVariant(context.TODO(), &store.ColorschemeVariant{ + Name: name, + RepositoryID: repository.ID, + Background: store.BackgroundDark, + ColorData: dataMap[name].Dark, }) + if err != nil { + log.Printf("Error upserting colorscheme variant: %s", err) + } + } } - - repository.VimColorSchemes = vimColorSchemes - repository.GenerateValid = len(repository.VimColorSchemes) > 0 - updateRepositoryAfterGenerate(repository) } cleanUp() @@ -105,12 +120,6 @@ func Generate(force bool, debug bool, repoKey string) int { return len(repositories) } -func updateRepositoryAfterGenerate(repository repoHelper.Repository) { - log.Printf("Generate valid: %v", repository.GenerateValid) - // generateObject := getGenerateRepositoryObject(repository) - // database.UpsertRepository(repository.ID, generateObject) -} - // Initializes a temporary directory for vim configuration files func initVimFiles() { workingDirectory, err := os.Getwd() @@ -258,22 +267,23 @@ func deletePlugin(key string) error { return err } -// Gathers the colorscheme data from vimcolorschemes/extractor.nvim -func getVimColorSchemeColorData() (map[string]repoHelper.VimColorSchemeData, error) { +// Gathers the colorscheme data from vimcolorschemes/extractor.nvim mapped by +// colorscheme name +func getColorschemeDataMap() (map[string]ColorschemeData, error) { err := executePreviewGenerator() if err != nil { log.Printf("Error executing nvim: %s", err) return nil, err } - vimColorSchemeOutput, err := file.GetLocalFileContent(colorDataFilePath) + colorschemeOutput, err := file.GetLocalFileContent(colorDataFilePath) if err != nil { log.Printf("Error getting local file content from \"%s\": %s", colorDataFilePath, err) return nil, err } - var data map[string]repoHelper.VimColorSchemeData - err = json.Unmarshal([]byte(vimColorSchemeOutput), &data) + var data map[string]ColorschemeData + err = json.Unmarshal([]byte(colorschemeOutput), &data) if err != nil { return nil, err } @@ -324,11 +334,3 @@ func cleanUp() { log.Panic(err) } } - -func getGenerateRepositoryObject(repository repoHelper.Repository) bson.M { - return bson.M{ - "vimColorSchemes": repository.VimColorSchemes, - "generateValid": repository.GenerateValid, - "generatedAt": time.Now(), - } -} From 0a3af8e6dbaa682966e1d98ddaab3d0a753ff813 Mon Sep 17 00:00:00 2001 From: Robin Gagnon Date: Sat, 25 Oct 2025 22:27:59 -0500 Subject: [PATCH 17/17] feat: Insert job reports --- cli/generate.go | 5 +- cli/import.go | 5 +- cli/update.go | 5 +- cmd/worker/main.go | 19 +++++++- db/migrations/20251005200040_job_reports.sql | 5 +- internal/database/database.go | 2 + internal/store/colorscheme_variant.go | 3 -- internal/store/job_report.go | 51 ++++++++++++++++++++ 8 files changed, 82 insertions(+), 13 deletions(-) create mode 100644 internal/store/job_report.go diff --git a/cli/generate.go b/cli/generate.go index 579604b..6f375fd 100644 --- a/cli/generate.go +++ b/cli/generate.go @@ -38,7 +38,7 @@ var colorDataFilePath string var debugMode bool // Generate vim color scheme data for all valid repositories -func Generate(force bool, debug bool, repoKey string) int { +func Generate(force bool, debug bool, repoKey string) database.JSONB { debugMode = debug initVimFiles() @@ -117,7 +117,8 @@ func Generate(force bool, debug bool, repoKey string) int { cleanUp() - return len(repositories) + log.Printf("Generated %d repositories", len(repositories)) + return database.JSONB{"count": len(repositories)} } // Initializes a temporary directory for vim configuration files diff --git a/cli/import.go b/cli/import.go index c7550da..e39d307 100644 --- a/cli/import.go +++ b/cli/import.go @@ -44,7 +44,7 @@ func init() { } // Import potential vim color scheme repositories from Github -func Import(_force bool, _debug bool, repoKey string) int { +func Import(_force bool, _debug bool, repoKey string) database.JSONB { log.Printf("Repository limit: %d", repositoryCountLimit) var repositories []*gogithub.Repository @@ -79,5 +79,6 @@ func Import(_force bool, _debug bool, repoKey string) int { } } - return len(repositories) + log.Printf("Imported %d repositories", len(repositories)) + return database.JSONB{"count": len(repositories)} } diff --git a/cli/update.go b/cli/update.go index daeb5f5..7fd6096 100644 --- a/cli/update.go +++ b/cli/update.go @@ -16,7 +16,7 @@ func init() { } // Update the imported repositories with all kinds of useful information -func Update(_force bool, _debug bool, repoKey string) int { +func Update(_force bool, _debug bool, repoKey string) database.JSONB { var repositories []store.Repository if repoKey != "" { repository, err := repositoryStore.GetByKey(context.TODO(), repoKey) @@ -61,5 +61,6 @@ func Update(_force bool, _debug bool, repoKey string) int { } } - return len(repositories) + log.Printf("Updated %d repositories", len(repositories)) + return database.JSONB{"count": len(repositories)} } diff --git a/cmd/worker/main.go b/cmd/worker/main.go index c9f821a..ec2ae3b 100644 --- a/cmd/worker/main.go +++ b/cmd/worker/main.go @@ -1,6 +1,7 @@ package main import ( + "context" "errors" "fmt" "log" @@ -9,15 +10,20 @@ import ( "time" "github.com/vimcolorschemes/worker/cli" + "github.com/vimcolorschemes/worker/internal/database" + "github.com/vimcolorschemes/worker/internal/store" ) -var jobRunnerMap = map[string]func(force bool, debug bool, repoKey string) int{ +var jobRunnerMap = map[string]func(force bool, debug bool, repoKey string) database.JSONB{ "import": cli.Import, "update": cli.Update, "generate": cli.Generate, } func main() { + database := database.Connect() + jobReportStore := *store.NewJobReportStore(database) + job, force, debug, repoKey, err := getJobArgs(os.Args) log.Printf("Running %s", job) @@ -46,10 +52,19 @@ func main() { fmt.Println() data := runner(force, debug, repoKey) - log.Printf("Processed %d items", data) elapsedTime := time.Since(startTime) + err = jobReportStore.Create(context.TODO(), store.JobReport{ + Job: store.Job(job), + ReportData: data, + ElapsedTimeInSeconds: int64(elapsedTime.Seconds()), + CreatedAt: time.Now(), + }) + if err != nil { + log.Panic(err) + } + fmt.Println() log.Printf("Elapsed time: %s\n", elapsedTime) log.Print(":wq") diff --git a/db/migrations/20251005200040_job_reports.sql b/db/migrations/20251005200040_job_reports.sql index 0f6c006..ffacfa7 100644 --- a/db/migrations/20251005200040_job_reports.sql +++ b/db/migrations/20251005200040_job_reports.sql @@ -1,7 +1,8 @@ CREATE TABLE job_reports ( - id SERIAL PRIMARY KEY, job TEXT NOT NULL CHECK (job IN ('import', 'update', 'generate')), report_data JSONB NOT NULL, elapsed_time_in_seconds INTEGER NOT NULL, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + + PRIMARY KEY (job, created_at) ); diff --git a/internal/database/database.go b/internal/database/database.go index f2f54fc..4056574 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -10,6 +10,8 @@ import ( _ "github.com/tursodatabase/libsql-client-go/libsql" ) +type JSONB map[string]any + func Connect() *sql.DB { url := os.Getenv("TURSO_DATABASE_URL") token := os.Getenv("TURSO_AUTH_TOKEN") diff --git a/internal/store/colorscheme_variant.go b/internal/store/colorscheme_variant.go index 50c90b6..daf87ff 100644 --- a/internal/store/colorscheme_variant.go +++ b/internal/store/colorscheme_variant.go @@ -4,7 +4,6 @@ import ( "context" "database/sql" "encoding/json" - "log" ) type Background string @@ -40,8 +39,6 @@ func (store *ColorschemeVariantStore) UpsertColorschemeVariant(ctx context.Conte return err } - log.Printf("Upserting colorscheme variant %s: %v", colorschemeVariant.Name, colorData) - _, err = store.database.ExecContext(ctx, ` INSERT INTO colorscheme_variants (colorscheme_name, repository_id, background, color_data) VALUES (?, ?, ?, ?) diff --git a/internal/store/job_report.go b/internal/store/job_report.go new file mode 100644 index 0000000..d9964ec --- /dev/null +++ b/internal/store/job_report.go @@ -0,0 +1,51 @@ +package store + +import ( + "context" + "database/sql" + "encoding/json" + "time" + + "github.com/vimcolorschemes/worker/internal/database" +) + +type Job string + +const ( + JobImport Job = "import" + JobUpdate Job = "update" + JobGenerate Job = "generate" +) + +type JobReport struct { + Job Job `db:"job"` + ReportData database.JSONB `db:"report_data"` + ElapsedTimeInSeconds int64 `db:"elapsed_time_in_seconds"` + CreatedAt time.Time `db:"created_at"` +} + +type JobReportStore struct { + database *sql.DB +} + +func NewJobReportStore(database *sql.DB) *JobReportStore { + return &JobReportStore{database: database} +} + +func (store *JobReportStore) Create(ctx context.Context, jobReport JobReport) error { + reportData, err := json.Marshal(jobReport.ReportData) + if err != nil { + return err + } + + _, err = store.database.ExecContext(ctx, ` + INSERT INTO job_reports (job, report_data, elapsed_time_in_seconds, created_at) + VALUES (?, ?, ?, ?) + `, + jobReport.Job, + reportData, + jobReport.ElapsedTimeInSeconds, + jobReport.CreatedAt, + ) + return err +}