diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..6489772 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,122 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version: '1.24' + + - name: Download dependencies + run: go mod download + + - name: Run tests + # Note: -race disabled due to pre-existing race conditions in tests that spawn + # background goroutines. These should be fixed by adding proper synchronization. + run: go test -v ./... + + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version: '1.24' + + - name: golangci-lint + uses: golangci/golangci-lint-action@v6 + with: + version: latest + + security: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version: '1.24' + + - name: Run gosec + uses: securego/gosec@master + with: + # Exclude G101 (hardcoded credentials - false positive on env var names) + # Exclude G115 (integer overflow - false positive for PR numbers) + # Exclude G304 (file inclusion - intentional for CLI tools) + # Exclude G306 (file permissions - config files don't need 0600) + args: -exclude=G101,G115,G304,G306 ./... + + build: + runs-on: ubuntu-latest + needs: [test, lint] + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version: '1.24' + + - name: Build + run: go build -v ./... + + deploy: + runs-on: ubuntu-latest + needs: [build, security] + # Only deploy on push to main (not on PRs) + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + + permissions: + contents: read + id-token: write # Required for Workload Identity Federation + + env: + PROJECT_ID: "github-copy-code-examples" + SERVICE_NAME: "examples-copier" + REGION: "us-central1" + + steps: + - uses: actions/checkout@v4 + + - name: Authenticate to Google Cloud + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }} + + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@v2 + + - name: Deploy to Cloud Run + run: | + gcloud run deploy $SERVICE_NAME \ + --source . \ + --region $REGION \ + --project $PROJECT_ID \ + --allow-unauthenticated \ + --env-vars-file=env-cloudrun.yaml \ + --max-instances=10 \ + --cpu=1 \ + --memory=512Mi \ + --timeout=300s \ + --concurrency=80 \ + --port=8080 \ + --platform=managed + + - name: Show deployment URL + run: | + URL=$(gcloud run services describe $SERVICE_NAME \ + --region $REGION \ + --project $PROJECT_ID \ + --format='value(status.url)') + echo "🚀 Deployed to: $URL" + diff --git a/.gitignore b/.gitignore index 99d319b..37a0615 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ # Binaries -examples-copier +github-copier code-copier copier *.exe @@ -23,19 +23,21 @@ go.work # Environment files with secrets (working files - create from templates in configs/) # Working files (create from templates): # env.yaml - App Engine deployment config (from configs/env.yaml.*) -# env-cloudrun.yaml - Cloud Run deployment config (from configs/env.yaml.*) # .env - Local development config (from configs/.env.local.example) +# .env.test - Test config (from testdata/.env.test) +# Note: env-cloudrun.yaml is committed (contains no secrets, only Secret Manager references) env.yaml -env-cloudrun.yaml .env .env.local +.env.test .env.production .env.*.local -# Explicitly keep template files in configs/ directory (these should be tracked) +# Explicitly keep template files (these should be tracked) !configs/env.yaml.example !configs/env.yaml.production !configs/.env.local.example +!testdata/.env.test # Private keys *.pem @@ -58,3 +60,4 @@ Thumbs.db # Temporary files tmp/ temp/ +RECOMMENDATIONS.md diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..d8c909a --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,36 @@ +repos: + # Secrets detection + - repo: https://github.com/gitleaks/gitleaks + rev: v8.21.2 + hooks: + - id: gitleaks + + # Go linting + - repo: https://github.com/golangci/golangci-lint + rev: v1.62.2 + hooks: + - id: golangci-lint + + # Local Go hooks + - repo: local + hooks: + - id: go-fmt + name: go fmt + entry: gofmt -w + language: system + types: [go] + + - id: go-vet + name: go vet + entry: go vet ./... + language: system + pass_filenames: false + types: [go] + + - id: go-build + name: go build + entry: go build ./... + language: system + pass_filenames: false + types: [go] + diff --git a/AGENT.md b/AGENT.md new file mode 100644 index 0000000..ab2277c --- /dev/null +++ b/AGENT.md @@ -0,0 +1,90 @@ +# Agent Context: GitHub Copier + +Webhook service: PR merged → match files → transform paths → copy to target repos. + +## File Map + +``` +app.go # entrypoint, HTTP server +services/ + webhook_handler_new.go # HandleWebhookWithContainer() + workflow_processor.go # ProcessWorkflow() - core logic + pattern_matcher.go # MatchFile(pattern, path) bool + github_auth.go # ConfigurePermissions() error + github_read.go # GetFilesChangedInPr(), RetrieveFileContents() + github_write_to_target.go # AddFilesToTargetRepoBranch() + github_write_to_source.go # UpdateDeprecationFile() + file_state_service.go # tracks upload/deprecate queues + main_config_loader.go # LoadConfig() with $ref support + service_container.go # DI container +types/ + config.go # Workflow, Transformation, SourcePattern structs + types.go # ChangedFile, UploadKey, UploadFileContent +configs/environment.go # Config struct, LoadEnvironment() +tests/utils.go # test helpers, httpmock setup +``` + +## Key Types + +```go +// types/config.go +type PatternType string // "prefix" | "glob" | "regex" +type TransformationType string // "move" | "copy" | "glob" | "regex" + +type Workflow struct { + Name string + Source SourceConfig // Repo, Branch, Patterns []SourcePattern + Destination DestinationConfig // Repo, Branch + Transformations []Transformation // Type, From, To, Pattern, Replacement + Commit CommitConfig // Strategy, Message, PRTitle, AutoMerge +} + +// types/types.go +type ChangedFile struct { Path, Status string } // Status: "ADDED"|"MODIFIED"|"DELETED" +type UploadKey struct { RepoName, BranchPath string } +``` + +## Global State (⚠️ mutable) + +```go +// services/github_write_to_target.go +var FilesToUpload map[UploadKey]UploadFileContent +// services/github_auth.go +var InstallationAccessToken string +var OrgTokens map[string]string +``` + +## Config Example + +```yaml +workflows: + - name: "sync-docs" + source: { repo: "org/src", branch: "main", patterns: [{type: glob, pattern: "docs/**"}] } + destination: { repo: "org/dest", branch: "main" } + transformations: [{ type: move, from: "docs/", to: "public/" }] + commit: { strategy: pr, message: "Sync" } # strategy: direct|pr +``` + +## Test Commands + +```bash +go test ./... # all +go test ./services/... -run TestWorkflow -v # specific +``` + +## Edit Patterns + +| Task | Files to modify | +|------|-----------------| +| New transformation | `types/config.go` (TransformationType) → `workflow_processor.go` (processFileForWorkflow) | +| New pattern type | `types/config.go` (PatternType) → `pattern_matcher.go` | +| New config field | `types/config.go` (struct) → consumers in `workflow_processor.go` | +| Webhook logic | `webhook_handler_new.go` | + +## Conventions + +- Return `error`, never `log.Fatal` +- Wrap errors: `fmt.Errorf("context: %w", err)` +- Nil-check GitHub API responses before dereference +- Tests use `httpmock`; see `tests/utils.go` +- **Changelog**: Update `CHANGELOG.md` for all notable changes (follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/)) diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..96daa5d --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,54 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +## 17 Dec 2025 + +### Added +- CI/CD pipeline with GitHub Actions (`.github/workflows/ci.yml`) + - Test job + - Lint job with golangci-lint + - Security scanning with gosec + - Build verification + - Automated deployment to Cloud Run on merge to main (via Workload Identity Federation) +- Pre-commit hooks for secrets detection and Go linting (`.pre-commit-config.yaml`) +- AGENT.md for AI agent context +- Comprehensive test suite for `workflow_processor.go` (843 lines, 94%+ coverage) +- Integration test harness for local testing (`scripts/integration-test.sh`) +- Test environment configuration (`testdata/.env.test`) + +### Changed +- Renamed module from `github.com/mongodb/code-example-tooling/code-copier` to `github.com/grove-platform/github-copier` +- Renamed binary from `examples-copier` to `github-copier` +- Renamed `test-payloads/` to `testdata/` (Go convention) +- All `log.Fatal` calls replaced with proper error returns for graceful error handling +- `FileStateService.filesToDeprecate` changed from single-entry map to slice-based accumulation + +### Fixed +- Deprecation file accumulation bug: multiple deprecated files now correctly accumulate instead of overwriting +- Nil pointer dereference bugs across GitHub API calls in: + - `services/github_read.go` + - `services/github_write_to_source.go` + - `services/main_config_loader.go` + - `services/config_loader.go` +- DELETED file status handling: GitHub GraphQL API returns uppercase `DELETED` but code checked for lowercase `removed` +- Graceful shutdown now properly waits for in-flight requests and cleans up resources + +### Security +- Added gitleaks pre-commit hook for secrets detection +- Added gosec security scanning in CI pipeline + +## Initial Release (Migration from mongodb/code-example-tooling) + +### Features +- Webhook service for automated file copying on PR merge +- Pattern matching support: prefix, glob, regex +- Transformation types: move, copy, glob, regex +- Main config system with `$ref` support for distributed workflow configs +- Commit strategies: direct commit or pull request +- Health and metrics endpoints +- Slack notifications for operational visibility +- MongoDB audit logging (optional) +- Google Cloud Logging integration +- Dry-run mode for testing + diff --git a/QUICK-REFERENCE.md b/QUICK-REFERENCE.md index 193ed77..28fe49a 100644 --- a/QUICK-REFERENCE.md +++ b/QUICK-REFERENCE.md @@ -6,19 +6,19 @@ ```bash # Run with default settings -./examples-copier +./github-copier # Run with custom environment -./examples-copier -env ./configs/.env.production +./github-copier -env ./configs/.env.production # Dry-run mode (no actual commits) -./examples-copier -dry-run +./github-copier -dry-run # Validate configuration only -./examples-copier -validate +./github-copier -validate # Show help -./examples-copier -help +./github-copier -help ``` ### CLI Validator @@ -317,10 +317,10 @@ go test ./services -cover go build -o test-webhook ./cmd/test-webhook # Send example payload -./test-webhook -payload test-payloads/example-pr-merged.json +./test-webhook -payload testdata/example-pr-merged.json # Dry-run (see payload without sending) -./test-webhook -payload test-payloads/example-pr-merged.json -dry-run +./test-webhook -payload testdata/example-pr-merged.json -dry-run ``` #### Option 2: Use Real PR Data @@ -349,7 +349,7 @@ chmod +x scripts/test-with-pr.sh ### Test in Dry-Run Mode ```bash # Start app in dry-run mode -DRY_RUN=true ./examples-copier & +DRY_RUN=true ./github-copier & # Send test webhook ./test-webhook -pr 123 -owner myorg -repo myrepo @@ -360,7 +360,7 @@ DRY_RUN=true ./examples-copier & ### Build ```bash # Main application -go build -o examples-copier . +go build -o github-copier . # CLI validator go build -o config-validator ./cmd/config-validator @@ -369,7 +369,7 @@ go build -o config-validator ./cmd/config-validator go build -o test-webhook ./cmd/test-webhook # All tools -go build -o examples-copier . && \ +go build -o github-copier . && \ go build -o config-validator ./cmd/config-validator && \ go build -o test-webhook ./cmd/test-webhook ``` @@ -442,7 +442,7 @@ workflows: gcloud app logs tail -s default # Local logs -LOG_LEVEL=debug ./examples-copier +LOG_LEVEL=debug ./github-copier ``` ### Validate Config @@ -460,7 +460,7 @@ LOG_LEVEL=debug ./examples-copier ### Dry Run ```bash -DRY_RUN=true ./examples-copier +DRY_RUN=true ./github-copier ``` ### Check Health @@ -499,7 +499,7 @@ gcloud secrets list ## File Locations ``` -examples-copier/ +github-copier/ ├── README.md # Main documentation ├── QUICK-REFERENCE.md # This file ├── docs/ @@ -529,8 +529,8 @@ examples-copier/ - [ ] Set required environment variables - [ ] Create `copier-config.yaml` in source repo - [ ] Validate config: `./config-validator validate -config copier-config.yaml` -- [ ] Test in dry-run: `DRY_RUN=true ./examples-copier` -- [ ] Deploy: `./examples-copier` +- [ ] Test in dry-run: `DRY_RUN=true ./github-copier` +- [ ] Deploy: `./github-copier` - [ ] Configure GitHub webhook - [ ] Monitor: `curl http://localhost:8080/health` diff --git a/README.md b/README.md index 4d2b29b..db0a9aa 100644 --- a/README.md +++ b/README.md @@ -40,13 +40,13 @@ A GitHub app that automatically copies code examples and files from source repos ```bash # Clone the repository git clone https://github.com/your-org/code-example-tooling.git -cd code-example-tooling/examples-copier +cd code-example-tooling/github-copier # Install dependencies go mod download # Build the application -go build -o examples-copier . +go build -o github-copier . # Build CLI tools go build -o config-validator ./cmd/config-validator @@ -137,16 +137,16 @@ workflows: ```bash # Run with default settings -./examples-copier +./github-copier # Run with custom environment file -./examples-copier -env ./configs/.env.production +./github-copier -env ./configs/.env.production # Run in dry-run mode (no actual commits) -./examples-copier -dry-run +./github-copier -dry-run # Validate configuration only -./examples-copier -validate +./github-copier -validate ``` ## Configuration @@ -467,7 +467,7 @@ go tool cover -html=coverage.out Test without making actual changes: ```bash -DRY_RUN=true ./examples-copier +DRY_RUN=true ./github-copier ``` In dry-run mode: @@ -481,9 +481,9 @@ In dry-run mode: Enable detailed logging: ```bash -LOG_LEVEL=debug ./examples-copier +LOG_LEVEL=debug ./github-copier # or -COPIER_DEBUG=true ./examples-copier +COPIER_DEBUG=true ./github-copier ``` ## Architecture @@ -491,7 +491,7 @@ COPIER_DEBUG=true ./examples-copier ### Project Structure ``` -examples-copier/ +github-copier/ ├── app.go # Main application entry point ├── cmd/ │ ├── config-validator/ # CLI validation tool @@ -540,15 +540,15 @@ See [DEPLOYMENT.md](./docs/DEPLOYMENT.md) for complete deployment guide. ### Google Cloud Run ```bash -cd examples-copier +cd github-copier ./scripts/deploy-cloudrun.sh ``` ### Docker ```bash -docker build -t examples-copier . -docker run -p 8080:8080 --env-file env.yaml examples-copier +docker build -t github-copier . +docker run -p 8080:8080 --env-file env.yaml github-copier ``` ## Security diff --git a/app.go b/app.go index ac99a28..f104059 100644 --- a/app.go +++ b/app.go @@ -11,8 +11,8 @@ import ( "syscall" "time" - "github.com/mongodb/code-example-tooling/code-copier/configs" - "github.com/mongodb/code-example-tooling/code-copier/services" + "github.com/grove-platform/github-copier/configs" + "github.com/grove-platform/github-copier/services" ) func main() { @@ -78,7 +78,10 @@ func main() { defer services.CloseGoogleLogger() // Configure GitHub permissions - services.ConfigurePermissions() + if err := services.ConfigurePermissions(); err != nil { + fmt.Printf("❌ Failed to configure GitHub permissions: %v\n", err) + os.Exit(1) + } // Print startup banner printBanner(config, container) @@ -170,27 +173,50 @@ func startWebServer(config *configs.Config, container *services.ServiceContainer IdleTimeout: 120 * time.Second, } - // Handle graceful shutdown + // Channel to signal server errors + serverErr := make(chan error, 1) + + // Start server in goroutine go func() { - sigChan := make(chan os.Signal, 1) - signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM) - <-sigChan + services.LogInfo(fmt.Sprintf("Starting web server on port %s", port)) + if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed { + serverErr <- fmt.Errorf("server error: %w", err) + } + close(serverErr) + }() - log.Println("Shutting down server...") - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() + // Wait for interrupt signal + sigChan := make(chan os.Signal, 1) + signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM) - if err := server.Shutdown(ctx); err != nil { - log.Printf("Server shutdown error: %v\n", err) + // Block until we receive a signal or server error + select { + case err := <-serverErr: + if err != nil { + return err } - }() + case sig := <-sigChan: + log.Printf("Received signal %v, initiating graceful shutdown...", sig) + } + + // Graceful shutdown with timeout + shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + log.Println("Waiting for in-flight requests to complete...") + if err := server.Shutdown(shutdownCtx); err != nil { + log.Printf("Server shutdown error: %v", err) + } else { + log.Println("Server stopped accepting new connections") + } - // Start server - services.LogInfo(fmt.Sprintf("Starting web server on port %s", port)) - if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed { - return fmt.Errorf("server error: %w", err) + // Cleanup resources (flush audit logs, close connections) + log.Println("Cleaning up resources...") + if err := container.Close(shutdownCtx); err != nil { + log.Printf("Cleanup error: %v", err) } + log.Println("Shutdown complete") return nil } diff --git a/cmd/config-validator/README.md b/cmd/config-validator/README.md index b1ae4d4..cfd0c52 100644 --- a/cmd/config-validator/README.md +++ b/cmd/config-validator/README.md @@ -1,6 +1,6 @@ # config-validator -Command-line tool for validating and testing examples-copier workflow configurations. +Command-line tool for validating and testing github-copier workflow configurations. > **Note:** This tool validates individual workflow config files. It does not validate main config files. Main config validation is built into the application itself. @@ -15,7 +15,7 @@ The `config-validator` tool helps you: ## Installation ```bash -cd examples-copier +cd github-copier go build -o config-validator ./cmd/config-validator ``` diff --git a/cmd/config-validator/main.go b/cmd/config-validator/main.go index 866cdcb..9c3eee7 100644 --- a/cmd/config-validator/main.go +++ b/cmd/config-validator/main.go @@ -6,8 +6,8 @@ import ( "os" "strings" - "github.com/mongodb/code-example-tooling/code-copier/services" - "github.com/mongodb/code-example-tooling/code-copier/types" + "github.com/grove-platform/github-copier/services" + "github.com/grove-platform/github-copier/types" ) func main() { @@ -37,7 +37,7 @@ func main() { switch os.Args[1] { case "validate": - validateCmd.Parse(os.Args[2:]) + _ = validateCmd.Parse(os.Args[2:]) if *validateFile == "" { fmt.Println("Error: -config is required") validateCmd.Usage() @@ -46,7 +46,7 @@ func main() { validateConfig(*validateFile, *validateVerbose) case "test-pattern": - testPatternCmd.Parse(os.Args[2:]) + _ = testPatternCmd.Parse(os.Args[2:]) if *pattern == "" || *filePath == "" { fmt.Println("Error: -pattern and -file are required") testPatternCmd.Usage() @@ -55,7 +55,7 @@ func main() { testPattern(*patternType, *pattern, *filePath) case "test-transform": - testTransformCmd.Parse(os.Args[2:]) + _ = testTransformCmd.Parse(os.Args[2:]) if *transformSource == "" || *transformTemplate == "" { fmt.Println("Error: -source and -template are required") testTransformCmd.Usage() @@ -64,7 +64,7 @@ func main() { testTransform(*transformSource, *transformTemplate, *transformVars) case "init": - initCmd.Parse(os.Args[2:]) + _ = initCmd.Parse(os.Args[2:]) initConfig(*initTemplate, *initOutput) default: diff --git a/cmd/test-webhook/README.md b/cmd/test-webhook/README.md index 69db911..10070bf 100644 --- a/cmd/test-webhook/README.md +++ b/cmd/test-webhook/README.md @@ -1,6 +1,6 @@ # test-webhook -Command-line tool for testing the examples-copier webhook endpoint with example or real PR data. +Command-line tool for testing the github-copier webhook endpoint with example or real PR data. ## Overview @@ -14,7 +14,7 @@ The `test-webhook` tool helps you: ## Installation ```bash -cd examples-copier +cd github-copier go build -o test-webhook ./cmd/test-webhook ``` @@ -37,10 +37,10 @@ Send a pre-made example payload to the webhook endpoint. ```bash # Use example payload -./test-webhook -payload test-payloads/example-pr-merged.json +./test-webhook -payload testdata/example-pr-merged.json # Use custom URL -./test-webhook -payload test-payloads/example-pr-merged.json \ +./test-webhook -payload testdata/example-pr-merged.json \ -url http://localhost:8080/webhook ``` @@ -48,7 +48,7 @@ Send a pre-made example payload to the webhook endpoint. ``` Testing webhook with example payload... -✓ Loaded payload from test-payloads/example-pr-merged.json +✓ Loaded payload from testdata/example-pr-merged.json ✓ Response: 200 OK ✓ Webhook sent successfully @@ -111,7 +111,7 @@ Test your configuration locally before deploying: DRY_RUN=true make run-local-quick # 2. In another terminal, send test webhook -./test-webhook -payload test-payloads/example-pr-merged.json +./test-webhook -payload testdata/example-pr-merged.json # 3. Check logs tail -f logs/app.log @@ -139,10 +139,10 @@ Verify files are copied to correct locations: ```bash # 1. Start app in dry-run mode -DRY_RUN=true ./examples-copier & +DRY_RUN=true ./github-copier & # 2. Send test webhook -./test-webhook -payload test-payloads/example-pr-merged.json +./test-webhook -payload testdata/example-pr-merged.json # 3. Check logs for transformed paths grep "transformed path" logs/app.log @@ -155,10 +155,10 @@ Test Slack integration: ```bash # 1. Start app with Slack enabled export SLACK_WEBHOOK_URL="https://hooks.slack.com/services/..." -./examples-copier & +./github-copier & # 2. Send test webhook -./test-webhook -payload test-payloads/example-pr-merged.json +./test-webhook -payload testdata/example-pr-merged.json # 3. Check Slack channel for notification ``` @@ -170,10 +170,10 @@ Debug webhook processing: ```bash # 1. Enable debug logging export LOG_LEVEL=debug -./examples-copier & +./github-copier & # 2. Send test webhook -./test-webhook -payload test-payloads/example-pr-merged.json +./test-webhook -payload testdata/example-pr-merged.json # 3. Review detailed logs grep "DEBUG" logs/app.log @@ -181,7 +181,7 @@ grep "DEBUG" logs/app.log ## Example Payloads -The `test-payloads/` directory contains example webhook payloads: +The `testdata/` directory contains example webhook payloads: ### example-pr-merged.json @@ -192,7 +192,7 @@ A complete merged PR payload with: **Usage:** ```bash -./test-webhook -payload test-payloads/example-pr-merged.json +./test-webhook -payload testdata/example-pr-merged.json ``` ### Creating Custom Payloads @@ -201,13 +201,13 @@ Create custom payloads for specific test scenarios: ```bash # Copy example -cp test-payloads/example-pr-merged.json test-payloads/my-test.json +cp testdata/example-pr-merged.json testdata/my-test.json # Edit to match your test case -vim test-payloads/my-test.json +vim testdata/my-test.json # Test with custom payload -./test-webhook -payload test-payloads/my-test.json +./test-webhook -payload testdata/my-test.json ``` **Example custom payload:** @@ -234,10 +234,10 @@ vim test-payloads/my-test.json ```bash # 1. Start app in dry-run mode -DRY_RUN=true ./examples-copier & +DRY_RUN=true ./github-copier & # 2. Test with example payload -./test-webhook -payload test-payloads/example-pr-merged.json +./test-webhook -payload testdata/example-pr-merged.json # 3. Check metrics curl http://localhost:8080/metrics | jq @@ -274,7 +274,7 @@ Response: 401 Unauthorized **Solution:** Disable webhook signature verification for testing: ```bash unset WEBHOOK_SECRET -./examples-copier & +./github-copier & ``` ### 404 Not Found @@ -311,7 +311,7 @@ Error: invalid JSON payload **Solution:** Validate your JSON: ```bash -cat test-payloads/my-test.json | jq +cat testdata/my-test.json | jq ``` ## Advanced Usage @@ -344,12 +344,12 @@ cat > run-tests.sh << 'EOF' set -e echo "Starting app..." -DRY_RUN=true ./examples-copier & +DRY_RUN=true ./github-copier & APP_PID=$! sleep 2 echo "Running tests..." -./test-webhook -payload test-payloads/example-pr-merged.json +./test-webhook -payload testdata/example-pr-merged.json echo "Checking metrics..." curl -s http://localhost:8080/metrics | jq '.files.matched' @@ -385,14 +385,14 @@ jobs: - name: Build run: | - go build -o examples-copier . + go build -o github-copier . go build -o test-webhook ./cmd/test-webhook - name: Test run: | - DRY_RUN=true ./examples-copier & + DRY_RUN=true ./github-copier & sleep 2 - ./test-webhook -payload test-payloads/example-pr-merged.json + ./test-webhook -payload testdata/example-pr-merged.json ``` ## Exit Codes @@ -404,6 +404,6 @@ jobs: - [Webhook Testing Guide](../../docs/WEBHOOK-TESTING.md) - Comprehensive testing guide - [Local Testing](../../docs/LOCAL-TESTING.md) - Local development -- [Test Payloads](../../test-payloads/README.md) - Example payloads +- [Test Payloads](../../testdata/README.md) - Example payloads - [Quick Reference](../../QUICK-REFERENCE.md) - All commands diff --git a/configs/README.md b/configs/README.md index 068e6a1..5a4ab16 100644 --- a/configs/README.md +++ b/configs/README.md @@ -142,7 +142,7 @@ REPO_OWNER: "mongodb" cp configs/env.yaml.production env-cloudrun.yaml # Remove the 'env_variables:' wrapper # Edit env-cloudrun.yaml with your values -gcloud run deploy examples-copier --source . --env-vars-file=env-cloudrun.yaml +gcloud run deploy github-copier --source . --env-vars-file=env-cloudrun.yaml ``` **Best for:** Cost-effective, scales to zero, serverless @@ -299,7 +299,7 @@ diff configs/env.yaml.production configs/env.yaml.example ## File Locations ``` -examples-copier/ +github-copier/ ├── configs/ │ ├── env.yaml.example # ← Complete reference (all variables) │ ├── env.yaml.production # ← Production template (essential only) diff --git a/configs/copier-config-examples/SOURCE-REPO-README.md b/configs/copier-config-examples/SOURCE-REPO-README.md index 0e1ef61..b48b54c 100644 --- a/configs/copier-config-examples/SOURCE-REPO-README.md +++ b/configs/copier-config-examples/SOURCE-REPO-README.md @@ -8,7 +8,7 @@ This directory contains workflow configurations for automatically copying code e **What do I need to know?** 1. When you **merge a PR** in this repo, the copier automatically runs -2. Files are **matched against patterns** in `.copier/workflows.yaml` +2. Files are **matched against patterns** in `.copier/config.yaml` 3. Matched files are **copied to destination repositories** 4. A **PR is created** in each destination repository (usually) 5. Someone needs to **review and merge** the destination PRs (unless auto-merge is enabled) @@ -31,7 +31,7 @@ gcloud app logs read --limit=100 | grep "your-repo-name" ### File Location -Place your workflow configuration at: `.copier/workflows.yaml` +Place your workflow configuration at: `.copier/config.yaml` ### Basic Workflow Structure @@ -51,7 +51,7 @@ workflows: **All file paths in transformations are relative to the repository root**, not to the config file location. -Even though your config is at `.copier/workflows.yaml`, patterns and paths are matched against the full repository path: +Even though your config is at `.copier/config.yaml`, patterns and paths are matched against the full repository path: ```yaml # ✅ Correct - paths from repository root @@ -69,7 +69,7 @@ transformations: ## Adding a New Workflow -1. **Edit `.copier/workflows.yaml`** in your repository +1. **Edit `.copier/config.yaml`** in your repository 2. **Add a new workflow entry:** @@ -89,7 +89,7 @@ workflows: ## Modifying an Existing Workflow -Simply edit the workflow in `.copier/workflows.yaml` and commit your changes. The updated configuration will be used for the next PR merge. +Simply edit the workflow in `.copier/config.yaml` and commit your changes. The updated configuration will be used for the next PR merge. ## Common Transformation Types @@ -262,7 +262,7 @@ Before committing, you can validate your configuration: ```bash # Validate syntax -./config-validator validate -config .copier/workflows.yaml +./config-validator validate -config .copier/config.yaml # Test a pattern match ./config-validator test-pattern \ @@ -277,7 +277,7 @@ Before committing, you can validate your configuration: 1. **You merge a PR** in this repository 2. **GitHub sends a webhook** to the copier application -3. **Copier loads your workflows** from `.copier/workflows.yaml` +3. **Copier loads your workflows** from `.copier/config.yaml` 4. **Files are matched** against transformation patterns 5. **Files are copied** to destination repositories 6. **PRs are created** in destination repositories (or committed directly) @@ -401,16 +401,16 @@ The deprecation file is stored in **this repository** (source): ## Need Help? -- **Full Documentation**: [Code Example Tooling Repository](https://github.com/mongodb/code-example-tooling) -- **Configuration Examples**: See `examples-copier/configs/copier-config-examples/` -- **Pattern Matching Guide**: See `examples-copier/docs/PATTERN-MATCHING-GUIDE.md` -- **Main Config Architecture**: See `examples-copier/configs/copier-config-examples/MAIN-CONFIG-README.md` -- **Deprecation Tracking**: See `examples-copier/docs/DEPRECATION-TRACKING-EXPLAINED.md` +- **Full Documentation**: [Code Example Tooling Repository](https://github.com/grove-platform/github-copier) +- **Configuration Examples**: See `github-copier/configs/copier-config-examples/` +- **Pattern Matching Guide**: See `github-copier/docs/PATTERN-MATCHING-GUIDE.md` +- **Main Config Architecture**: See `github-copier/configs/copier-config-examples/MAIN-CONFIG-README.md` +- **Deprecation Tracking**: See `github-copier/docs/DEPRECATION-TRACKING-EXPLAINED.md` ## Example: Complete Workflow ```yaml -# .copier/workflows.yaml +# .copier/config.yaml defaults: commit_strategy: @@ -450,7 +450,7 @@ workflows: 1. Was it a merged PR? (not just closed) 2. Do the changed files match your transformation patterns? 3. Check the copier logs (see below) -4. Verify `.copier/workflows.yaml` is valid YAML +4. Verify `.copier/config.yaml` is valid YAML ### How do I view the logs? @@ -469,7 +469,7 @@ gcloud app logs read --limit=200 | grep "your-repo-name" ```bash # Validate YAML syntax -./config-validator validate -config .copier/workflows.yaml +./config-validator validate -config .copier/config.yaml # Test a pattern match ./config-validator test-pattern \ @@ -506,5 +506,5 @@ workflows: ## Questions? -Contact the Developer Docs team or open an issue in the [code-example-tooling repository](https://github.com/mongodb/code-example-tooling/issues). +Contact the Developer Docs team or open an issue in the [github-copier repository](https://github.com/grove-platform/github-copier/issues). diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 0098d30..32c29cb 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -1,6 +1,6 @@ -# Examples Copier Architecture +# GitHub Copier Architecture -This document describes the architecture and design of the examples-copier application, including its core components, main config system, pattern matching, configuration management, deprecation tracking, and operational features. +This document describes the architecture and design of the github-copier application, including its core components, main config system, pattern matching, configuration management, deprecation tracking, and operational features. ## Core Architecture diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index 6717bba..b7d312d 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -69,7 +69,7 @@ gcloud config get-value project ┌─────────────────────────────────────────────────────────────┐ │ Google Cloud Run │ │ ┌──────────────────────────────────────────────────────┐ │ -│ │ examples-copier Service (Container) │ │ +│ │ github-copier Service (Container) │ │ │ │ - Receives webhook │ │ │ │ - Validates signature │ │ │ │ - Loads config from source repo │ │ @@ -108,7 +108,7 @@ This application uses **Google Cloud Run** (serverless containers): **Deployment:** ```bash -gcloud run deploy examples-copier \ +gcloud run deploy github-copier \ --source . \ --region us-central1 \ --env-vars-file=env-cloudrun.yaml @@ -191,7 +191,7 @@ gcloud secrets add-iam-policy-binding mongo-uri \ **Or use the provided script:** ```bash -cd examples-copier +cd github-copier ./scripts/grant-secret-access.sh ``` @@ -215,7 +215,7 @@ gcloud secrets get-iam-policy CODE_COPIER_PEM The `env-cloudrun.yaml` file contains environment variables for Cloud Run deployment. ```bash -cd examples-copier +cd github-copier # Copy from example or create new cp env.yaml env-cloudrun.yaml @@ -335,7 +335,7 @@ services.LoadMongoURI(config) // Loads from Secret Manager The simplest way to deploy is using the provided script: ```bash -cd examples-copier +cd github-copier # Deploy to default region (us-central1) ./scripts/deploy-cloudrun.sh @@ -356,9 +356,9 @@ The script will: If you prefer to run the command directly: ```bash -cd examples-copier +cd github-copier -gcloud run deploy examples-copier \ +gcloud run deploy github-copier \ --source . \ --region us-central1 \ --env-vars-file=env-cloudrun.yaml \ @@ -390,16 +390,16 @@ gcloud run deploy examples-copier \ gcloud run services list --region=us-central1 # Get service URL -SERVICE_URL=$(gcloud run services describe examples-copier \ +SERVICE_URL=$(gcloud run services describe github-copier \ --region=us-central1 \ --format="value(status.url)") echo "Service URL: ${SERVICE_URL}" # View logs -gcloud run services logs read examples-copier --region=us-central1 --limit=50 +gcloud run services logs read github-copier --region=us-central1 --limit=50 # Or tail logs in real-time -gcloud run services logs tail examples-copier --region=us-central1 +gcloud run services logs tail github-copier --region=us-central1 ``` ### Test Health Endpoint @@ -608,7 +608,7 @@ echo -n "mongodb+srv://..." | gcloud secrets create mongo-uri \ ```bash # Run the grant script -cd examples-copier +cd github-copier ./scripts/grant-secret-access.sh ``` @@ -641,7 +641,7 @@ gcloud secrets get-iam-policy mongo-uri | grep @appspot ### ☐ 5. Create env.yaml ```bash -cd examples-copier +cd github-copier # Copy from template cp configs/env.yaml.production env.yaml @@ -698,7 +698,7 @@ env: flex ### ☐ 8. Deploy to Cloud Run ```bash -cd examples-copier +cd github-copier # Deploy using the deployment script ./scripts/deploy-cloudrun.sh diff --git a/docs/DEPRECATION-TRACKING-EXPLAINED.md b/docs/DEPRECATION-TRACKING-EXPLAINED.md index c2d22bc..a5947f8 100644 --- a/docs/DEPRECATION-TRACKING-EXPLAINED.md +++ b/docs/DEPRECATION-TRACKING-EXPLAINED.md @@ -218,7 +218,7 @@ After cleanup, remove the entry from `deprecated_examples.json`. In dry-run mode, deprecation tracking is **simulated**: ```bash -DRY_RUN=true ./examples-copier +DRY_RUN=true ./github-copier ``` **Output:** diff --git a/docs/FAQ.md b/docs/FAQ.md index e26c4c5..2b8a479 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -1,14 +1,14 @@ # Frequently Asked Questions (FAQ) -Common questions about the examples-copier application. +Common questions about the github-copier application. ## General Questions -### What is examples-copier? +### What is github-copier? -Examples-copier is a GitHub app that automatically copies code examples and files from a source repository to one or more target repositories when pull requests are merged. It features advanced pattern matching, path transformations, and audit logging. +The GitHub copier is a GitHub app that automatically copies code examples and files from a source repository to one or more target repositories when pull requests are merged. It features advanced pattern matching, path transformations, and audit logging. -### Why use examples-copier? +### Why use github-copier? - **Automate file synchronization** between repositories - **Maintain consistency** across multiple documentation repos @@ -74,7 +74,7 @@ Yes. A file can match multiple workflows and be copied to multiple targets. This **Workflow configs:** Store in `.copier/workflows/config.yaml` in source repositories, or reference them from the main config. -**For local testing:** Store config files in the examples-copier directory and set appropriate environment variables. +**For local testing:** Store config files in the github-copier directory and set appropriate environment variables. ## Pattern Matching @@ -204,7 +204,7 @@ The app uses Google Cloud Secret Manager for storing GitHub credentials. You cou 2. Send a test webhook: ```bash - ./test-webhook -payload test-payloads/example-pr-merged.json + ./test-webhook -payload testdata/example-pr-merged.json ``` See [Local Testing](LOCAL-TESTING.md) for details. diff --git a/docs/LOCAL-TESTING.md b/docs/LOCAL-TESTING.md index 06ee239..0c4ee64 100644 --- a/docs/LOCAL-TESTING.md +++ b/docs/LOCAL-TESTING.md @@ -1,6 +1,6 @@ # Local Testing Guide -This guide explains how to run and test the examples-copier application locally without requiring Google Cloud or MongoDB. +This guide explains how to run and test the github-copier application locally without requiring Google Cloud or MongoDB. ## Quick Start @@ -18,7 +18,7 @@ make run-local ```bash # One-liner for quick testing -COPIER_DISABLE_CLOUD_LOGGING=true DRY_RUN=true ./examples-copier +COPIER_DISABLE_CLOUD_LOGGING=true DRY_RUN=true ./github-copier ``` ### Option 3: Use Makefile @@ -93,7 +93,7 @@ make test-webhook-example # Or send webhook manually with secret export WEBHOOK_SECRET=$(gcloud secrets versions access latest --secret=webhook-secret) -./test-webhook -payload test-payloads/example-pr-merged.json -secret "$WEBHOOK_SECRET" +./test-webhook -payload testdata/example-pr-merged.json -secret "$WEBHOOK_SECRET" # Or test with real PR export GITHUB_TOKEN=ghp_... @@ -155,7 +155,7 @@ nano .copier/workflows/main.yaml make run-local # 4. Send test webhook -./test-webhook -payload test-payloads/example-pr-merged.json +./test-webhook -payload testdata/example-pr-merged.json # 5. Check logs to verify changes work ``` @@ -311,7 +311,7 @@ AUDIT_COLLECTION=audit_events **Solution:** ```bash # Disable cloud logging for local testing -COPIER_DISABLE_CLOUD_LOGGING=true ./examples-copier +COPIER_DISABLE_CLOUD_LOGGING=true ./github-copier ``` ### Error: "connection refused" when sending webhook @@ -324,12 +324,12 @@ COPIER_DISABLE_CLOUD_LOGGING=true ./examples-copier make run-local-quick # Terminal 2: In a NEW terminal window, send the webhook -cd examples-copier +cd github-copier make test-webhook-example # Or manually: export WEBHOOK_SECRET=$(gcloud secrets versions access latest --secret=webhook-secret) -./test-webhook -payload test-payloads/example-pr-merged.json -secret "$WEBHOOK_SECRET" +./test-webhook -payload testdata/example-pr-merged.json -secret "$WEBHOOK_SECRET" ``` **Note:** The `make test-webhook-example` command requires the server to be running in a separate terminal. You cannot run both commands in the same terminal unless you background the server process. @@ -384,7 +384,7 @@ make build make run-local # 5. In Terminal 2, test with example payload -./test-webhook -payload test-payloads/example-pr-merged.json +./test-webhook -payload testdata/example-pr-merged.json # 6. Check metrics curl http://localhost:8080/metrics | jq @@ -438,7 +438,7 @@ make test-webhook-example # Or test manually with webhook secret export WEBHOOK_SECRET=$(gcloud secrets versions access latest --secret=webhook-secret) -./test-webhook -payload test-payloads/example-pr-merged.json -secret "$WEBHOOK_SECRET" +./test-webhook -payload testdata/example-pr-merged.json -secret "$WEBHOOK_SECRET" # Test with real PR export GITHUB_TOKEN=ghp_... diff --git a/docs/PATTERN-MATCHING-GUIDE.md b/docs/PATTERN-MATCHING-GUIDE.md index 18fe427..59bf661 100644 --- a/docs/PATTERN-MATCHING-GUIDE.md +++ b/docs/PATTERN-MATCHING-GUIDE.md @@ -1,6 +1,6 @@ # Pattern Matching Guide -Complete guide to pattern matching and path transformation in the examples-copier. +Complete guide to pattern matching and path transformation in the github-copier. ## Table of Contents @@ -18,7 +18,7 @@ Complete guide to pattern matching and path transformation in the examples-copie ## Overview -The examples-copier uses a powerful pattern matching system to: +The github-copier uses a powerful pattern matching system to: 1. **Match files** from merged PRs based on their paths 2. **Extract variables** from file paths (e.g., language, category) 3. **Transform paths** to determine where files should be copied diff --git a/docs/SLACK-NOTIFICATIONS.md b/docs/SLACK-NOTIFICATIONS.md index ed35e19..50fef96 100644 --- a/docs/SLACK-NOTIFICATIONS.md +++ b/docs/SLACK-NOTIFICATIONS.md @@ -1,6 +1,6 @@ # Slack Notifications -The examples-copier supports sending notifications to Slack when PRs are processed, files are copied, or errors occur. +The github-copier supports sending notifications to Slack when PRs are processed, files are copied, or errors occur. ## Features @@ -47,7 +47,7 @@ CONFIG_FILE=copier-config.yaml \ make run-local-quick # Send a test webhook -./test-webhook -payload test-payloads/example-pr-merged.json +./test-webhook -payload testdata/example-pr-merged.json ``` You should see a notification in your Slack channel! @@ -217,7 +217,7 @@ export SLACK_WEBHOOK_URL="https://hooks.slack.com/services/..." CONFIG_FILE=copier-config.yaml make run-local-quick # Send test webhook -./test-webhook -payload test-payloads/example-pr-merged.json +./test-webhook -payload testdata/example-pr-merged.json ``` ### Test with Real PR @@ -283,7 +283,7 @@ To reduce notification frequency: Add environment variables to your Cloud Run service: ```bash -gcloud run services update examples-copier \ +gcloud run services update github-copier \ --set-env-vars="SLACK_WEBHOOK_URL=https://hooks.slack.com/services/..." \ --set-env-vars="SLACK_CHANNEL=#code-examples" ``` @@ -294,7 +294,7 @@ Add to your `docker-compose.yml`: ```yaml services: - examples-copier: + github-copier: environment: - SLACK_WEBHOOK_URL=https://hooks.slack.com/services/... - SLACK_CHANNEL=#code-examples diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md index 254f703..f2cc55d 100644 --- a/docs/TROUBLESHOOTING.md +++ b/docs/TROUBLESHOOTING.md @@ -1,6 +1,6 @@ # Troubleshooting Guide -Common issues and solutions for the examples-copier application. +Common issues and solutions for the github-copier application. ## Table of Contents @@ -38,7 +38,7 @@ Common issues and solutions for the examples-copier application. cp copier-config.yaml /path/to/source-repo/copier-config.yaml cd /path/to/source-repo git add copier-config.yaml - git commit -m "Add examples-copier config" + git commit -m "Add github-copier config" git push ``` @@ -221,7 +221,7 @@ pattern: "^examples/(?P[^/]+)/(?P.+)$" 3. **Test with dry-run mode:** ```bash - DRY_RUN=true ./examples-copier + DRY_RUN=true ./github-copier ``` ### No Response from Webhook @@ -241,7 +241,7 @@ pattern: "^examples/(?P[^/]+)/(?P.+)$" 3. **Test with curl:** ```bash - ./test-webhook -payload test-payloads/example-pr-merged.json + ./test-webhook -payload testdata/example-pr-merged.json ``` ## Deployment Issues @@ -256,7 +256,7 @@ pattern: "^examples/(?P[^/]+)/(?P.+)$" **Solution:** Disable cloud logging for local testing: ```bash export COPIER_DISABLE_CLOUD_LOGGING=true -./examples-copier +./github-copier ``` ### MongoDB Connection Failed @@ -305,7 +305,7 @@ export COPIER_DISABLE_CLOUD_LOGGING=true 3. **For testing, use dry-run mode:** ```bash - DRY_RUN=true ./examples-copier + DRY_RUN=true ./github-copier ``` ## Slack Notification Issues @@ -394,7 +394,7 @@ export SLACK_CHANNEL="#your-channel" 1. **Check for memory leaks:** ```bash # Monitor memory usage - top -p $(pgrep examples-copier) + top -p $(pgrep github-copier) ``` 2. **Common causes:** @@ -413,7 +413,7 @@ export SLACK_CHANNEL="#your-channel" ```bash export LOG_LEVEL=debug -./examples-copier +./github-copier ``` ### Check Health Endpoint @@ -455,8 +455,8 @@ curl http://localhost:8080/metrics | jq ### Test with Dry-Run Mode ```bash -DRY_RUN=true ./examples-copier & -./test-webhook -payload test-payloads/example-pr-merged.json +DRY_RUN=true ./github-copier & +./test-webhook -payload testdata/example-pr-merged.json ``` ### Check Audit Logs diff --git a/docs/WEBHOOK-TESTING.md b/docs/WEBHOOK-TESTING.md index 00a0d7e..5042e90 100644 --- a/docs/WEBHOOK-TESTING.md +++ b/docs/WEBHOOK-TESTING.md @@ -1,6 +1,6 @@ # Webhook Testing Guide -This guide explains how to test the examples-copier application with webhooks using real PR data or example payloads. +This guide explains how to test the github-copier application with webhooks using real PR data or example payloads. ## Quick Start @@ -18,10 +18,10 @@ go build -o test-webhook ./cmd/test-webhook ```bash # Send example payload to local server -./test-webhook -payload test-payloads/example-pr-merged.json +./test-webhook -payload testdata/example-pr-merged.json # See payload without sending -./test-webhook -payload test-payloads/example-pr-merged.json -dry-run +./test-webhook -payload testdata/example-pr-merged.json -dry-run ``` ### 3. Test with Real PR Data @@ -45,10 +45,10 @@ Test your configuration changes locally before deploying: ```bash # Terminal 1: Start app in dry-run mode -DRY_RUN=true ./examples-copier +DRY_RUN=true ./github-copier # Terminal 2: Send test webhook -./test-webhook -payload test-payloads/example-pr-merged.json +./test-webhook -payload testdata/example-pr-merged.json # Check Terminal 1 for processing logs ``` @@ -125,7 +125,7 @@ cat > test-go-only.json < /dev/null 2>&1; then + log_info "App is running" + return 0 + else + log_error "App is not running at $APP_URL" + log_info "Start the app with: go run app.go" + return 1 + fi +} + +# Generate HMAC signature for webhook +generate_signature() { + local payload="$1" + echo -n "$payload" | openssl dgst -sha256 -hmac "$WEBHOOK_SECRET" | sed 's/^.* //' +} + +# Send webhook payload +send_webhook() { + local payload_file="$1" + + if [[ ! -f "$payload_file" ]]; then + log_error "Payload file not found: $payload_file" + return 1 + fi + + local payload=$(cat "$payload_file") + local signature="sha256=$(generate_signature "$payload")" + + log_info "Sending webhook payload from $payload_file..." + log_info "Signature: $signature" + + response=$(curl -s -w "\n%{http_code}" \ + -X POST "$APP_URL/webhook" \ + -H "Content-Type: application/json" \ + -H "X-GitHub-Event: pull_request" \ + -H "X-Hub-Signature-256: $signature" \ + -d "$payload") + + http_code=$(echo "$response" | tail -n1) + body=$(echo "$response" | sed '$d') + + if [[ "$http_code" == "200" ]]; then + log_info "Webhook accepted (HTTP $http_code)" + echo "$body" + return 0 + else + log_error "Webhook rejected (HTTP $http_code)" + echo "$body" + return 1 + fi +} + +# Verify files in destination repo (requires gh CLI) +verify_dest_repo() { + local repo="$1" + local path="$2" + + log_info "Checking $repo for $path..." + if gh api "repos/$repo/contents/$path" > /dev/null 2>&1; then + log_info "✓ Found $path in $repo" + return 0 + else + log_warn "✗ Not found: $path in $repo" + return 1 + fi +} + +# Main +main() { + echo "==========================================" + echo "GitHub Copier Integration Test" + echo "==========================================" + + case "${1:-webhook}" in + webhook) + check_app || exit 1 + send_webhook "$PAYLOAD_FILE" + ;; + verify) + log_info "Verifying destination repos..." + verify_dest_repo "cbullinger/copier-app-dest-1" "go-examples" + verify_dest_repo "cbullinger/copier-app-dest-2" "python-examples" + ;; + full) + check_app || exit 1 + send_webhook "$PAYLOAD_FILE" + log_info "Waiting 10s for processing..." + sleep 10 + verify_dest_repo "cbullinger/copier-app-dest-1" "go-examples" + verify_dest_repo "cbullinger/copier-app-dest-2" "python-examples" + ;; + *) + echo "Usage: $0 [webhook|verify|full]" + echo " webhook - Send test webhook to app (default)" + echo " verify - Check destination repos for expected files" + echo " full - Send webhook and verify results" + exit 1 + ;; + esac +} + +main "$@" + diff --git a/scripts/test-and-check.sh b/scripts/test-and-check.sh index cff02a3..2cf2508 100755 --- a/scripts/test-and-check.sh +++ b/scripts/test-and-check.sh @@ -15,7 +15,7 @@ echo -e "${BLUE}Testing webhook with example payload...${NC}" echo "" # Send webhook -./test-webhook -payload test-payloads/example-pr-merged.json +./test-webhook -payload testdata/example-pr-merged.json echo "" echo -e "${GREEN}Webhook sent! Waiting 2 seconds for processing...${NC}" diff --git a/service.yaml.example b/service.yaml.example deleted file mode 100644 index e757792..0000000 --- a/service.yaml.example +++ /dev/null @@ -1,136 +0,0 @@ -apiVersion: serving.knative.dev/v1 -kind: Service -metadata: - annotations: - run.googleapis.com/build-enable-automatic-updates: 'false' - run.googleapis.com/build-id: b757980d-a5c4-4c4e-8080-e592075e3d98 - run.googleapis.com/build-image-uri: us-central1-docker.pkg.dev/github-copy-code-examples/cloud-run-source-deploy/examples-copier - run.googleapis.com/build-name: projects/1054147886816/locations/us-central1/builds/b757980d-a5c4-4c4e-8080-e592075e3d98 - run.googleapis.com/build-source-location: gs://run-sources-github-copy-code-examples-us-central1/services/examples-copier/1761840094.638811-19107807ada84a65915888b814b6e6dd.zip#1761840095211493 - run.googleapis.com/client-name: gcloud - run.googleapis.com/client-version: 542.0.0 - run.googleapis.com/ingress: all - run.googleapis.com/ingress-status: all - run.googleapis.com/operation-id: 5b7f8ffb-c7c1-42f7-a0c5-8ec5532bc0c4 - run.googleapis.com/urls: '["https://examples-copier-1054147886816.us-central1.run.app","https://examples-copier-7c5nckqo6a-uc.a.run.app"]' - serving.knative.dev/creator: cory.bullinger@gcp.corp.mongodb.com - serving.knative.dev/lastModifier: cory.bullinger@gcp.corp.mongodb.com - creationTimestamp: '2025-10-06T18:32:15.311946Z' - generation: 24 - labels: - cloud.googleapis.com/location: us-central1 - name: examples-copier - namespace: '1054147886816' - resourceVersion: AAZCYmPd434 - selfLink: /apis/serving.knative.dev/v1/namespaces/1054147886816/services/examples-copier - uid: c8854b49-365c-4168-a33a-f8a58682a348 -spec: - template: - metadata: - annotations: - autoscaling.knative.dev/maxScale: '10' - run.googleapis.com/client-name: gcloud - run.googleapis.com/client-version: 542.0.0 - run.googleapis.com/startup-cpu-boost: 'true' - labels: - client.knative.dev/nonce: xoklrkvxac - run.googleapis.com/startupProbeType: Default - spec: - containerConcurrency: 80 - containers: - - env: - - name: WEBHOOK_SECRET - valueFrom: - secretKeyRef: - key: latest - name: webhook-secret - - name: MONGO_URI - valueFrom: - secretKeyRef: - key: latest - name: mongo-uri - - name: ADMIN_TOKEN - valueFrom: - secretKeyRef: - key: latest - name: admin-token - - name: GITHUB_APP_ID - value: '1166559' - - name: INSTALLATION_ID - value: '62138132' - - name: REPO_OWNER - value: mongodb - - name: REPO_NAME - value: docs-sample-apps - - name: SRC_BRANCH - value: main - - name: GITHUB_APP_PRIVATE_KEY_SECRET_NAME - value: projects/1054147886816/secrets/CODE_COPIER_PEM/versions/latest - - name: WEBHOOK_SECRET_NAME - value: projects/1054147886816/secrets/webhook-secret/versions/latest - - name: MONGO_URI_SECRET_NAME - value: projects/1054147886816/secrets/mongo-uri/versions/latest - - name: WEBSERVER_PATH - value: /events - - name: CONFIG_FILE - value: copier-config.yaml - - name: DEPRECATION_FILE - value: deprecated_examples.json - - name: COMMITTER_NAME - value: GitHub Copier App - - name: COMMITTER_EMAIL - value: bot@mongodb.com - - name: GOOGLE_CLOUD_PROJECT_ID - value: github-copy-code-examples - - name: COPIER_LOG_NAME - value: code-copier-log - - name: LOG_LEVEL - value: debug - - name: COPIER_DEBUG - value: 'true' - - name: AUDIT_ENABLED - value: 'false' - - name: METRICS_ENABLED - value: 'true' - - name: DRY_RUN - value: 'false' - image: us-central1-docker.pkg.dev/github-copy-code-examples/cloud-run-source-deploy/examples-copier@sha256:d61a9184ff45bea59d3dceba098f99c0bbce2242898607dac65009b8f9f0eae7 - ports: - - containerPort: 8080 - name: http1 - resources: - limits: - cpu: '1' - memory: 512Mi - startupProbe: - failureThreshold: 1 - periodSeconds: 240 - tcpSocket: - port: 8080 - timeoutSeconds: 240 - serviceAccountName: 1054147886816-compute@developer.gserviceaccount.com - timeoutSeconds: 300 - traffic: - - latestRevision: true - percent: 100 -status: - address: - url: https://examples-copier-7c5nckqo6a-uc.a.run.app - conditions: - - lastTransitionTime: '2025-10-30T16:03:29.978238Z' - status: 'True' - type: Ready - - lastTransitionTime: '2025-10-30T16:03:25.908981Z' - status: 'True' - type: ConfigurationsReady - - lastTransitionTime: '2025-10-30T16:03:29.928929Z' - status: 'True' - type: RoutesReady - latestCreatedRevisionName: examples-copier-00024-h29 - latestReadyRevisionName: examples-copier-00024-h29 - observedGeneration: 24 - traffic: - - latestRevision: true - percent: 100 - revisionName: examples-copier-00024-h29 - url: https://examples-copier-7c5nckqo6a-uc.a.run.app diff --git a/services/config_loader.go b/services/config_loader.go index 32177b4..f3567dd 100644 --- a/services/config_loader.go +++ b/services/config_loader.go @@ -8,8 +8,8 @@ import ( "github.com/google/go-github/v48/github" "gopkg.in/yaml.v3" - "github.com/mongodb/code-example-tooling/code-copier/configs" - "github.com/mongodb/code-example-tooling/code-copier/types" + "github.com/grove-platform/github-copier/configs" + "github.com/grove-platform/github-copier/types" ) // ConfigLoader handles loading and parsing configuration files @@ -93,6 +93,9 @@ func retrieveConfigFileContent(ctx context.Context, filePath string, config *con if err != nil { return "", fmt.Errorf("failed to get config file: %w", err) } + if fileContent == nil { + return "", fmt.Errorf("config file content is nil for path: %s", filePath) + } // Decode content content, err := fileContent.GetContent() diff --git a/services/exclude_patterns_test.go b/services/exclude_patterns_test.go index 6c9c49e..6bf5d49 100644 --- a/services/exclude_patterns_test.go +++ b/services/exclude_patterns_test.go @@ -3,7 +3,7 @@ package services import ( "testing" - "github.com/mongodb/code-example-tooling/code-copier/types" + "github.com/grove-platform/github-copier/types" ) func TestExcludePatterns_PrefixPattern(t *testing.T) { @@ -318,4 +318,3 @@ func TestExcludePatterns_ComplexScenarios(t *testing.T) { }) } } - diff --git a/services/file_state_service.go b/services/file_state_service.go index bdb72f2..cdbac21 100644 --- a/services/file_state_service.go +++ b/services/file_state_service.go @@ -3,15 +3,17 @@ package services import ( "sync" - "github.com/mongodb/code-example-tooling/code-copier/types" + "github.com/grove-platform/github-copier/types" ) // FileStateService manages the state of files to upload and deprecate type FileStateService interface { GetFilesToUpload() map[types.UploadKey]types.UploadFileContent - GetFilesToDeprecate() map[string]types.DeprecatedFileEntry + // GetFilesToDeprecate returns all deprecated file entries grouped by deprecation file name + GetFilesToDeprecate() map[string][]types.DeprecatedFileEntry AddFileToUpload(key types.UploadKey, content types.UploadFileContent) - AddFileToDeprecate(file string, entry types.DeprecatedFileEntry) + // AddFileToDeprecate adds a file entry to the deprecation list for the given deprecation file + AddFileToDeprecate(deprecationFile string, entry types.DeprecatedFileEntry) ClearFilesToUpload() ClearFilesToDeprecate() } @@ -20,14 +22,14 @@ type FileStateService interface { type DefaultFileStateService struct { mu sync.RWMutex filesToUpload map[types.UploadKey]types.UploadFileContent - filesToDeprecate map[string]types.DeprecatedFileEntry + filesToDeprecate map[string][]types.DeprecatedFileEntry // Changed: now stores slice of entries per file } // NewFileStateService creates a new file state service func NewFileStateService() FileStateService { return &DefaultFileStateService{ filesToUpload: make(map[types.UploadKey]types.UploadFileContent), - filesToDeprecate: make(map[string]types.DeprecatedFileEntry), + filesToDeprecate: make(map[string][]types.DeprecatedFileEntry), } } @@ -35,7 +37,7 @@ func NewFileStateService() FileStateService { func (fss *DefaultFileStateService) GetFilesToUpload() map[types.UploadKey]types.UploadFileContent { fss.mu.RLock() defer fss.mu.RUnlock() - + // Return a copy to prevent external modification result := make(map[types.UploadKey]types.UploadFileContent, len(fss.filesToUpload)) for k, v := range fss.filesToUpload { @@ -45,14 +47,18 @@ func (fss *DefaultFileStateService) GetFilesToUpload() map[types.UploadKey]types } // GetFilesToDeprecate returns a copy of the files to deprecate map -func (fss *DefaultFileStateService) GetFilesToDeprecate() map[string]types.DeprecatedFileEntry { +// The map key is the deprecation file name, and the value is a slice of all entries for that file +func (fss *DefaultFileStateService) GetFilesToDeprecate() map[string][]types.DeprecatedFileEntry { fss.mu.RLock() defer fss.mu.RUnlock() - // Return a copy to prevent external modification - result := make(map[string]types.DeprecatedFileEntry, len(fss.filesToDeprecate)) + // Return a deep copy to prevent external modification + result := make(map[string][]types.DeprecatedFileEntry, len(fss.filesToDeprecate)) for k, v := range fss.filesToDeprecate { - result[k] = v + // Copy the slice + entriesCopy := make([]types.DeprecatedFileEntry, len(v)) + copy(entriesCopy, v) + result[k] = entriesCopy } return result } @@ -65,19 +71,20 @@ func (fss *DefaultFileStateService) AddFileToUpload(key types.UploadKey, content fss.filesToUpload[key] = content } -// AddFileToDeprecate adds a file to the deprecation list -func (fss *DefaultFileStateService) AddFileToDeprecate(file string, entry types.DeprecatedFileEntry) { +// AddFileToDeprecate appends a file entry to the deprecation list for the given deprecation file +// Multiple entries can be added for the same deprecation file +func (fss *DefaultFileStateService) AddFileToDeprecate(deprecationFile string, entry types.DeprecatedFileEntry) { fss.mu.Lock() defer fss.mu.Unlock() - fss.filesToDeprecate[file] = entry + fss.filesToDeprecate[deprecationFile] = append(fss.filesToDeprecate[deprecationFile], entry) } // ClearFilesToUpload clears the files to upload map func (fss *DefaultFileStateService) ClearFilesToUpload() { fss.mu.Lock() defer fss.mu.Unlock() - + fss.filesToUpload = make(map[types.UploadKey]types.UploadFileContent) } @@ -86,6 +93,5 @@ func (fss *DefaultFileStateService) ClearFilesToDeprecate() { fss.mu.Lock() defer fss.mu.Unlock() - fss.filesToDeprecate = make(map[string]types.DeprecatedFileEntry) + fss.filesToDeprecate = make(map[string][]types.DeprecatedFileEntry) } - diff --git a/services/file_state_service_test.go b/services/file_state_service_test.go index 50cdb55..39bb2d9 100644 --- a/services/file_state_service_test.go +++ b/services/file_state_service_test.go @@ -5,8 +5,8 @@ import ( "testing" "github.com/google/go-github/v48/github" - "github.com/mongodb/code-example-tooling/code-copier/services" - "github.com/mongodb/code-example-tooling/code-copier/types" + "github.com/grove-platform/github-copier/services" + "github.com/grove-platform/github-copier/types" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -61,11 +61,87 @@ func TestFileStateService_AddAndGetFilesToDeprecate(t *testing.T) { files := service.GetFilesToDeprecate() require.Len(t, files, 1) - retrieved, exists := files["deprecated.json"] + entries, exists := files["deprecated.json"] require.True(t, exists) - assert.Equal(t, "old_example.go", retrieved.FileName) - assert.Equal(t, "org/repo", retrieved.Repo) - assert.Equal(t, "main", retrieved.Branch) + require.Len(t, entries, 1) + assert.Equal(t, "old_example.go", entries[0].FileName) + assert.Equal(t, "org/repo", entries[0].Repo) + assert.Equal(t, "main", entries[0].Branch) +} + +func TestFileStateService_MultipleDeprecatedFilesAccumulate(t *testing.T) { + service := services.NewFileStateService() + + // Add multiple files to the same deprecation file + entry1 := types.DeprecatedFileEntry{ + FileName: "file1.go", + Repo: "org/repo", + Branch: "main", + } + entry2 := types.DeprecatedFileEntry{ + FileName: "file2.go", + Repo: "org/repo", + Branch: "main", + } + entry3 := types.DeprecatedFileEntry{ + FileName: "file3.go", + Repo: "org/repo", + Branch: "main", + } + + service.AddFileToDeprecate("deprecated.json", entry1) + service.AddFileToDeprecate("deprecated.json", entry2) + service.AddFileToDeprecate("deprecated.json", entry3) + + // Get files - should have all 3 entries + files := service.GetFilesToDeprecate() + require.Len(t, files, 1) // One deprecation file + + entries, exists := files["deprecated.json"] + require.True(t, exists) + require.Len(t, entries, 3) // Three entries accumulated + + // Verify all entries are present + fileNames := make([]string, len(entries)) + for i, e := range entries { + fileNames[i] = e.FileName + } + assert.Contains(t, fileNames, "file1.go") + assert.Contains(t, fileNames, "file2.go") + assert.Contains(t, fileNames, "file3.go") +} + +func TestFileStateService_MultipleDeprecationFiles(t *testing.T) { + service := services.NewFileStateService() + + // Add entries to different deprecation files + entry1 := types.DeprecatedFileEntry{ + FileName: "file1.go", + Repo: "org/repo1", + Branch: "main", + } + entry2 := types.DeprecatedFileEntry{ + FileName: "file2.go", + Repo: "org/repo2", + Branch: "develop", + } + + service.AddFileToDeprecate("deprecated_repo1.json", entry1) + service.AddFileToDeprecate("deprecated_repo2.json", entry2) + + // Get files - should have 2 deprecation files + files := service.GetFilesToDeprecate() + require.Len(t, files, 2) + + entries1, exists := files["deprecated_repo1.json"] + require.True(t, exists) + require.Len(t, entries1, 1) + assert.Equal(t, "file1.go", entries1[0].FileName) + + entries2, exists := files["deprecated_repo2.json"] + require.True(t, exists) + require.Len(t, entries2, 1) + assert.Equal(t, "file2.go", entries2[0].FileName) } func TestFileStateService_ClearFilesToUpload(t *testing.T) { @@ -301,4 +377,3 @@ func TestFileStateService_CommitStrategyTypes(t *testing.T) { }) } } - diff --git a/services/github_auth.go b/services/github_auth.go index 6e2f57a..683463f 100644 --- a/services/github_auth.go +++ b/services/github_auth.go @@ -7,7 +7,6 @@ import ( "encoding/json" "fmt" "io" - "log" "net/http" "os" "time" @@ -16,8 +15,7 @@ import ( "cloud.google.com/go/secretmanager/apiv1/secretmanagerpb" "github.com/golang-jwt/jwt/v5" "github.com/google/go-github/v48/github" - "github.com/mongodb/code-example-tooling/code-copier/configs" - "github.com/pkg/errors" + "github.com/grove-platform/github-copier/configs" "github.com/shurcooL/graphql" "golang.org/x/oauth2" ) @@ -40,32 +38,36 @@ var jwtExpiry time.Time // ConfigurePermissions sets up the necessary permissions to interact with the GitHub API. // It retrieves the GitHub App's private key from Google Secret Manager, generates a JWT, // and exchanges it for an installation access token. -func ConfigurePermissions() { +func ConfigurePermissions() error { envFilePath := os.Getenv("ENV_FILE") _, err := configs.LoadEnvironment(envFilePath) if err != nil { - log.Fatal(errors.Wrap(err, "Failed to load environment")) + return fmt.Errorf("failed to load environment: %w", err) + } + pemKey, err := getPrivateKeyFromSecret() + if err != nil { + return fmt.Errorf("failed to get private key: %w", err) } - pemKey := getPrivateKeyFromSecret() privateKey, err := jwt.ParseRSAPrivateKeyFromPEM(pemKey) if err != nil { - log.Fatal(errors.Wrap(err, "Unable to parse RSA private key")) + return fmt.Errorf("unable to parse RSA private key: %w", err) } // Generate JWT — use the numeric GitHub App ID (GITHUB_APP_ID) as "iss" token, err := generateGitHubJWT(os.Getenv(configs.AppId), privateKey) if err != nil { - log.Fatal(errors.Wrap(err, "Error generating JWT")) + return fmt.Errorf("error generating JWT: %w", err) } installationToken, err := getInstallationAccessToken("", token, HTTPClient) if err != nil { - log.Fatal(errors.Wrap(err, "Error getting installation access token")) + return fmt.Errorf("error getting installation access token: %w", err) } InstallationAccessToken = installationToken + return nil } // generateGitHubJWT creates a JWT for GitHub App authentication. @@ -88,25 +90,25 @@ func generateGitHubJWT(appID string, privateKey *rsa.PrivateKey) (string, error) // getPrivateKeyFromSecret retrieves the GitHub App's private key from Google Secret Manager. // It supports local testing by allowing the key to be provided via environment variables. -func getPrivateKeyFromSecret() []byte { +func getPrivateKeyFromSecret() ([]byte, error) { if os.Getenv("SKIP_SECRET_MANAGER") == "true" { // for tests and local runs if pem := os.Getenv("GITHUB_APP_PRIVATE_KEY"); pem != "" { - return []byte(pem) + return []byte(pem), nil } if b64 := os.Getenv("GITHUB_APP_PRIVATE_KEY_B64"); b64 != "" { dec, err := base64.StdEncoding.DecodeString(b64) if err != nil { - log.Fatalf("Invalid base64 private key: %v", err) + return nil, fmt.Errorf("invalid base64 private key: %w", err) } - return dec + return dec, nil } - log.Fatalf("SKIP_SECRET_MANAGER=true but no GITHUB_APP_PRIVATE_KEY or GITHUB_APP_PRIVATE_KEY_B64 set") + return nil, fmt.Errorf("SKIP_SECRET_MANAGER=true but no GITHUB_APP_PRIVATE_KEY or GITHUB_APP_PRIVATE_KEY_B64 set") } ctx := context.Background() client, err := secretmanager.NewClient(ctx) if err != nil { - log.Fatalf("Failed to create Secret Manager client: %v", err) + return nil, fmt.Errorf("failed to create Secret Manager client: %w", err) } defer client.Close() @@ -120,9 +122,9 @@ func getPrivateKeyFromSecret() []byte { } result, err := client.AccessSecretVersion(ctx, req) if err != nil { - log.Fatalf("Failed to access secret version: %v", err) + return nil, fmt.Errorf("failed to access secret version: %w", err) } - return result.Payload.Data + return result.Payload.Data, nil } // getWebhookSecretFromSecretManager retrieves the webhook secret from Google Cloud Secret Manager @@ -273,14 +275,16 @@ func GetRestClient() *github.Client { return github.NewClient(httpClient) } -func GetGraphQLClient() *graphql.Client { +func GetGraphQLClient() (*graphql.Client, error) { if InstallationAccessToken == "" { - ConfigurePermissions() + if err := ConfigurePermissions(); err != nil { + return nil, fmt.Errorf("failed to configure permissions: %w", err) + } } client := graphql.NewClient("https://api.github.com/graphql", &http.Client{ Transport: &transport{token: InstallationAccessToken}, }) - return client + return client, nil } // getOrRefreshJWT returns a valid JWT token, generating a new one if expired @@ -291,7 +295,11 @@ func getOrRefreshJWT() (string, error) { } // Generate new JWT - pemKey := getPrivateKeyFromSecret() + pemKey, err := getPrivateKeyFromSecret() + if err != nil { + return "", fmt.Errorf("failed to get private key: %w", err) + } + privateKey, err := jwt.ParseRSAPrivateKeyFromPEM(pemKey) if err != nil { return "", fmt.Errorf("unable to parse RSA private key: %w", err) diff --git a/services/github_auth_test.go b/services/github_auth_test.go index d922869..cbb6f6c 100644 --- a/services/github_auth_test.go +++ b/services/github_auth_test.go @@ -5,7 +5,7 @@ import ( "testing" "time" - "github.com/mongodb/code-example-tooling/code-copier/configs" + "github.com/grove-platform/github-copier/configs" ) func TestGenerateGitHubJWT_EmptyAppID(t *testing.T) { diff --git a/services/github_read.go b/services/github_read.go index fa6be3b..e213712 100644 --- a/services/github_read.go +++ b/services/github_read.go @@ -7,8 +7,8 @@ import ( "os" "github.com/google/go-github/v48/github" - "github.com/mongodb/code-example-tooling/code-copier/configs" - . "github.com/mongodb/code-example-tooling/code-copier/types" + "github.com/grove-platform/github-copier/configs" + . "github.com/grove-platform/github-copier/types" "github.com/shurcooL/githubv4" ) @@ -21,10 +21,15 @@ import ( func GetFilesChangedInPr(owner string, repo string, pr_number int) ([]ChangedFile, error) { if InstallationAccessToken == "" { log.Println("No installation token provided") - ConfigurePermissions() + if err := ConfigurePermissions(); err != nil { + return nil, fmt.Errorf("failed to configure permissions: %w", err) + } } - client := GetGraphQLClient() + client, err := GetGraphQLClient() + if err != nil { + return nil, fmt.Errorf("failed to get GraphQL client: %w", err) + } ctx := context.Background() var changedFiles []ChangedFile @@ -106,7 +111,10 @@ func RetrieveFileContents(filePath string) (github.RepositoryContent, error) { }) if err != nil { - LogCritical(fmt.Sprintf("Error getting file content: %v", err)) + return github.RepositoryContent{}, fmt.Errorf("failed to get file content for %s: %w", filePath, err) + } + if fileContent == nil { + return github.RepositoryContent{}, fmt.Errorf("file content is nil for path: %s", filePath) } return *fileContent, nil } diff --git a/services/github_read_test.go b/services/github_read_test.go index 5bc0bcc..3f79a59 100644 --- a/services/github_read_test.go +++ b/services/github_read_test.go @@ -5,10 +5,10 @@ import ( "testing" "github.com/google/go-github/v48/github" - "github.com/mongodb/code-example-tooling/code-copier/services" + "github.com/grove-platform/github-copier/services" "github.com/stretchr/testify/require" - test "github.com/mongodb/code-example-tooling/code-copier/tests" + test "github.com/grove-platform/github-copier/tests" ) // Helper to b64-encode inline strings diff --git a/services/github_write_to_source.go b/services/github_write_to_source.go index 237d378..c1941cd 100644 --- a/services/github_write_to_source.go +++ b/services/github_write_to_source.go @@ -8,8 +8,8 @@ import ( "time" "github.com/google/go-github/v48/github" - "github.com/mongodb/code-example-tooling/code-copier/configs" - . "github.com/mongodb/code-example-tooling/code-copier/types" + "github.com/grove-platform/github-copier/configs" + . "github.com/grove-platform/github-copier/types" ) func UpdateDeprecationFile() { @@ -36,6 +36,10 @@ func UpdateDeprecationFile() { LogError(fmt.Sprintf("Error getting deprecation file: %v", err)) return } + if fileContent == nil { + LogError("Deprecation file content is nil") + return + } content, err := fileContent.GetContent() if err != nil { @@ -80,6 +84,11 @@ func uploadDeprecationFileChanges(message string, newDeprecationFileContents str if err != nil { LogError(fmt.Sprintf("Error getting deprecation file contents: %v", err)) + return + } + if targetFileContent == nil { + LogError("Target deprecation file content is nil") + return } options := &github.RepositoryContentFileOptions{ @@ -96,5 +105,5 @@ func uploadDeprecationFileChanges(message string, newDeprecationFileContents str LogError(fmt.Sprintf("Cannot update deprecation file: %v", err)) } - LogInfo(fmt.Sprintf("Deprecation file updated.")) + LogInfo("Deprecation file updated.") } diff --git a/services/github_write_to_source_test.go b/services/github_write_to_source_test.go index d98c88a..c9af808 100644 --- a/services/github_write_to_source_test.go +++ b/services/github_write_to_source_test.go @@ -3,7 +3,7 @@ package services import ( "testing" - . "github.com/mongodb/code-example-tooling/code-copier/types" + . "github.com/grove-platform/github-copier/types" ) func TestUpdateDeprecationFile_EmptyList(t *testing.T) { diff --git a/services/github_write_to_target.go b/services/github_write_to_target.go index 8d0eedb..e77066e 100644 --- a/services/github_write_to_target.go +++ b/services/github_write_to_target.go @@ -3,15 +3,14 @@ package services import ( "context" "fmt" - "log" "net/http" "os" "strings" "time" "github.com/google/go-github/v48/github" - "github.com/mongodb/code-example-tooling/code-copier/configs" - . "github.com/mongodb/code-example-tooling/code-copier/types" + "github.com/grove-platform/github-copier/configs" + . "github.com/grove-platform/github-copier/types" "github.com/pkg/errors" ) @@ -22,8 +21,6 @@ import ( var FilesToUpload map[UploadKey]UploadFileContent var FilesToDeprecate map[string]Configs - - // repoOwner returns the config repository owner from environment variables. func repoOwner() string { return os.Getenv(configs.ConfigRepoOwner) } @@ -241,7 +238,10 @@ func addFilesViaPR(ctx context.Context, client *github.Client, key UploadKey, if err = mergePR(ctx, client, key.RepoName, pr.GetNumber()); err != nil { return fmt.Errorf("merge PR: %w", err) } - deleteBranchIfExists(ctx, client, key.RepoName, &github.Reference{Ref: github.String("refs/heads/" + tempBranch)}) + if err = deleteBranchIfExists(ctx, client, key.RepoName, &github.Reference{Ref: github.String("refs/heads/" + tempBranch)}); err != nil { + // Log but don't fail - branch cleanup is not critical + LogWarning(fmt.Sprintf("Failed to delete temp branch after merge: %v", err)) + } } else { LogInfo(fmt.Sprintf("PR created and awaiting review: #%d", pr.GetNumber())) } @@ -289,8 +289,10 @@ func createBranch(ctx context.Context, client *github.Client, repo, newBranch st } // *** Check if branch (newBranchRef) already exists and delete it *** - newBranchRef, _, err := client.Git.GetRef(ctx, owner, repoName, fmt.Sprintf("%s%s", "refs/heads/", newBranch)) - deleteBranchIfExists(ctx, client, normalizedRepo, newBranchRef) + newBranchRef, _, _ := client.Git.GetRef(ctx, owner, repoName, fmt.Sprintf("%s%s", "refs/heads/", newBranch)) + if err := deleteBranchIfExists(ctx, client, normalizedRepo, newBranchRef); err != nil { + return nil, fmt.Errorf("failed to delete existing branch %s: %w", newBranch, err) + } newRef := &github.Reference{ Ref: github.String(fmt.Sprintf("%s%s", "refs/heads/", newBranch)), @@ -441,10 +443,11 @@ func mergePR(ctx context.Context, client *github.Client, repo string, pr_number } // deleteBranchIfExists deletes the specified branch if it exists, except for 'main'. -func deleteBranchIfExists(backgroundContext context.Context, client *github.Client, repo string, ref *github.Reference) { +// Returns an error if attempting to delete the main branch or if deletion fails. +func deleteBranchIfExists(backgroundContext context.Context, client *github.Client, repo string, ref *github.Reference) error { // Early return if ref is nil (branch doesn't exist) if ref == nil { - return + return nil } // Normalize repo name for consistent logging @@ -453,7 +456,7 @@ func deleteBranchIfExists(backgroundContext context.Context, client *github.Clie if ref.GetRef() == "refs/heads/main" { LogError("I refuse to delete branch 'main'.") - log.Fatal() + return fmt.Errorf("refusing to delete protected branch 'main'") } LogInfo(fmt.Sprintf("Deleting branch %s on %s", ref.GetRef(), normalizedRepo)) @@ -463,13 +466,15 @@ func deleteBranchIfExists(backgroundContext context.Context, client *github.Clie _, err = client.Git.DeleteRef(backgroundContext, owner, repoName, ref.GetRef()) if err != nil { LogCritical(fmt.Sprintf("Error deleting branch: %v\n", err)) + return fmt.Errorf("failed to delete branch %s: %w", ref.GetRef(), err) } } + return nil } // DeleteBranchIfExistsExported is an exported wrapper for testing deleteBranchIfExists -func DeleteBranchIfExistsExported(ctx context.Context, client *github.Client, repo string, ref *github.Reference) { - deleteBranchIfExists(ctx, client, repo, ref) +func DeleteBranchIfExistsExported(ctx context.Context, client *github.Client, repo string, ref *github.Reference) error { + return deleteBranchIfExists(ctx, client, repo, ref) } // parseIntWithDefault parses a string to int, returning defaultValue on error diff --git a/services/github_write_to_target_test.go b/services/github_write_to_target_test.go index be93035..a645836 100644 --- a/services/github_write_to_target_test.go +++ b/services/github_write_to_target_test.go @@ -15,14 +15,14 @@ import ( "testing" "github.com/google/go-github/v48/github" + "github.com/grove-platform/github-copier/configs" + "github.com/grove-platform/github-copier/services" + "github.com/grove-platform/github-copier/types" "github.com/jarcoal/httpmock" - "github.com/mongodb/code-example-tooling/code-copier/configs" - "github.com/mongodb/code-example-tooling/code-copier/services" - "github.com/mongodb/code-example-tooling/code-copier/types" "github.com/stretchr/testify/require" // test helpers (utils.go) - test "github.com/mongodb/code-example-tooling/code-copier/tests" + test "github.com/grove-platform/github-copier/tests" ) func TestMain(m *testing.M) { @@ -280,7 +280,8 @@ func TestAddFilesToTargetRepoBranch_ViaPR_Succeeds(t *testing.T) { // Force fresh token; stub token endpoint then configure permissions. services.InstallationAccessToken = "" test.MockGitHubAppTokenEndpoint(os.Getenv(configs.InstallationId)) - services.ConfigurePermissions() + err := services.ConfigurePermissions() + require.NoError(t, err, "ConfigurePermissions should succeed") // Set up cached token for the org to bypass GitHub App auth test.SetupOrgToken(owner, "test-token") @@ -389,8 +390,6 @@ func TestAddFilesToTargetRepoBranch_ViaPR_Succeeds(t *testing.T) { services.FilesToUpload = nil } - - // --- Added critical tests for merge conflicts and configuration/default priorities --- func TestAddFiles_DirectConflict_NonFastForward(t *testing.T) { @@ -445,7 +444,8 @@ func TestAddFiles_ViaPR_MergeConflict_Dirty_NotMerged(t *testing.T) { // Fresh token path services.InstallationAccessToken = "" test.MockGitHubAppTokenEndpoint(os.Getenv(configs.InstallationId)) - services.ConfigurePermissions() + err := services.ConfigurePermissions() + require.NoError(t, err, "ConfigurePermissions should succeed") // Set up cached token for the org to bypass GitHub App auth test.SetupOrgToken(owner, "test-token") @@ -467,6 +467,11 @@ func TestAddFiles_ViaPR_MergeConflict_Dirty_NotMerged(t *testing.T) { "ref": "refs/heads/copier/20250101-000000", "object": map[string]any{"sha": "baseSha"}, }), ) + // Mock DELETE for existing temp branch cleanup + httpmock.RegisterRegexpResponder("DELETE", + regexp.MustCompile(`^https://api\.github\.com/repos/`+owner+`/`+repo+`/git/refs/heads/`+tempHead+`$`), + httpmock.NewStringResponder(204, ""), + ) httpmock.RegisterRegexpResponder("POST", regexp.MustCompile(`^https://api\.github\.com/repos/`+owner+`/`+repo+`/git/trees(\?.*)?$`), httpmock.NewJsonResponderOrPanic(201, map[string]any{"sha": "newTreeSha"}), @@ -484,7 +489,7 @@ func TestAddFiles_ViaPR_MergeConflict_Dirty_NotMerged(t *testing.T) { pr_number := 77 httpmock.RegisterResponder("POST", "https://api.github.com/repos/"+owner+"/"+repo+"/pulls", - httpmock.NewJsonResponderOrPanic(201, map[string]any{"number": pr_number, "html_url": "https://github.com/"+owner+"/"+repo+"/pull/77"}), + httpmock.NewJsonResponderOrPanic(201, map[string]any{"number": pr_number, "html_url": "https://github.com/" + owner + "/" + repo + "/pull/77"}), ) // PR mergeability check returns dirty -> not mergeable httpmock.RegisterResponder("GET", @@ -513,13 +518,14 @@ func TestAddFiles_ViaPR_MergeConflict_Dirty_NotMerged(t *testing.T) { // Assertions info := httpmock.GetCallCountInfo() require.Equal(t, 1, info["POST "+createRefURL]) - require.Equal(t, 1, test.CountByMethodAndURLRegexp("POST", + require.Equal(t, 1, test.CountByMethodAndURLRegexp("POST", regexp.MustCompile(`/repos/`+regexp.QuoteMeta(owner)+`/`+regexp.QuoteMeta(repo)+`/pulls$`))) // No merge call should have been made - require.Equal(t, 0, test.CountByMethodAndURLRegexp("PUT", + require.Equal(t, 0, test.CountByMethodAndURLRegexp("PUT", regexp.MustCompile(`/repos/`+regexp.QuoteMeta(owner)+`/`+regexp.QuoteMeta(repo)+`/pulls/77/merge$`))) - // No delete of temp ref because we returned early - require.Equal(t, 0, test.CountByMethodAndURLRegexp("DELETE", + // Only 1 DELETE call for initial cleanup of existing branch (before creating new one) + // No additional DELETE after merge conflict because we returned early + require.Equal(t, 1, test.CountByMethodAndURLRegexp("DELETE", regexp.MustCompile(`/repos/`+regexp.QuoteMeta(owner)+`/`+regexp.QuoteMeta(repo)+`/git/refs/heads/copier/\d{8}-\d{6}$`))) services.FilesToUpload = nil @@ -593,7 +599,8 @@ func TestPriority_PRTitleDefaultsToCommitMessage_And_NoAutoMergeWhenConfigPresen // Token setup services.InstallationAccessToken = "" test.MockGitHubAppTokenEndpoint(os.Getenv(configs.InstallationId)) - services.ConfigurePermissions() + err := services.ConfigurePermissions() + require.NoError(t, err, "ConfigurePermissions should succeed") // Set up cached token for the org to bypass GitHub App auth test.SetupOrgToken(owner, "test-token") @@ -604,11 +611,16 @@ func TestPriority_PRTitleDefaultsToCommitMessage_And_NoAutoMergeWhenConfigPresen httpmock.NewJsonResponderOrPanic(200, map[string]any{"ref": "refs/heads/" + baseBranch, "object": map[string]any{"sha": "baseSha"}}), ) _ = test.MockCreateRef(owner, repo) - tempHead := `copier/\d{8}-\d{6}` + tempHead := `copier/\d{8}-\d{6}` httpmock.RegisterRegexpResponder("GET", regexp.MustCompile(`^https://api\.github\.com/repos/`+owner+`/`+repo+`/git/ref/(?:refs/)?heads/`+tempHead+`$`), httpmock.NewJsonResponderOrPanic(200, map[string]any{"ref": "refs/heads/copier/20250101-000000", "object": map[string]any{"sha": "baseSha"}}), ) + // Mock DELETE for existing temp branch cleanup + httpmock.RegisterRegexpResponder("DELETE", + regexp.MustCompile(`^https://api\.github\.com/repos/`+owner+`/`+repo+`/git/refs/heads/`+tempHead+`$`), + httpmock.NewStringResponder(204, ""), + ) httpmock.RegisterRegexpResponder("POST", regexp.MustCompile(`^https://api\.github\.com/repos/`+owner+`/`+repo+`/git/trees(\?.*)?$`), httpmock.NewJsonResponderOrPanic(201, map[string]any{"sha": "t"}), @@ -666,7 +678,8 @@ func TestDeleteBranchIfExists_NilReference(t *testing.T) { // Force fresh token services.InstallationAccessToken = "" test.MockGitHubAppTokenEndpoint(os.Getenv(configs.InstallationId)) - services.ConfigurePermissions() + err := services.ConfigurePermissions() + require.NoError(t, err, "ConfigurePermissions should succeed") // This should not panic or make any API calls when ref is nil // We're testing that the function returns early without attempting to delete @@ -674,7 +687,8 @@ func TestDeleteBranchIfExists_NilReference(t *testing.T) { client := services.GetRestClient() // Call with nil reference - should return immediately without error - services.DeleteBranchIfExistsExported(ctx, client, "test-org/test-repo", nil) + err = services.DeleteBranchIfExistsExported(ctx, client, "test-org/test-repo", nil) + require.NoError(t, err, "DeleteBranchIfExistsExported should succeed with nil ref") // Verify no DELETE requests were made (since ref was nil) require.Equal(t, 0, test.CountByMethodAndURLRegexp("DELETE", regexp.MustCompile(`/git/refs/`))) diff --git a/services/health_metrics.go b/services/health_metrics.go index 9e48e97..104e4d2 100644 --- a/services/health_metrics.go +++ b/services/health_metrics.go @@ -9,18 +9,18 @@ import ( // HealthStatus represents the health status of the application type HealthStatus struct { - Status string `json:"status"` - Started bool `json:"started"` - GitHub GitHubHealthStatus `json:"github"` - Queues QueueHealthStatus `json:"queues"` - AuditLogger AuditLoggerHealthStatus `json:"audit_logger,omitempty"` - Uptime string `json:"uptime"` + Status string `json:"status"` + Started bool `json:"started"` + GitHub GitHubHealthStatus `json:"github"` + Queues QueueHealthStatus `json:"queues"` + AuditLogger AuditLoggerHealthStatus `json:"audit_logger,omitempty"` + Uptime string `json:"uptime"` } // GitHubHealthStatus represents GitHub API health type GitHubHealthStatus struct { - Status string `json:"status"` - Authenticated bool `json:"authenticated"` + Status string `json:"status"` + Authenticated bool `json:"authenticated"` } // QueueHealthStatus represents queue health @@ -37,40 +37,40 @@ type AuditLoggerHealthStatus struct { // MetricsData represents application metrics type MetricsData struct { - Webhooks WebhookMetrics `json:"webhooks"` - Files FileMetrics `json:"files"` - GitHubAPI GitHubAPIMetrics `json:"github_api"` - Queues QueueMetrics `json:"queues"` - System SystemMetrics `json:"system"` + Webhooks WebhookMetrics `json:"webhooks"` + Files FileMetrics `json:"files"` + GitHubAPI GitHubAPIMetrics `json:"github_api"` + Queues QueueMetrics `json:"queues"` + System SystemMetrics `json:"system"` } // WebhookMetrics represents webhook processing metrics type WebhookMetrics struct { - Received int64 `json:"received"` - Processed int64 `json:"processed"` - Failed int64 `json:"failed"` - Ignored int64 `json:"ignored"` // Non-PR events - EventTypes map[string]int64 `json:"event_types"` // Count by event type - SuccessRate float64 `json:"success_rate"` + Received int64 `json:"received"` + Processed int64 `json:"processed"` + Failed int64 `json:"failed"` + Ignored int64 `json:"ignored"` // Non-PR events + EventTypes map[string]int64 `json:"event_types"` // Count by event type + SuccessRate float64 `json:"success_rate"` ProcessingTime ProcessingTimeStats `json:"processing_time"` } // FileMetrics represents file operation metrics type FileMetrics struct { - Matched int64 `json:"matched"` - Uploaded int64 `json:"uploaded"` - UploadFailed int64 `json:"upload_failed"` - Deprecated int64 `json:"deprecated"` - UploadSuccessRate float64 `json:"upload_success_rate"` - UploadTime ProcessingTimeStats `json:"upload_time"` + Matched int64 `json:"matched"` + Uploaded int64 `json:"uploaded"` + UploadFailed int64 `json:"upload_failed"` + Deprecated int64 `json:"deprecated"` + UploadSuccessRate float64 `json:"upload_success_rate"` + UploadTime ProcessingTimeStats `json:"upload_time"` } // GitHubAPIMetrics represents GitHub API usage metrics type GitHubAPIMetrics struct { - Calls int64 `json:"calls"` - Errors int64 `json:"errors"` - ErrorRate float64 `json:"error_rate"` - RateLimit RateLimitInfo `json:"rate_limit"` + Calls int64 `json:"calls"` + Errors int64 `json:"errors"` + ErrorRate float64 `json:"error_rate"` + RateLimit RateLimitInfo `json:"rate_limit"` } // RateLimitInfo represents GitHub API rate limit info @@ -103,21 +103,21 @@ type ProcessingTimeStats struct { // MetricsCollector collects and manages application metrics type MetricsCollector struct { - mu sync.RWMutex - startTime time.Time - webhookReceived int64 - webhookProcessed int64 - webhookFailed int64 - webhookIgnored int64 // Non-PR events that were ignored - eventTypes map[string]int64 // Count by event type - filesMatched int64 - filesUploaded int64 + mu sync.RWMutex + startTime time.Time + webhookReceived int64 + webhookProcessed int64 + webhookFailed int64 + webhookIgnored int64 // Non-PR events that were ignored + eventTypes map[string]int64 // Count by event type + filesMatched int64 + filesUploaded int64 filesUploadFailed int64 - filesDeprecated int64 - githubAPICalls int64 - githubAPIErrors int64 - processingTimes []time.Duration - uploadTimes []time.Duration + filesDeprecated int64 + githubAPICalls int64 + githubAPIErrors int64 + processingTimes []time.Duration + uploadTimes []time.Duration } // NewMetricsCollector creates a new metrics collector @@ -143,7 +143,7 @@ func (mc *MetricsCollector) RecordWebhookProcessed(duration time.Duration) { defer mc.mu.Unlock() mc.webhookProcessed++ mc.processingTimes = append(mc.processingTimes, duration) - + // Keep only last 1000 entries if len(mc.processingTimes) > 1000 { mc.processingTimes = mc.processingTimes[len(mc.processingTimes)-1000:] @@ -178,7 +178,7 @@ func (mc *MetricsCollector) RecordFileUploaded(duration time.Duration) { defer mc.mu.Unlock() mc.filesUploaded++ mc.uploadTimes = append(mc.uploadTimes, duration) - + // Keep only last 1000 entries if len(mc.uploadTimes) > 1000 { mc.uploadTimes = mc.uploadTimes[len(mc.uploadTimes)-1000:] @@ -276,12 +276,12 @@ func (mc *MetricsCollector) GetMetrics(fileStateService FileStateService) Metric ProcessingTime: calculateStats(mc.processingTimes), }, Files: FileMetrics{ - Matched: mc.filesMatched, - Uploaded: mc.filesUploaded, - UploadFailed: mc.filesUploadFailed, - Deprecated: mc.filesDeprecated, + Matched: mc.filesMatched, + Uploaded: mc.filesUploaded, + UploadFailed: mc.filesUploadFailed, + Deprecated: mc.filesDeprecated, UploadSuccessRate: uploadSuccessRate, - UploadTime: calculateStats(mc.uploadTimes), + UploadTime: calculateStats(mc.uploadTimes), }, GitHubAPI: GitHubAPIMetrics{ Calls: mc.githubAPICalls, @@ -351,7 +351,7 @@ func HealthHandler(fileStateService FileStateService, startTime time.Time) http. Status: "healthy", Started: true, GitHub: GitHubHealthStatus{ - Status: "healthy", + Status: "healthy", Authenticated: true, }, Queues: QueueHealthStatus{ @@ -362,7 +362,7 @@ func HealthHandler(fileStateService FileStateService, startTime time.Time) http. } w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(health) + _ = json.NewEncoder(w).Encode(health) } } @@ -371,7 +371,6 @@ func MetricsHandler(metricsCollector *MetricsCollector, fileStateService FileSta return func(w http.ResponseWriter, r *http.Request) { metrics := metricsCollector.GetMetrics(fileStateService) w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(metrics) + _ = json.NewEncoder(w).Encode(metrics) } } - diff --git a/services/health_metrics_test.go b/services/health_metrics_test.go index 9a3dc9b..c3ede8d 100644 --- a/services/health_metrics_test.go +++ b/services/health_metrics_test.go @@ -7,8 +7,8 @@ import ( "testing" "time" - "github.com/mongodb/code-example-tooling/code-copier/services" - "github.com/mongodb/code-example-tooling/code-copier/types" + "github.com/grove-platform/github-copier/services" + "github.com/grove-platform/github-copier/types" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -228,11 +228,11 @@ func TestMetricsCollector_ZeroValues(t *testing.T) { func TestMetricsCollector_SuccessRateCalculation(t *testing.T) { tests := []struct { - name string - received int - processed int - failed int - wantRate float64 + name string + received int + processed int + failed int + wantRate float64 }{ { name: "all success", @@ -319,4 +319,3 @@ func TestMetricsCollector_ConcurrentAccess(t *testing.T) { metrics := collector.GetMetrics(fileStateService) assert.Greater(t, metrics.Webhooks.Received, int64(0)) } - diff --git a/services/logger.go b/services/logger.go index 4d51c27..e6c13b5 100644 --- a/services/logger.go +++ b/services/logger.go @@ -11,9 +11,15 @@ import ( "time" "cloud.google.com/go/logging" - "github.com/mongodb/code-example-tooling/code-copier/configs" + "github.com/grove-platform/github-copier/configs" ) +// contextKey is a custom type for context keys to avoid collisions +type contextKey string + +// requestIDKey is the context key for request IDs +const requestIDKey contextKey = "request_id" + var googleInfoLogger *log.Logger var googleWarningLogger *log.Logger var googleErrorLogger *log.Logger @@ -121,7 +127,6 @@ func isCloudLoggingDisabled() bool { return strings.EqualFold(os.Getenv("COPIER_DISABLE_CLOUD_LOGGING"), "true") } - // Context-aware logging functions // LogInfoCtx logs an info message with context and additional fields @@ -197,7 +202,7 @@ func LogAndReturnError(ctx context.Context, operation string, message string, er // formatLogMessage formats a log message with context and fields func formatLogMessage(ctx context.Context, message string, fields map[string]interface{}) string { - if fields == nil || len(fields) == 0 { + if len(fields) == 0 { return message } @@ -215,8 +220,8 @@ func WithRequestID(r *http.Request) (context.Context, string) { // Generate a simple request ID requestID := fmt.Sprintf("%d", time.Now().UnixNano()) - // Add to context - ctx := context.WithValue(r.Context(), "request_id", requestID) + // Add to context using typed key to avoid collisions + ctx := context.WithValue(r.Context(), requestIDKey, requestID) return ctx, requestID } diff --git a/services/logger_test.go b/services/logger_test.go index a764673..19f148f 100644 --- a/services/logger_test.go +++ b/services/logger_test.go @@ -13,11 +13,11 @@ import ( func TestLogDebug(t *testing.T) { tests := []struct { - name string - logLevel string - copierDebug string - message string - shouldLog bool + name string + logLevel string + copierDebug string + message string + shouldLog bool }{ { name: "debug enabled via LOG_LEVEL", @@ -289,15 +289,15 @@ func TestLogFileOperation(t *testing.T) { func TestWithRequestID(t *testing.T) { req := httptest.NewRequest("GET", "/test", nil) - + ctx, requestID := WithRequestID(req) - + if requestID == "" { t.Error("Expected non-empty request ID") } - // Check that request ID is in context - ctxValue := ctx.Value("request_id") + // Check that request ID is in context using the typed key + ctxValue := ctx.Value(requestIDKey) if ctxValue == nil { t.Error("Expected request_id in context") } @@ -405,4 +405,3 @@ func TestIsCloudLoggingDisabled(t *testing.T) { }) } } - diff --git a/services/main_config_loader.go b/services/main_config_loader.go index 48f0892..274663b 100644 --- a/services/main_config_loader.go +++ b/services/main_config_loader.go @@ -9,8 +9,8 @@ import ( "github.com/google/go-github/v48/github" "gopkg.in/yaml.v3" - "github.com/mongodb/code-example-tooling/code-copier/configs" - "github.com/mongodb/code-example-tooling/code-copier/types" + "github.com/grove-platform/github-copier/configs" + "github.com/grove-platform/github-copier/types" ) // DefaultMainConfigLoader implements the ConfigLoader interface with main config support @@ -246,6 +246,9 @@ func (mcl *DefaultMainConfigLoader) loadLocalWorkflowConfig(ctx context.Context, if err != nil { return nil, fmt.Errorf("failed to get workflow config file: %w", err) } + if fileContent == nil { + return nil, fmt.Errorf("workflow config file content is nil for path: %s", ref.Path) + } content, err = fileContent.GetContent() if err != nil { @@ -295,6 +298,9 @@ func (mcl *DefaultMainConfigLoader) loadRemoteWorkflowConfig(ctx context.Context if err != nil { return nil, fmt.Errorf("failed to get workflow config file from %s: %w", ref.Repo, err) } + if fileContent == nil { + return nil, fmt.Errorf("workflow config file content is nil for path: %s in repo %s", ref.Path, ref.Repo) + } content, err := fileContent.GetContent() if err != nil { @@ -415,7 +421,7 @@ func (mcl *DefaultMainConfigLoader) resolveReference(ctx context.Context, ref st // Supports: // - Relative paths: "strategies/pr-strategy.yaml" // - Repo references: "repo://owner/repo/path/to/file.yaml@branch" - + if strings.HasPrefix(ref, "repo://") { // Remote repo reference return mcl.resolveRemoteReference(ctx, ref) @@ -429,7 +435,7 @@ func (mcl *DefaultMainConfigLoader) resolveReference(ctx context.Context, ref st func (mcl *DefaultMainConfigLoader) resolveRemoteReference(ctx context.Context, ref string) (string, error) { // Parse: repo://owner/repo/path/to/file.yaml@branch ref = strings.TrimPrefix(ref, "repo://") - + // Split by @ to get branch parts := strings.Split(ref, "@") branch := "main" @@ -466,6 +472,9 @@ func (mcl *DefaultMainConfigLoader) resolveRemoteReference(ctx context.Context, if err != nil { return "", fmt.Errorf("failed to get referenced file: %w", err) } + if fileContent == nil { + return "", fmt.Errorf("referenced file content is nil for path: %s", filePath) + } return fileContent.GetContent() } @@ -502,7 +511,9 @@ func (mcl *DefaultMainConfigLoader) resolveRelativeReference(ctx context.Context if err != nil { return "", fmt.Errorf("failed to get referenced file: %w", err) } + if fileContent == nil { + return "", fmt.Errorf("referenced file content is nil for path: %s", resolvedPath) + } return fileContent.GetContent() } - diff --git a/services/main_config_loader_test.go b/services/main_config_loader_test.go index 7c5cc56..eab8f3a 100644 --- a/services/main_config_loader_test.go +++ b/services/main_config_loader_test.go @@ -8,9 +8,9 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/mongodb/code-example-tooling/code-copier/configs" - "github.com/mongodb/code-example-tooling/code-copier/services" - test "github.com/mongodb/code-example-tooling/code-copier/tests" + "github.com/grove-platform/github-copier/configs" + "github.com/grove-platform/github-copier/services" + test "github.com/grove-platform/github-copier/tests" ) // Helper to encode YAML content to base64 for main config tests @@ -980,4 +980,3 @@ workflow_configs: assert.Equal(t, "mongodb/working-repo", workflow.Source.Repo) assert.Equal(t, "mongodb/dest-repo", workflow.Destination.Repo) } - diff --git a/services/pattern_matcher.go b/services/pattern_matcher.go index 46e1026..ee3feb0 100644 --- a/services/pattern_matcher.go +++ b/services/pattern_matcher.go @@ -7,7 +7,7 @@ import ( "strings" "github.com/bmatcuk/doublestar/v4" - "github.com/mongodb/code-example-tooling/code-copier/types" + "github.com/grove-platform/github-copier/types" ) // PatternMatcher handles pattern matching for file paths @@ -56,20 +56,20 @@ func (pm *DefaultPatternMatcher) Match(filePath string, pattern types.SourcePatt func (pm *DefaultPatternMatcher) matchPrefix(filePath, pattern string) types.MatchResult { // Normalize paths (remove trailing slashes) pattern = strings.TrimSuffix(pattern, "/") - + if strings.HasPrefix(filePath, pattern) { // Extract the relative path after the prefix relPath := strings.TrimPrefix(filePath, pattern) relPath = strings.TrimPrefix(relPath, "/") - + variables := map[string]string{ "matched_prefix": pattern, "relative_path": relPath, } - + return types.NewMatchResult(true, variables) } - + return types.NewMatchResult(false, nil) } @@ -95,8 +95,6 @@ func (pm *DefaultPatternMatcher) matchGlob(filePath, pattern string) types.Match return types.NewMatchResult(false, nil) } - - // matchRegex matches using regular expressions with named capture groups func (pm *DefaultPatternMatcher) matchRegex(filePath, pattern string) types.MatchResult { re, err := regexp.Compile(pattern) @@ -148,14 +146,14 @@ func (pt *DefaultPathTransformer) Transform(sourcePath string, template string, // Create transformation context ctx := types.NewTransformContext(sourcePath, variables) ctx.AddBuiltInVariables() - + // Replace variables in template result := template for key, value := range ctx.Variables { placeholder := fmt.Sprintf("${%s}", key) result = strings.ReplaceAll(result, placeholder, value) } - + // Check for unreplaced variables if strings.Contains(result, "${") { // Extract unreplaced variable names for better error message @@ -164,7 +162,7 @@ func (pt *DefaultPathTransformer) Transform(sourcePath string, template string, return "", fmt.Errorf("unreplaced variables in template: %v", unreplaced) } } - + return result, nil } @@ -215,7 +213,7 @@ func (mt *DefaultMessageTemplater) RenderPRTitle(template string, ctx *types.Mes // RenderPRBody renders a PR body template func (mt *DefaultMessageTemplater) RenderPRBody(template string, ctx *types.MessageContext) string { if template == "" { - return fmt.Sprintf("Automated update of %d file(s) from %s (PR #%d)", + return fmt.Sprintf("Automated update of %d file(s) from %s (PR #%d)", ctx.FileCount, ctx.SourceRepo, ctx.PRNumber) } return mt.render(template, ctx) @@ -224,7 +222,7 @@ func (mt *DefaultMessageTemplater) RenderPRBody(template string, ctx *types.Mess // render performs the actual template rendering func (mt *DefaultMessageTemplater) render(template string, ctx *types.MessageContext) string { result := template - + // Built-in context variables replacements := map[string]string{ "${rule_name}": ctx.RuleName, @@ -236,18 +234,18 @@ func (mt *DefaultMessageTemplater) render(template string, ctx *types.MessageCon "${pr_number}": fmt.Sprintf("%d", ctx.PRNumber), "${commit_sha}": ctx.CommitSHA, } - + // Apply built-in replacements for placeholder, value := range replacements { result = strings.ReplaceAll(result, placeholder, value) } - + // Apply custom variables from pattern matching for key, value := range ctx.Variables { placeholder := fmt.Sprintf("${%s}", key) result = strings.ReplaceAll(result, placeholder, value) } - + return result } @@ -273,4 +271,3 @@ func (pm *DefaultPatternMatcher) shouldExclude(filePath string, excludePatterns return false } - diff --git a/services/pattern_matcher_test.go b/services/pattern_matcher_test.go index c999a64..90dbf29 100644 --- a/services/pattern_matcher_test.go +++ b/services/pattern_matcher_test.go @@ -3,8 +3,8 @@ package services_test import ( "testing" - "github.com/mongodb/code-example-tooling/code-copier/services" - "github.com/mongodb/code-example-tooling/code-copier/types" + "github.com/grove-platform/github-copier/services" + "github.com/grove-platform/github-copier/types" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -13,9 +13,9 @@ func TestPatternMatcher_Prefix(t *testing.T) { matcher := services.NewPatternMatcher() tests := []struct { - name string - pattern types.SourcePattern - filePath string + name string + pattern types.SourcePattern + filePath string wantMatch bool }{ { diff --git a/services/service_container.go b/services/service_container.go index a112eab..e9b6879 100644 --- a/services/service_container.go +++ b/services/service_container.go @@ -3,9 +3,10 @@ package services import ( "context" "fmt" + "sync" "time" - "github.com/mongodb/code-example-tooling/code-copier/configs" + "github.com/grove-platform/github-copier/configs" ) // ServiceContainer holds all application services with dependency injection @@ -25,6 +26,10 @@ type ServiceContainer struct { // Server state StartTime time.Time + + // Shutdown state + closeOnce sync.Once + closed bool } // NewServiceContainer creates and initializes all services @@ -84,10 +89,19 @@ func NewServiceContainer(config *configs.Config) (*ServiceContainer, error) { }, nil } -// Close cleans up resources +// Close cleans up resources. Safe to call multiple times. func (sc *ServiceContainer) Close(ctx context.Context) error { - if sc.AuditLogger != nil { - return sc.AuditLogger.Close(ctx) - } - return nil + var closeErr error + sc.closeOnce.Do(func() { + if sc.AuditLogger != nil { + closeErr = sc.AuditLogger.Close(ctx) + } + sc.closed = true + }) + return closeErr +} + +// IsClosed returns true if the container has been closed +func (sc *ServiceContainer) IsClosed() bool { + return sc.closed } diff --git a/services/service_container_test.go b/services/service_container_test.go index b230bfb..ed8bcc7 100644 --- a/services/service_container_test.go +++ b/services/service_container_test.go @@ -5,7 +5,7 @@ import ( "testing" "time" - "github.com/mongodb/code-example-tooling/code-copier/configs" + "github.com/grove-platform/github-copier/configs" ) func TestNewServiceContainer(t *testing.T) { @@ -46,7 +46,7 @@ func TestNewServiceContainer(t *testing.T) { ConfigRepoOwner: "test-owner", ConfigRepoName: "test-repo", AuditEnabled: true, - MongoURI: "", + MongoURI: "", }, wantErr: true, checkServices: false, @@ -126,9 +126,9 @@ func TestNewServiceContainer(t *testing.T) { func TestServiceContainer_Close(t *testing.T) { tests := []struct { - name string - config *configs.Config - wantErr bool + name string + config *configs.Config + wantErr bool }{ { name: "close with NoOp audit logger", @@ -196,12 +196,12 @@ func TestServiceContainer_ConfigPropagation(t *testing.T) { func TestServiceContainer_SlackNotifierConfiguration(t *testing.T) { tests := []struct { - name string - webhookURL string - channel string - username string - iconEmoji string - wantEnabled bool + name string + webhookURL string + channel string + username string + iconEmoji string + wantEnabled bool }{ { name: "Slack enabled", @@ -276,8 +276,8 @@ func TestServiceContainer_AuditLoggerConfiguration(t *testing.T) { ConfigRepoOwner: "test-owner", ConfigRepoName: "test-repo", AuditEnabled: tt.auditEnabled, - MongoURI: tt.mongoURI, - AuditDatabase: "test-db", + MongoURI: tt.mongoURI, + AuditDatabase: "test-db", AuditCollection: "test-coll", } @@ -357,4 +357,3 @@ func TestServiceContainer_StartTimeTracking(t *testing.T) { t.Error("StartTime is after container creation") } } - diff --git a/services/slack_notifier_test.go b/services/slack_notifier_test.go index 03b7655..6dd9030 100644 --- a/services/slack_notifier_test.go +++ b/services/slack_notifier_test.go @@ -13,27 +13,27 @@ import ( func TestNewSlackNotifier(t *testing.T) { tests := []struct { - name string - webhookURL string - channel string - username string - iconEmoji string + name string + webhookURL string + channel string + username string + iconEmoji string wantEnabled bool }{ { - name: "enabled with webhook URL", - webhookURL: "https://hooks.slack.com/services/TEST", - channel: "#test", - username: "Test Bot", - iconEmoji: ":robot:", + name: "enabled with webhook URL", + webhookURL: "https://hooks.slack.com/services/TEST", + channel: "#test", + username: "Test Bot", + iconEmoji: ":robot:", wantEnabled: true, }, { - name: "disabled without webhook URL", - webhookURL: "", - channel: "#test", - username: "Test Bot", - iconEmoji: ":robot:", + name: "disabled without webhook URL", + webhookURL: "", + channel: "#test", + username: "Test Bot", + iconEmoji: ":robot:", wantEnabled: false, }, } @@ -93,7 +93,7 @@ func TestSlackNotifier_NotifyPRProcessed(t *testing.T) { var receivedMessage *SlackMessage server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { body, _ := io.ReadAll(r.Body) - json.Unmarshal(body, &receivedMessage) + _ = json.Unmarshal(body, &receivedMessage) w.WriteHeader(http.StatusOK) })) defer server.Close() @@ -138,7 +138,7 @@ func TestSlackNotifier_NotifyError(t *testing.T) { var receivedMessage *SlackMessage server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { body, _ := io.ReadAll(r.Body) - json.Unmarshal(body, &receivedMessage) + _ = json.Unmarshal(body, &receivedMessage) w.WriteHeader(http.StatusOK) })) defer server.Close() @@ -171,18 +171,18 @@ func TestSlackNotifier_NotifyError(t *testing.T) { func TestSlackNotifier_NotifyFilesCopied(t *testing.T) { tests := []struct { - name string - fileCount int + name string + fileCount int wantTruncated bool }{ { - name: "few files", - fileCount: 5, + name: "few files", + fileCount: 5, wantTruncated: false, }, { - name: "many files", - fileCount: 15, + name: "many files", + fileCount: 15, wantTruncated: true, }, } @@ -206,7 +206,7 @@ func TestSlackNotifier_NotifyFilesCopied(t *testing.T) { var receivedMessage *SlackMessage server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { body, _ := io.ReadAll(r.Body) - json.Unmarshal(body, &receivedMessage) + _ = json.Unmarshal(body, &receivedMessage) w.WriteHeader(http.StatusOK) })) defer server.Close() @@ -245,7 +245,7 @@ func TestSlackNotifier_NotifyDeprecation(t *testing.T) { var receivedMessage *SlackMessage server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { body, _ := io.ReadAll(r.Body) - json.Unmarshal(body, &receivedMessage) + _ = json.Unmarshal(body, &receivedMessage) w.WriteHeader(http.StatusOK) })) defer server.Close() @@ -329,4 +329,3 @@ func containsMiddle(s, substr string) bool { } return false } - diff --git a/services/webhook_handler_new.go b/services/webhook_handler_new.go index 8d090e7..dd7ec04 100644 --- a/services/webhook_handler_new.go +++ b/services/webhook_handler_new.go @@ -12,8 +12,8 @@ import ( "time" "github.com/google/go-github/v48/github" - "github.com/mongodb/code-example-tooling/code-copier/configs" - "github.com/mongodb/code-example-tooling/code-copier/types" + "github.com/grove-platform/github-copier/configs" + "github.com/grove-platform/github-copier/types" ) const ( @@ -184,7 +184,7 @@ func HandleWebhookWithContainer(w http.ResponseWriter, r *http.Request, config * w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusAccepted) - w.Write([]byte(`{"status":"accepted"}`)) + _, _ = w.Write([]byte(`{"status":"accepted"}`)) LogInfoCtx(ctx, "response sent", map[string]interface{}{ "elapsed_ms": time.Since(startTime).Milliseconds(), @@ -210,7 +210,11 @@ func handleMergedPRWithContainer(ctx context.Context, prNumber int, sourceCommit // Configure GitHub permissions if InstallationAccessToken == "" { - ConfigurePermissions() + if err := ConfigurePermissions(); err != nil { + LogAndReturnError(ctx, "auth", "failed to configure GitHub permissions", err) + container.MetricsCollector.RecordWebhookFailed() + return + } } // Load configuration using new loader @@ -222,7 +226,7 @@ func handleMergedPRWithContainer(ctx context.Context, prNumber int, sourceCommit container.MetricsCollector.RecordWebhookFailed() // Send error notification to Slack - container.SlackNotifier.NotifyError(ctx, &ErrorEvent{ + _ = container.SlackNotifier.NotifyError(ctx, &ErrorEvent{ Operation: "config_load", Error: err, PRNumber: prNumber, @@ -267,7 +271,7 @@ func handleMergedPRWithContainer(ctx context.Context, prNumber int, sourceCommit container.MetricsCollector.RecordWebhookFailed() // Send error notification to Slack - container.SlackNotifier.NotifyError(ctx, &ErrorEvent{ + _ = container.SlackNotifier.NotifyError(ctx, &ErrorEvent{ Operation: "get_files", Error: err, PRNumber: prNumber, @@ -294,12 +298,16 @@ func handleMergedPRWithContainer(ctx context.Context, prNumber int, sourceCommit container.FileStateService.ClearFilesToUpload() // Update deprecation file - copy from FileStateService to global map for legacy function + // The deprecationMap is keyed by deprecation file name, with a slice of entries per file deprecationMap := container.FileStateService.GetFilesToDeprecate() FilesToDeprecate = make(map[string]types.Configs) - for _, entry := range deprecationMap { - FilesToDeprecate[entry.FileName] = types.Configs{ - TargetRepo: entry.Repo, - TargetBranch: entry.Branch, + for _, entries := range deprecationMap { + // Iterate over all entries for each deprecation file + for _, entry := range entries { + FilesToDeprecate[entry.FileName] = types.Configs{ + TargetRepo: entry.Repo, + TargetBranch: entry.Branch, + } } } UpdateDeprecationFile() @@ -317,7 +325,7 @@ func handleMergedPRWithContainer(ctx context.Context, prNumber int, sourceCommit }) // Send success notification to Slack - container.SlackNotifier.NotifyPRProcessed(ctx, &PRProcessedEvent{ + _ = container.SlackNotifier.NotifyPRProcessed(ctx, &PRProcessedEvent{ PRNumber: prNumber, PRTitle: fmt.Sprintf("PR #%d", prNumber), // TODO: Get actual PR title from GitHub PRURL: fmt.Sprintf("https://github.com/%s/pull/%d", webhookRepo, prNumber), diff --git a/services/webhook_handler_new_test.go b/services/webhook_handler_new_test.go index 1f0e129..d488ac3 100644 --- a/services/webhook_handler_new_test.go +++ b/services/webhook_handler_new_test.go @@ -17,7 +17,7 @@ import ( "testing" "github.com/google/go-github/v48/github" - "github.com/mongodb/code-example-tooling/code-copier/configs" + "github.com/grove-platform/github-copier/configs" ) func TestSimpleVerifySignature(t *testing.T) { @@ -87,8 +87,8 @@ func TestHandleWebhookWithContainer_MissingEventType(t *testing.T) { config := &configs.Config{ ConfigRepoOwner: "test-owner", ConfigRepoName: "test-repo", - - AuditEnabled: false, + + AuditEnabled: false, } container, err := NewServiceContainer(config) @@ -117,9 +117,9 @@ func TestHandleWebhookWithContainer_InvalidSignature(t *testing.T) { config := &configs.Config{ ConfigRepoOwner: "test-owner", ConfigRepoName: "test-repo", - - WebhookSecret: "test-secret", - AuditEnabled: false, + + WebhookSecret: "test-secret", + AuditEnabled: false, } container, err := NewServiceContainer(config) @@ -146,9 +146,9 @@ func TestHandleWebhookWithContainer_ValidSignature(t *testing.T) { config := &configs.Config{ ConfigRepoOwner: "test-owner", ConfigRepoName: "test-repo", - - WebhookSecret: secret, - AuditEnabled: false, + + WebhookSecret: secret, + AuditEnabled: false, } container, err := NewServiceContainer(config) @@ -190,8 +190,8 @@ func TestHandleWebhookWithContainer_NonPREvent(t *testing.T) { config := &configs.Config{ ConfigRepoOwner: "test-owner", ConfigRepoName: "test-repo", - - AuditEnabled: false, + + AuditEnabled: false, } container, err := NewServiceContainer(config) @@ -222,8 +222,8 @@ func TestHandleWebhookWithContainer_NonMergedPR(t *testing.T) { config := &configs.Config{ ConfigRepoOwner: "test-owner", ConfigRepoName: "test-repo", - - AuditEnabled: false, + + AuditEnabled: false, } container, err := NewServiceContainer(config) @@ -328,7 +328,7 @@ func TestHandleWebhookWithContainer_MergedPR(t *testing.T) { // Check response body var response map[string]string - json.Unmarshal(w.Body.Bytes(), &response) + _ = json.Unmarshal(w.Body.Bytes(), &response) if response["status"] != "accepted" { t.Errorf("Response status = %v, want accepted", response["status"]) } @@ -404,7 +404,7 @@ func TestHandleWebhookWithContainer_MergedPRToDevelopmentBranch(t *testing.T) { // Check response body var response map[string]string - json.Unmarshal(w.Body.Bytes(), &response) + _ = json.Unmarshal(w.Body.Bytes(), &response) if response["status"] != "accepted" { t.Errorf("Response status = %v, want accepted", response["status"]) } @@ -509,7 +509,7 @@ func TestHandleWebhookWithContainer_MergedPRWithDifferentBranches(t *testing.T) // Check response body var response map[string]string - json.Unmarshal(w.Body.Bytes(), &response) + _ = json.Unmarshal(w.Body.Bytes(), &response) if response["status"] != "accepted" { t.Errorf("Response status = %v, want accepted", response["status"]) } @@ -543,4 +543,3 @@ func TestStatusDeleted(t *testing.T) { t.Errorf("statusDeleted = %s, want DELETED", statusDeleted) } } - diff --git a/services/workflow_processor.go b/services/workflow_processor.go index 86cf222..a766907 100644 --- a/services/workflow_processor.go +++ b/services/workflow_processor.go @@ -9,7 +9,7 @@ import ( "github.com/bmatcuk/doublestar/v4" "github.com/google/go-github/v48/github" - . "github.com/mongodb/code-example-tooling/code-copier/types" + . "github.com/grove-platform/github-copier/types" ) // WorkflowProcessor processes workflows and applies transformations @@ -52,10 +52,10 @@ func (wp *workflowProcessor) ProcessWorkflow( sourceCommitSHA string, ) error { LogInfoCtx(ctx, "Processing workflow", map[string]interface{}{ - "workflow_name": workflow.Name, - "source_repo": workflow.Source.Repo, + "workflow_name": workflow.Name, + "source_repo": workflow.Source.Repo, "destination_repo": workflow.Destination.Repo, - "file_count": len(changedFiles), + "file_count": len(changedFiles), }) // Track files matched and skipped @@ -81,9 +81,9 @@ func (wp *workflowProcessor) ProcessWorkflow( } LogInfoCtx(ctx, "Workflow processing complete", map[string]interface{}{ - "workflow_name": workflow.Name, - "files_matched": filesMatched, - "files_skipped": filesSkipped, + "workflow_name": workflow.Name, + "files_matched": filesMatched, + "files_skipped": filesSkipped, }) return nil @@ -119,15 +119,16 @@ func (wp *workflowProcessor) processFileForWorkflow( // File matched this transformation LogInfoCtx(ctx, "File matched transformation", map[string]interface{}{ - "workflow_name": workflow.Name, - "transformation_idx": i, + "workflow_name": workflow.Name, + "transformation_idx": i, "transformation_type": transformation.GetType(), - "source_path": file.Path, - "target_path": targetPath, + "source_path": file.Path, + "target_path": targetPath, }) // Handle file based on status - if file.Status == "removed" { + // GitHub GraphQL API returns uppercase status: "DELETED", "ADDED", "MODIFIED", etc. + if file.Status == "DELETED" || file.Status == "removed" { // Add to deprecation map wp.addToDeprecationMap(workflow, targetPath) } else { @@ -178,12 +179,12 @@ func (wp *workflowProcessor) applyMoveTransformation( ) (matched bool, targetPath string, err error) { // Check if source path starts with the "from" prefix from := strings.TrimSuffix(move.From, "/") - + if sourcePath == from { // Exact match - move the file to the "to" path return true, move.To, nil } - + if strings.HasPrefix(sourcePath, from+"/") { // Path is under the "from" directory - preserve relative path relativePath := strings.TrimPrefix(sourcePath, from+"/") @@ -242,7 +243,7 @@ func (wp *workflowProcessor) applyRegexTransformation( Type: PatternTypeRegex, Pattern: regex.Pattern, } - + matchResult := wp.patternMatcher.Match(sourcePath, sourcePattern) if !matchResult.Matched { return false, "", nil @@ -260,11 +261,11 @@ func (wp *workflowProcessor) applyRegexTransformation( // extractGlobVariables extracts variables from a glob pattern match func (wp *workflowProcessor) extractGlobVariables(pattern, path string) map[string]string { variables := make(map[string]string) - + // Extract common variables // For pattern "mflix/server/**" matching "mflix/server/java-spring/src/main.java" // Extract relative_path = "java-spring/src/main.java" - + // Find the ** in the pattern starStarIdx := strings.Index(pattern, "**") if starStarIdx >= 0 { @@ -275,7 +276,7 @@ func (wp *workflowProcessor) extractGlobVariables(pattern, path string) map[stri variables["relative_path"] = relativePath } } - + return variables } @@ -406,27 +407,6 @@ func getCommitStrategyType(workflow Workflow) string { return "pull_request" // default } -func getCommitMessage(workflow Workflow) string { - if workflow.CommitStrategy != nil && workflow.CommitStrategy.CommitMessage != "" { - return workflow.CommitStrategy.CommitMessage - } - return fmt.Sprintf("Update from workflow: %s", workflow.Name) -} - -func getPRTitle(workflow Workflow) string { - if workflow.CommitStrategy != nil && workflow.CommitStrategy.PRTitle != "" { - return workflow.CommitStrategy.PRTitle - } - return getCommitMessage(workflow) -} - -func getPRBody(workflow Workflow) string { - if workflow.CommitStrategy != nil && workflow.CommitStrategy.PRBody != "" { - return workflow.CommitStrategy.PRBody - } - return "" -} - func getUsePRTemplate(workflow Workflow) bool { if workflow.CommitStrategy != nil { return workflow.CommitStrategy.UsePRTemplate diff --git a/services/workflow_processor_test.go b/services/workflow_processor_test.go new file mode 100644 index 0000000..6ea30db --- /dev/null +++ b/services/workflow_processor_test.go @@ -0,0 +1,833 @@ +package services_test + +import ( + "context" + "testing" + + "github.com/grove-platform/github-copier/services" + "github.com/grove-platform/github-copier/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// ============================================================================ +// Mock implementations +// ============================================================================ + +// mockMessageTemplater is a mock implementation of MessageTemplater +type mockMessageTemplater struct{} + +func (m *mockMessageTemplater) RenderCommitMessage(template string, ctx *types.MessageContext) string { + if template == "" { + return "Default commit message" + } + return template +} + +func (m *mockMessageTemplater) RenderPRTitle(template string, ctx *types.MessageContext) string { + if template == "" { + return "Default PR title" + } + return template +} + +func (m *mockMessageTemplater) RenderPRBody(template string, ctx *types.MessageContext) string { + if template == "" { + return "Default PR body" + } + return template +} + +// ============================================================================ +// Test helper functions +// ============================================================================ + +func createTestWorkflow(name string, transformations []types.Transformation) types.Workflow { + return types.Workflow{ + Name: name, + Source: types.Source{ + Repo: "test-org/source-repo", + Branch: "main", + }, + Destination: types.Destination{ + Repo: "test-org/dest-repo", + Branch: "main", + }, + Transformations: transformations, + } +} + +func createMoveTransformation(from, to string) types.Transformation { + return types.Transformation{ + Move: &types.MoveTransform{From: from, To: to}, + } +} + +func createCopyTransformation(from, to string) types.Transformation { + return types.Transformation{ + Copy: &types.CopyTransform{From: from, To: to}, + } +} + +func createGlobTransformation(pattern, transform string) types.Transformation { + return types.Transformation{ + Glob: &types.GlobTransform{Pattern: pattern, Transform: transform}, + } +} + +func createRegexTransformation(pattern, transform string) types.Transformation { + return types.Transformation{ + Regex: &types.RegexTransform{Pattern: pattern, Transform: transform}, + } +} + +// ============================================================================ +// Tests for Move Transformation +// ============================================================================ + +func TestWorkflowProcessor_MoveTransformation(t *testing.T) { + tests := []struct { + name string + from string + to string + sourcePath string + wantMatch bool + wantTarget string + }{ + { + name: "simple directory move", + from: "src", + to: "dest", + sourcePath: "src/file.go", + wantMatch: true, + wantTarget: "dest/file.go", + }, + { + name: "nested directory move", + from: "examples/go", + to: "code/golang", + sourcePath: "examples/go/main.go", + wantMatch: true, + wantTarget: "code/golang/main.go", + }, + { + name: "deeply nested path", + from: "src", + to: "dest", + sourcePath: "src/pkg/internal/utils/helper.go", + wantMatch: true, + wantTarget: "dest/pkg/internal/utils/helper.go", + }, + { + name: "exact file match", + from: "README.md", + to: "docs/README.md", + sourcePath: "README.md", + wantMatch: true, + wantTarget: "docs/README.md", + }, + { + name: "no match - different prefix", + from: "src", + to: "dest", + sourcePath: "other/file.go", + wantMatch: false, + wantTarget: "", + }, + { + name: "no match - partial prefix", + from: "src", + to: "dest", + sourcePath: "srcfile.go", + wantMatch: false, + wantTarget: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Create processor with real implementations for move (doesn't need mocks) + fileStateService := services.NewFileStateService() + processor := services.NewWorkflowProcessor( + services.NewPatternMatcher(), + services.NewPathTransformer(), + fileStateService, + nil, // metrics collector + &mockMessageTemplater{}, + ) + + workflow := createTestWorkflow("test-workflow", []types.Transformation{ + createMoveTransformation(tt.from, tt.to), + }) + + // Create a removed file to avoid GitHub API calls + changedFiles := []types.ChangedFile{ + {Path: tt.sourcePath, Status: "removed"}, + } + + err := processor.ProcessWorkflow(context.Background(), workflow, changedFiles, 1, "abc123") + require.NoError(t, err) + + // Check deprecation map for removed files + deprecated := fileStateService.GetFilesToDeprecate() + if tt.wantMatch { + assert.NotEmpty(t, deprecated, "expected file to be added to deprecation map") + } else { + assert.Empty(t, deprecated, "expected no files in deprecation map") + } + }) + } +} + +// ============================================================================ +// Tests for Copy Transformation +// ============================================================================ + +func TestWorkflowProcessor_CopyTransformation(t *testing.T) { + tests := []struct { + name string + from string + to string + sourcePath string + wantMatch bool + }{ + { + name: "exact file copy", + from: "config.yaml", + to: "settings/config.yaml", + sourcePath: "config.yaml", + wantMatch: true, + }, + { + name: "nested file copy", + from: "src/main.go", + to: "app/main.go", + sourcePath: "src/main.go", + wantMatch: true, + }, + { + name: "no match - different file", + from: "config.yaml", + to: "settings/config.yaml", + sourcePath: "other.yaml", + wantMatch: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fileStateService := services.NewFileStateService() + processor := services.NewWorkflowProcessor( + services.NewPatternMatcher(), + services.NewPathTransformer(), + fileStateService, + nil, + &mockMessageTemplater{}, + ) + + workflow := createTestWorkflow("test-workflow", []types.Transformation{ + createCopyTransformation(tt.from, tt.to), + }) + + changedFiles := []types.ChangedFile{ + {Path: tt.sourcePath, Status: "removed"}, + } + + err := processor.ProcessWorkflow(context.Background(), workflow, changedFiles, 1, "abc123") + require.NoError(t, err) + + deprecated := fileStateService.GetFilesToDeprecate() + if tt.wantMatch { + assert.NotEmpty(t, deprecated, "expected file to be added to deprecation map") + } else { + assert.Empty(t, deprecated, "expected no files in deprecation map") + } + }) + } +} + +// ============================================================================ +// Tests for Glob Transformation +// ============================================================================ + +func TestWorkflowProcessor_GlobTransformation(t *testing.T) { + tests := []struct { + name string + pattern string + transform string + sourcePath string + wantMatch bool + }{ + { + name: "simple glob pattern", + pattern: "src/**/*.go", + transform: "dest/${relative_path}", + sourcePath: "src/main.go", + wantMatch: true, + }, + { + name: "nested glob pattern", + pattern: "examples/**/*.js", + transform: "code/${relative_path}", + sourcePath: "examples/app/index.js", + wantMatch: true, + }, + { + name: "no match - wrong extension", + pattern: "src/**/*.go", + transform: "dest/${relative_path}", + sourcePath: "src/main.py", + wantMatch: false, + }, + { + name: "no match - wrong directory", + pattern: "src/**/*.go", + transform: "dest/${relative_path}", + sourcePath: "other/main.go", + wantMatch: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fileStateService := services.NewFileStateService() + processor := services.NewWorkflowProcessor( + services.NewPatternMatcher(), + services.NewPathTransformer(), + fileStateService, + nil, + &mockMessageTemplater{}, + ) + + workflow := createTestWorkflow("test-workflow", []types.Transformation{ + createGlobTransformation(tt.pattern, tt.transform), + }) + + changedFiles := []types.ChangedFile{ + {Path: tt.sourcePath, Status: "removed"}, + } + + err := processor.ProcessWorkflow(context.Background(), workflow, changedFiles, 1, "abc123") + require.NoError(t, err) + + deprecated := fileStateService.GetFilesToDeprecate() + if tt.wantMatch { + assert.NotEmpty(t, deprecated, "expected file to be added to deprecation map") + } else { + assert.Empty(t, deprecated, "expected no files in deprecation map") + } + }) + } +} + +// ============================================================================ +// Tests for Regex Transformation +// ============================================================================ + +func TestWorkflowProcessor_RegexTransformation(t *testing.T) { + tests := []struct { + name string + pattern string + transform string + sourcePath string + wantMatch bool + }{ + { + name: "simple regex pattern", + pattern: `^src/(?P.+)\.go$`, + transform: "dest/${file}.go", + sourcePath: "src/main.go", + wantMatch: true, + }, + { + name: "regex with multiple groups", + pattern: `^examples/(?P[^/]+)/(?P.+)$`, + transform: "code/${lang}/${file}", + sourcePath: "examples/go/main.go", + wantMatch: true, + }, + { + name: "no match - pattern doesn't match", + pattern: `^src/(?P.+)\.go$`, + transform: "dest/${file}.go", + sourcePath: "other/main.go", + wantMatch: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fileStateService := services.NewFileStateService() + processor := services.NewWorkflowProcessor( + services.NewPatternMatcher(), + services.NewPathTransformer(), + fileStateService, + nil, + &mockMessageTemplater{}, + ) + + workflow := createTestWorkflow("test-workflow", []types.Transformation{ + createRegexTransformation(tt.pattern, tt.transform), + }) + + changedFiles := []types.ChangedFile{ + {Path: tt.sourcePath, Status: "removed"}, + } + + err := processor.ProcessWorkflow(context.Background(), workflow, changedFiles, 1, "abc123") + require.NoError(t, err) + + deprecated := fileStateService.GetFilesToDeprecate() + if tt.wantMatch { + assert.NotEmpty(t, deprecated, "expected file to be added to deprecation map") + } else { + assert.Empty(t, deprecated, "expected no files in deprecation map") + } + }) + } +} + +// ============================================================================ +// Tests for Exclude Patterns +// ============================================================================ + +func TestWorkflowProcessor_ExcludePatterns(t *testing.T) { + // Note: Exclude patterns use glob matching (doublestar), not regex + tests := []struct { + name string + exclude []string + sourcePath string + wantExcluded bool + }{ + { + name: "exclude by extension glob", + exclude: []string{"**/*_test.go"}, + sourcePath: "src/main_test.go", + wantExcluded: true, + }, + { + name: "exclude by directory glob", + exclude: []string{"vendor/**"}, + sourcePath: "vendor/pkg/lib.go", + wantExcluded: true, + }, + { + name: "exclude by filename glob", + exclude: []string{"**/.DS_Store"}, + sourcePath: "src/.DS_Store", + wantExcluded: true, + }, + { + name: "not excluded - no match", + exclude: []string{"**/*_test.go"}, + sourcePath: "src/main.go", + wantExcluded: false, + }, + { + name: "multiple exclude patterns - first matches", + exclude: []string{"**/*_test.go", "vendor/**"}, + sourcePath: "src/main_test.go", + wantExcluded: true, + }, + { + name: "multiple exclude patterns - second matches", + exclude: []string{"**/*_test.go", "vendor/**"}, + sourcePath: "vendor/lib.go", + wantExcluded: true, + }, + { + name: "multiple exclude patterns - none match", + exclude: []string{"**/*_test.go", "vendor/**"}, + sourcePath: "src/main.go", + wantExcluded: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fileStateService := services.NewFileStateService() + processor := services.NewWorkflowProcessor( + services.NewPatternMatcher(), + services.NewPathTransformer(), + fileStateService, + nil, + &mockMessageTemplater{}, + ) + + workflow := types.Workflow{ + Name: "test-workflow", + Source: types.Source{ + Repo: "test-org/source-repo", + Branch: "main", + }, + Destination: types.Destination{ + Repo: "test-org/dest-repo", + Branch: "main", + }, + Transformations: []types.Transformation{ + createMoveTransformation("src", "dest"), + createMoveTransformation("vendor", "vendor"), + }, + Exclude: tt.exclude, + } + + changedFiles := []types.ChangedFile{ + {Path: tt.sourcePath, Status: "removed"}, + } + + err := processor.ProcessWorkflow(context.Background(), workflow, changedFiles, 1, "abc123") + require.NoError(t, err) + + deprecated := fileStateService.GetFilesToDeprecate() + if tt.wantExcluded { + assert.Empty(t, deprecated, "expected file to be excluded") + } else { + // Only check if the path matches a transformation + if tt.sourcePath == "src/main.go" { + assert.NotEmpty(t, deprecated, "expected file to be processed") + } + } + }) + } +} + +// ============================================================================ +// Tests for Multiple Transformations +// ============================================================================ + +func TestWorkflowProcessor_MultipleTransformations(t *testing.T) { + fileStateService := services.NewFileStateService() + processor := services.NewWorkflowProcessor( + services.NewPatternMatcher(), + services.NewPathTransformer(), + fileStateService, + nil, + &mockMessageTemplater{}, + ) + + workflow := types.Workflow{ + Name: "multi-transform-workflow", + Source: types.Source{ + Repo: "test-org/source-repo", + Branch: "main", + }, + Destination: types.Destination{ + Repo: "test-org/dest-repo", + Branch: "main", + }, + Transformations: []types.Transformation{ + createMoveTransformation("src", "code"), + createMoveTransformation("docs", "documentation"), + createCopyTransformation("README.md", "docs/README.md"), + }, + } + + changedFiles := []types.ChangedFile{ + {Path: "src/main.go", Status: "removed"}, + {Path: "docs/guide.md", Status: "removed"}, + {Path: "README.md", Status: "removed"}, + {Path: "other/file.txt", Status: "removed"}, // Should not match + } + + err := processor.ProcessWorkflow(context.Background(), workflow, changedFiles, 1, "abc123") + require.NoError(t, err) + + deprecated := fileStateService.GetFilesToDeprecate() + assert.NotEmpty(t, deprecated, "expected files to be processed") + + // Verify all 3 matching files are accumulated in the deprecation map + entries, exists := deprecated["deprecated_examples.json"] + assert.True(t, exists, "expected deprecation entry to exist") + require.Len(t, entries, 3, "expected 3 files to be accumulated (src/main.go, docs/guide.md, README.md)") + + // Collect all file names + fileNames := make([]string, len(entries)) + for i, e := range entries { + fileNames[i] = e.FileName + } + + // Verify all expected files are present + assert.Contains(t, fileNames, "code/main.go", "expected src/main.go -> code/main.go") + assert.Contains(t, fileNames, "documentation/guide.md", "expected docs/guide.md -> documentation/guide.md") + assert.Contains(t, fileNames, "docs/README.md", "expected README.md -> docs/README.md") +} + +// ============================================================================ +// Tests for Edge Cases +// ============================================================================ + +func TestWorkflowProcessor_EmptyChangedFiles(t *testing.T) { + fileStateService := services.NewFileStateService() + processor := services.NewWorkflowProcessor( + services.NewPatternMatcher(), + services.NewPathTransformer(), + fileStateService, + nil, + &mockMessageTemplater{}, + ) + + workflow := createTestWorkflow("test-workflow", []types.Transformation{ + createMoveTransformation("src", "dest"), + }) + + changedFiles := []types.ChangedFile{} + + err := processor.ProcessWorkflow(context.Background(), workflow, changedFiles, 1, "abc123") + require.NoError(t, err) + + deprecated := fileStateService.GetFilesToDeprecate() + assert.Empty(t, deprecated, "expected no files to be processed") +} + +func TestWorkflowProcessor_NoTransformations(t *testing.T) { + fileStateService := services.NewFileStateService() + processor := services.NewWorkflowProcessor( + services.NewPatternMatcher(), + services.NewPathTransformer(), + fileStateService, + nil, + &mockMessageTemplater{}, + ) + + workflow := createTestWorkflow("test-workflow", []types.Transformation{}) + + changedFiles := []types.ChangedFile{ + {Path: "src/main.go", Status: "removed"}, + } + + err := processor.ProcessWorkflow(context.Background(), workflow, changedFiles, 1, "abc123") + require.NoError(t, err) + + deprecated := fileStateService.GetFilesToDeprecate() + assert.Empty(t, deprecated, "expected no files to be processed with no transformations") +} + +// ============================================================================ +// Tests for Invalid Patterns +// ============================================================================ + +func TestWorkflowProcessor_InvalidExcludePattern(t *testing.T) { + fileStateService := services.NewFileStateService() + processor := services.NewWorkflowProcessor( + services.NewPatternMatcher(), + services.NewPathTransformer(), + fileStateService, + nil, + &mockMessageTemplater{}, + ) + + workflow := types.Workflow{ + Name: "test-workflow", + Source: types.Source{ + Repo: "test-org/source-repo", + Branch: "main", + }, + Destination: types.Destination{ + Repo: "test-org/dest-repo", + Branch: "main", + }, + Transformations: []types.Transformation{ + createMoveTransformation("src", "dest"), + }, + // Invalid glob pattern - should be handled gracefully + Exclude: []string{"[invalid"}, + } + + changedFiles := []types.ChangedFile{ + {Path: "src/main.go", Status: "removed"}, + } + + // Should not error, just log warning and continue + err := processor.ProcessWorkflow(context.Background(), workflow, changedFiles, 1, "abc123") + require.NoError(t, err) + + // File should still be processed since invalid pattern is skipped + deprecated := fileStateService.GetFilesToDeprecate() + assert.NotEmpty(t, deprecated, "expected file to be processed despite invalid exclude pattern") +} + +// ============================================================================ +// Tests for Deprecation Config +// ============================================================================ + +func TestWorkflowProcessor_CustomDeprecationFile(t *testing.T) { + fileStateService := services.NewFileStateService() + processor := services.NewWorkflowProcessor( + services.NewPatternMatcher(), + services.NewPathTransformer(), + fileStateService, + nil, + &mockMessageTemplater{}, + ) + + workflow := types.Workflow{ + Name: "test-workflow", + Source: types.Source{ + Repo: "test-org/source-repo", + Branch: "main", + }, + Destination: types.Destination{ + Repo: "test-org/dest-repo", + Branch: "main", + }, + Transformations: []types.Transformation{ + createMoveTransformation("src", "dest"), + }, + DeprecationCheck: &types.DeprecationConfig{ + File: "custom_deprecation.json", + }, + } + + changedFiles := []types.ChangedFile{ + {Path: "src/main.go", Status: "removed"}, + } + + err := processor.ProcessWorkflow(context.Background(), workflow, changedFiles, 1, "abc123") + require.NoError(t, err) + + deprecated := fileStateService.GetFilesToDeprecate() + entries, exists := deprecated["custom_deprecation.json"] + assert.True(t, exists, "expected custom deprecation file to be used") + require.Len(t, entries, 1, "expected one entry in custom deprecation file") + assert.Equal(t, "dest/main.go", entries[0].FileName) +} + +// ============================================================================ +// Tests for File Status Handling +// ============================================================================ + +func TestWorkflowProcessor_FileStatusHandling(t *testing.T) { + tests := []struct { + name string + status string + expectDeprecated bool + }{ + { + name: "removed file goes to deprecation", + status: "removed", + expectDeprecated: true, + }, + { + name: "DELETED file goes to deprecation (GraphQL API format)", + status: "DELETED", + expectDeprecated: true, + }, + { + name: "added file does not go to deprecation", + status: "added", + expectDeprecated: false, + }, + { + name: "modified file does not go to deprecation", + status: "modified", + expectDeprecated: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fileStateService := services.NewFileStateService() + processor := services.NewWorkflowProcessor( + services.NewPatternMatcher(), + services.NewPathTransformer(), + fileStateService, + nil, + &mockMessageTemplater{}, + ) + + workflow := createTestWorkflow("test-workflow", []types.Transformation{ + createMoveTransformation("src", "dest"), + }) + + changedFiles := []types.ChangedFile{ + {Path: "src/main.go", Status: tt.status}, + } + + // Note: For non-removed files, this will try to call GitHub API + // which will fail, but we're testing the deprecation path + _ = processor.ProcessWorkflow(context.Background(), workflow, changedFiles, 1, "abc123") + + deprecated := fileStateService.GetFilesToDeprecate() + if tt.expectDeprecated { + assert.NotEmpty(t, deprecated, "expected file to be in deprecation map") + } else { + // For non-removed files, they go to upload queue (which fails without GitHub) + // but should NOT be in deprecation map + assert.Empty(t, deprecated, "expected file NOT to be in deprecation map") + } + }) + } +} + +// ============================================================================ +// Tests for Path Transformation Edge Cases +// ============================================================================ + +func TestWorkflowProcessor_PathTransformationEdgeCases(t *testing.T) { + tests := []struct { + name string + transform types.Transformation + sourcePath string + wantMatch bool + }{ + { + name: "move with trailing slash in from", + transform: createMoveTransformation("src/", "dest/"), + sourcePath: "src/main.go", + wantMatch: true, + }, + { + name: "move empty from does not match root file", + transform: createMoveTransformation("", "dest"), + sourcePath: "main.go", + wantMatch: false, // Empty prefix doesn't match in current implementation + }, + { + name: "glob single star does not match nested", + transform: createGlobTransformation("src/*.go", "dest/${relative_path}"), + sourcePath: "src/pkg/main.go", + wantMatch: false, + }, + { + name: "copy exact file match", + transform: createCopyTransformation("README.md", "docs/README.md"), + sourcePath: "README.md", + wantMatch: true, + }, + { + name: "copy does not match different file", + transform: createCopyTransformation("README.md", "docs/README.md"), + sourcePath: "CHANGELOG.md", + wantMatch: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fileStateService := services.NewFileStateService() + processor := services.NewWorkflowProcessor( + services.NewPatternMatcher(), + services.NewPathTransformer(), + fileStateService, + nil, + &mockMessageTemplater{}, + ) + + workflow := createTestWorkflow("test-workflow", []types.Transformation{tt.transform}) + + changedFiles := []types.ChangedFile{ + {Path: tt.sourcePath, Status: "removed"}, + } + + err := processor.ProcessWorkflow(context.Background(), workflow, changedFiles, 1, "abc123") + require.NoError(t, err) + + deprecated := fileStateService.GetFilesToDeprecate() + if tt.wantMatch { + assert.NotEmpty(t, deprecated, "expected file to match transformation") + } else { + assert.Empty(t, deprecated, "expected file NOT to match transformation") + } + }) + } +} diff --git a/testdata/.env.test b/testdata/.env.test new file mode 100644 index 0000000..40a9066 --- /dev/null +++ b/testdata/.env.test @@ -0,0 +1,59 @@ +# Test environment configuration +# Usage: cp testdata/.env.test .env.test && edit with your values + +# ============================================================================= +# REQUIRED - Fill these in +# ============================================================================= + +GITHUB_APP_ID=your-app-id +INSTALLATION_ID=your-installation-id + +# Config repo points to your test source repo +CONFIG_REPO_OWNER=cbullinger +CONFIG_REPO_NAME=copier-app-source-test +CONFIG_REPO_BRANCH=main + +# Main config file path in the source repo +MAIN_CONFIG_FILE=.copier/main.yaml +USE_MAIN_CONFIG=true + +# ============================================================================= +# SECRETS - Choose one method +# ============================================================================= + +# Option 1: Skip Secret Manager (easiest for local testing) +SKIP_SECRET_MANAGER=true +# Then set the private key directly (base64 encoded): +# GITHUB_APP_PRIVATE_KEY_B64=your-base64-encoded-pem + +# Option 2: Use Secret Manager (if you have GCP access) +# PEM_NAME=projects/YOUR_PROJECT/secrets/YOUR_SECRET/versions/latest + +# Webhook secret (must match what you set in GitHub) +WEBHOOK_SECRET=your-webhook-secret + +# ============================================================================= +# TEST SETTINGS +# ============================================================================= + +# Dry run - process webhooks but don't make actual commits/PRs +DRY_RUN=false + +# Disable cloud logging for local testing +COPIER_DISABLE_CLOUD_LOGGING=true + +# Debug logging +LOG_LEVEL=debug +COPIER_DEBUG=true + +# Disable audit logging (no MongoDB needed) +AUDIT_ENABLED=false + +# Server +PORT=8080 +WEBSERVER_PATH=/webhook + +# Committer info for test PRs +COMMITTER_NAME=Test Copier Bot +COMMITTER_EMAIL=test-bot@example.com + diff --git a/test-payloads/README.md b/testdata/README.md similarity index 60% rename from test-payloads/README.md rename to testdata/README.md index 2f9e474..8497088 100644 --- a/test-payloads/README.md +++ b/testdata/README.md @@ -1,6 +1,56 @@ # Test Payloads -This directory contains example webhook payloads for testing the examples-copier application. +Test files for local integration testing of the github-copier app. + +## Quick Start (Isolated Testing) + +```bash +# 1. Copy and configure test environment +cp testdata/.env.test .env.test +# Edit .env.test with your GitHub App credentials + +# 2. Copy config to your test source repo (cbullinger/copier-app-source-test) +# Upload testdata/source-repo-files/.copier/main.yaml to .copier/main.yaml + +# 3. Start the app with test config +ENV_FILE=.env.test go run app.go + +# 4. In another terminal, start webhook tunnel +smee --url https://smee.io/YOUR_CHANNEL --target http://localhost:8080/webhook + +# 5. Create a PR in your test source repo and merge it +``` + +## Directory Structure + +``` +testdata/ +├── .env.test # Test environment variables (copy to .env.test) +├── source-repo-files/ # Files to copy to your test source repo +│ └── .copier/ +│ └── main.yaml # Workflow config for test repos +├── test-pr-merged.json # Sample webhook payload +├── test-config.yaml # Example config (reference only) +└── example-pr-merged.json # Generic example payload +``` + +## Test Repositories + +| Repo | Purpose | +|------|---------| +| `cbullinger/copier-app-source-test` | Source repo (receives PRs, triggers webhooks) | +| `cbullinger/copier-app-dest-1` | Destination for Go examples | +| `cbullinger/copier-app-dest-2` | Destination for Python examples | + +## Configured Workflows + +The test config (`source-repo-files/.copier/main.yaml`) defines: + +| Workflow | Source Pattern | Destination | Transform | +|----------|---------------|-------------|-----------| +| `test-go-to-dest1` | `examples/go/**` | `copier-app-dest-1` | `examples/go/` → `go-examples/` | +| `test-python-to-dest2` | `examples/python/**` | `copier-app-dest-2` | `examples/python/` → `python-examples/` | +| `test-docs-to-dest1` | `docs/**` | `copier-app-dest-1` | `docs/` → `documentation/` | ## Files @@ -20,7 +70,7 @@ A complete example of a merged PR webhook payload with: go build -o test-webhook ./cmd/test-webhook # Send example payload -./test-webhook -payload test-payloads/example-pr-merged.json +./test-webhook -payload testdata/example-pr-merged.json ``` ### Option 2: Fetch Real PR Data @@ -124,10 +174,10 @@ Test without making actual commits: ```bash # Start app in dry-run mode -DRY_RUN=true ./examples-copier & +DRY_RUN=true ./github-copier & # Send test webhook -./test-webhook -payload test-payloads/example-pr-merged.json +./test-webhook -payload testdata/example-pr-merged.json # Check logs for pattern matching and transformations ``` @@ -164,7 +214,7 @@ After sending a test webhook: ### Test Case 1: New Go Examples ```bash -./test-webhook -payload test-payloads/example-pr-merged.json +./test-webhook -payload testdata/example-pr-merged.json ``` Expected: Files copied to target repo with transformed paths @@ -177,8 +227,8 @@ Expected: Real PR data fetched and processed ### Test Case 3: Dry-Run Validation ```bash -DRY_RUN=true ./examples-copier & -./test-webhook -payload test-payloads/example-pr-merged.json +DRY_RUN=true ./github-copier & +./test-webhook -payload testdata/example-pr-merged.json ``` Expected: Processing logged but no commits made diff --git a/test-payloads/example-pr-merged.json b/testdata/example-pr-merged.json similarity index 100% rename from test-payloads/example-pr-merged.json rename to testdata/example-pr-merged.json diff --git a/testdata/source-repo-files/.copier/main.yaml b/testdata/source-repo-files/.copier/main.yaml new file mode 100644 index 0000000..fab2660 --- /dev/null +++ b/testdata/source-repo-files/.copier/main.yaml @@ -0,0 +1,61 @@ +# Main config for test source repo: cbullinger/copier-app-source-test +# Copy this file to .copier/main.yaml in your source repo + +defaults: + commit_strategy: + type: "pull_request" + auto_merge: false + +workflow_configs: + - source: "inline" + workflows: + # Workflow 1: Copy Go files to dest-1 + - name: "test-go-to-dest1" + source: + patterns: + - "examples/go/**" + destination: + repo: "cbullinger/copier-app-dest-1" + branch: "main" + transformations: + - move: + from: "examples/go/" + to: "go-examples/" + commit_strategy: + type: "pull_request" + pr_title: "[Test] Sync Go examples" + auto_merge: false + + # Workflow 2: Copy Python files to dest-2 + - name: "test-python-to-dest2" + source: + patterns: + - "examples/python/**" + destination: + repo: "cbullinger/copier-app-dest-2" + branch: "main" + transformations: + - move: + from: "examples/python/" + to: "python-examples/" + commit_strategy: + type: "pull_request" + pr_title: "[Test] Sync Python examples" + auto_merge: false + + # Workflow 3: Copy shared docs to both destinations + - name: "test-docs-to-dest1" + source: + patterns: + - "docs/**" + destination: + repo: "cbullinger/copier-app-dest-1" + branch: "main" + transformations: + - move: + from: "docs/" + to: "documentation/" + commit_strategy: + type: "direct" + message: "Sync docs from source" + diff --git a/testdata/test-config.yaml b/testdata/test-config.yaml new file mode 100644 index 0000000..181c191 --- /dev/null +++ b/testdata/test-config.yaml @@ -0,0 +1,66 @@ +# Test configuration for local integration testing +# Source: cbullinger/copier-app-source-test +# Destinations: cbullinger/copier-app-dest-1, cbullinger/copier-app-dest-2 + +workflows: + # Workflow 1: Copy Go files to dest-1 + - name: "test-go-to-dest1" + source: + repo: "cbullinger/copier-app-source-test" + branch: "main" + patterns: + - type: glob + pattern: "examples/go/**" + destination: + repo: "cbullinger/copier-app-dest-1" + branch: "main" + transformations: + - type: move + from: "examples/go/" + to: "go-examples/" + commit: + strategy: pr + message: "Sync Go examples from source" + pr_title: "[Test] Sync Go examples" + auto_merge: false + + # Workflow 2: Copy Python files to dest-2 + - name: "test-python-to-dest2" + source: + repo: "cbullinger/copier-app-source-test" + branch: "main" + patterns: + - type: glob + pattern: "examples/python/**" + destination: + repo: "cbullinger/copier-app-dest-2" + branch: "main" + transformations: + - type: move + from: "examples/python/" + to: "python-examples/" + commit: + strategy: pr + message: "Sync Python examples from source" + pr_title: "[Test] Sync Python examples" + auto_merge: false + + # Workflow 3: Copy docs to both destinations + - name: "test-docs-to-dest1" + source: + repo: "cbullinger/copier-app-source-test" + branch: "main" + patterns: + - type: glob + pattern: "docs/**" + destination: + repo: "cbullinger/copier-app-dest-1" + branch: "main" + transformations: + - type: move + from: "docs/" + to: "documentation/" + commit: + strategy: direct + message: "Sync docs from source" + diff --git a/testdata/test-pr-merged.json b/testdata/test-pr-merged.json new file mode 100644 index 0000000..d8a799b --- /dev/null +++ b/testdata/test-pr-merged.json @@ -0,0 +1,67 @@ +{ + "action": "closed", + "number": 1, + "pull_request": { + "number": 1, + "state": "closed", + "merged": true, + "merge_commit_sha": "test123abc456def", + "title": "Test PR for integration testing", + "head": { + "ref": "test-branch", + "sha": "abc123test", + "repo": { + "name": "copier-app-source-test", + "full_name": "cbullinger/copier-app-source-test" + } + }, + "base": { + "ref": "main", + "repo": { + "name": "copier-app-source-test", + "full_name": "cbullinger/copier-app-source-test" + } + } + }, + "repository": { + "name": "copier-app-source-test", + "full_name": "cbullinger/copier-app-source-test", + "owner": { + "login": "cbullinger" + } + }, + "installation": { + "id": 12345678 + }, + "files": [ + { + "filename": "examples/go/main.go", + "status": "added", + "additions": 20, + "deletions": 0, + "changes": 20 + }, + { + "filename": "examples/go/utils/helper.go", + "status": "added", + "additions": 15, + "deletions": 0, + "changes": 15 + }, + { + "filename": "examples/python/app.py", + "status": "added", + "additions": 30, + "deletions": 0, + "changes": 30 + }, + { + "filename": "docs/README.md", + "status": "modified", + "additions": 5, + "deletions": 2, + "changes": 7 + } + ] +} + diff --git a/tests/utils.go b/tests/utils.go index ddf2b33..4bc2ff2 100644 --- a/tests/utils.go +++ b/tests/utils.go @@ -10,9 +10,9 @@ import ( "github.com/jarcoal/httpmock" - "github.com/mongodb/code-example-tooling/code-copier/configs" - "github.com/mongodb/code-example-tooling/code-copier/services" - "github.com/mongodb/code-example-tooling/code-copier/types" + "github.com/grove-platform/github-copier/configs" + "github.com/grove-platform/github-copier/services" + "github.com/grove-platform/github-copier/types" ) //