From 5abcce9031a81e3c6ccfe5f07b0e138cd240fe7d Mon Sep 17 00:00:00 2001 From: Francois Botha Date: Mon, 16 Feb 2026 20:58:20 +0200 Subject: [PATCH 1/5] Upgrade to Go 1.25.6 --- .github/workflows/ci.yml | 6 +++--- go.mod | 4 +--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7dac12c..68c8221 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,7 @@ jobs: fetch-depth: '0' - uses: actions/setup-go@v5 with: - go-version: "^1.20" + go-version: "^1.25" - name: golangci-lint uses: golangci/golangci-lint-action@v8 with: @@ -26,7 +26,7 @@ jobs: fetch-depth: '0' - uses: actions/setup-go@v5 with: - go-version: "^1.20" + go-version: "^1.25" - name: run tests on cmd run: go test ./cmd - name: run tests on pkg @@ -86,7 +86,7 @@ jobs: ## setup go - uses: actions/setup-go@v5 with: - go-version: "^1.22" + go-version: "^1.25" ## above is fine to get latest for now, also save a copy with short sha - id: vars run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT diff --git a/go.mod b/go.mod index 1bfeb3f..51ce192 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,6 @@ module github.com/bart6114/cheek -go 1.22 - -toolchain go1.22.4 +go 1.25.6 require github.com/stretchr/testify v1.10.0 From b010a721ec98046dd436bfd3a280f5371c569b8c Mon Sep 17 00:00:00 2001 From: Francois Botha Date: Mon, 16 Feb 2026 21:02:21 +0200 Subject: [PATCH 2/5] Upgrade golangci-lint to v2.9.0 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 68c8221..0f1423a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,7 +16,7 @@ jobs: - name: golangci-lint uses: golangci/golangci-lint-action@v8 with: - version: v2.1 + version: v2.9.0 tests: runs-on: ubuntu-latest From 29af3916efc62eb04e86c440a87065c6df7ad3d8 Mon Sep 17 00:00:00 2001 From: Francois Botha Date: Wed, 4 Feb 2026 13:09:38 +0200 Subject: [PATCH 3/5] Add log retention period on jobs --- go.mod | 3 +- go.sum | 6 +- pkg/db.go | 6 + pkg/job.go | 33 ++++- pkg/job_test.go | 234 +++++++++++++++++++++++++++++++++-- pkg/schedule.go | 30 +++++ testdata/readme_example.yaml | 1 + 7 files changed, 293 insertions(+), 20 deletions(-) diff --git a/go.mod b/go.mod index 51ce192..cc5140d 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( require ( github.com/jmoiron/sqlx v1.4.0 github.com/julienschmidt/httprouter v1.3.0 + github.com/pawelszydlo/humanize v0.0.0-20260212182641-ba25cf34090b github.com/spf13/cobra v1.9.1 github.com/spf13/viper v1.20.1 ) @@ -47,6 +48,6 @@ require ( github.com/subosito/gotenv v1.6.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/sys v0.29.0 // indirect - golang.org/x/text v0.21.0 // indirect + golang.org/x/text v0.34.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect ) diff --git a/go.sum b/go.sum index c09abf8..144bda5 100644 --- a/go.sum +++ b/go.sum @@ -49,6 +49,8 @@ github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/mattn/go-sqlite3 v1.14.28 h1:ThEiQrnbtumT+QMknw63Befp/ce/nUPgBPMlRFEum7A= github.com/mattn/go-sqlite3 v1.14.28/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/pawelszydlo/humanize v0.0.0-20260212182641-ba25cf34090b h1:jtxaEjubJjJkWdYZYsCrrbp39n6PFRPf94agT3nfqy8= +github.com/pawelszydlo/humanize v0.0.0-20260212182641-ba25cf34090b/go.mod h1:rx2jFGgY1C4LL7MSlavPTNRbUTgXHIXgNpGKuPHqaCM= github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -87,8 +89,8 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= -golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= +golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/pkg/db.go b/pkg/db.go index e39364f..6d9a6be 100644 --- a/pkg/db.go +++ b/pkg/db.go @@ -37,6 +37,12 @@ func InitDB(db *sqlx.DB) error { return fmt.Errorf("create log table: %w", err) } + // Create index for efficient log retention cleanup queries + _, err = db.Exec(`CREATE INDEX IF NOT EXISTS idx_log_job_triggered_at ON log (job, triggered_at)`) + if err != nil { + return fmt.Errorf("create log index: %w", err) + } + // Perform cleanup to remove old, non-conforming records _, err = db.Exec(` DELETE FROM log diff --git a/pkg/job.go b/pkg/job.go index 954cf03..0964097 100644 --- a/pkg/job.go +++ b/pkg/job.go @@ -37,12 +37,14 @@ type JobSpec struct { Cron string `yaml:"cron,omitempty" json:"cron,omitempty"` Command stringArray `yaml:"command" json:"command"` - OnSuccess OnEvent `yaml:"on_success,omitempty" json:"on_success,omitempty"` - OnError OnEvent `yaml:"on_error,omitempty" json:"on_error,omitempty"` + OnSuccess OnEvent `yaml:"on_success,omitempty" json:"on_success,omitempty"` + OnError OnEvent `yaml:"on_error,omitempty" json:"on_error,omitempty"` OnRetriesExhausted OnEvent `yaml:"on_retries_exhausted,omitempty" json:"on_retries_exhausted,omitempty"` Name string `json:"name"` Retries int `yaml:"retries,omitempty" json:"retries,omitempty"` + LogRetentionPeriod string `yaml:"log_retention_period,omitempty" json:"log_retention_period,omitempty"` + logRetentionDuration time.Duration `yaml:"-"` Env map[string]secret `yaml:"env,omitempty"` WorkingDirectory string `yaml:"working_directory,omitempty" json:"working_directory,omitempty"` DisableConcurrentExecution bool `yaml:"disable_concurrent_execution,omitempty" json:"disable_concurrent_execution,omitempty"` @@ -64,8 +66,8 @@ func (secret) MarshalText() ([]byte, error) { // JobRun holds information about a job execution. type JobRun struct { - LogEntryId int `json:"id,omitempty" db:"id"` - Status *int `json:"status,omitempty" db:"status,omitempty"` + LogEntryId int `json:"id,omitempty" db:"id"` + Status *int `json:"status,omitempty" db:"status,omitempty"` logBuf bytes.Buffer Log string `json:"log" db:"message"` Name string `json:"name" db:"job"` @@ -135,6 +137,8 @@ func (j *JobSpec) finalize(jr *JobRun) { if j.cfg.DB == nil { j.Runs = append(j.Runs, *jr) } + // cleanup old log entries if configured + j.cleanupOldLogs() // launch on_events j.OnEvent(jr) } @@ -208,7 +212,6 @@ func (j *JobSpec) execCommandWithRetry(ctx context.Context, trigger string, pare return jr } - func (j *JobSpec) now() time.Time { // defer for if schedule doesn't exist, allows for easy testing if j.globalSchedule != nil { @@ -319,7 +322,6 @@ func (j *JobSpec) execCommand(ctx context.Context, jr JobRun, trigger string) Jo return jr } - func (j *JobSpec) loadLogFromDb(id int) (JobRun, error) { var jr JobRun if j.cfg.DB == nil { @@ -372,6 +374,25 @@ func (j *JobSpec) loadRunsFromDb(nruns int, includeLogs bool) { j.Runs = jrs } +func (j *JobSpec) cleanupOldLogs() { + if j.logRetentionDuration <= 0 || j.cfg.DB == nil { + return + } + + cutoffTime := time.Now().Add(-j.logRetentionDuration) + + result, err := j.cfg.DB.Exec("DELETE FROM log WHERE job = ? AND triggered_at < ?", j.Name, cutoffTime) + if err != nil { + j.log.Warn().Str("job", j.Name).Err(err).Msg("Failed to cleanup old logs") + return + } + + rowsAffected, err := result.RowsAffected() + if err == nil && rowsAffected > 0 { + j.log.Info().Str("job", j.Name).Int64("rows_deleted", rowsAffected).Msg("Cleaned up old log entries") + } +} + func (j *JobSpec) setNextTick(refTime time.Time, includeRefTime bool) error { if j.Cron != "" { t, err := gronx.NextTickAfter(j.Cron, refTime, includeRefTime) diff --git a/pkg/job_test.go b/pkg/job_test.go index 88d1728..6b9a79e 100644 --- a/pkg/job_test.go +++ b/pkg/job_test.go @@ -85,7 +85,7 @@ func TestSpecialCron(t *testing.T) { t.Fatal(err) } - jobRun := JobRun{} // Create a JobRun instance + jobRun := JobRun{} // Create a JobRun instance jr := j.execCommand(context.Background(), jobRun, "test") // Pass JobRun instance and "test" assert.Equal(t, *jr.Status, 0) } @@ -134,7 +134,7 @@ env: t.Fatal("should contain foo") } - jobRun := JobRun{} // Create a JobRun instance + jobRun := JobRun{} // Create a JobRun instance jr := j.execCommand(context.Background(), jobRun, "test") // Pass JobRun instance and "test" jr.flushLogBuffer() @@ -154,7 +154,7 @@ func TestStdErrOut(t *testing.T) { cfg: cfg, } - jobRun := JobRun{} // Create a JobRun instance + jobRun := JobRun{} // Create a JobRun instance jr := j.execCommand(context.Background(), jobRun, "test") // Pass JobRun instance and "test" jr.flushLogBuffer() assert.Contains(t, jr.Log, "stdout") @@ -173,7 +173,7 @@ func TestFailingLog(t *testing.T) { cfg: cfg, } - jobRun := JobRun{} // Create a JobRun instance + jobRun := JobRun{} // Create a JobRun instance jr := j.execCommand(context.Background(), jobRun, "test") // Pass JobRun instance and "test" jr.flushLogBuffer() assert.Contains(t, jr.Log, "this fails") @@ -186,7 +186,7 @@ func TestJobRunNoCommand(t *testing.T) { cfg: NewConfig(), } - jobRun := JobRun{} // Create a JobRun instance + jobRun := JobRun{} // Create a JobRun instance jr := j.execCommand(context.Background(), jobRun, "test") // Pass JobRun instance and "test" assert.NotEqual(t, jr.Status, 0) } @@ -201,7 +201,7 @@ func TestJobNonZero(t *testing.T) { cfg: NewConfig(), } - jobRun := JobRun{} // Create a JobRun instance + jobRun := JobRun{} // Create a JobRun instance jr := j.execCommand(context.Background(), jobRun, "test") // Pass JobRun instance and "test" assert.NotEqual(t, jr.Status, 0) } @@ -246,7 +246,7 @@ func TestOnEventWebhook(t *testing.T) { NotifyWebhook: []string{testServer.URL}, }, } - jobRun := JobRun{} // Create a JobRun instance + jobRun := JobRun{} // Create a JobRun instance jr := j.execCommand(context.Background(), jobRun, "test") // Pass JobRun instance and "test" j.OnEvent(&jr) } @@ -277,7 +277,7 @@ func TestStringArray(t *testing.T) { } j.cfg = NewConfig() - jobRun := JobRun{} // Create a JobRun instance + jobRun := JobRun{} // Create a JobRun instance jr := j.execCommand(context.Background(), jobRun, "test") // Pass JobRun instance and "test" jr.flushLogBuffer() @@ -477,7 +477,7 @@ func TestRetryContextInWebhooks(t *testing.T) { // Check the final payload (should be the retries exhausted one) finalPayload := webhookPayloads[len(webhookPayloads)-1] - + // Verify retry context fields are present assert.Equal(t, float64(1), finalPayload["retry_attempt"], "Final retry attempt should be 1") assert.Equal(t, true, finalPayload["retries_exhausted"], "retries_exhausted should be true") @@ -524,7 +524,7 @@ func TestJobWithBashEval(t *testing.T) { j.log = log j.cfg = cfg - jobRun := JobRun{} // Create a JobRun instance + jobRun := JobRun{} // Create a JobRun instance jr := j.execCommand(context.Background(), jobRun, "test") // Pass JobRun instance and "test" jr.flushLogBuffer() @@ -653,7 +653,7 @@ func TestTriggeredByJobRunContext(t *testing.T) { // Create a child job that will be triggered by the parent childJob := &JobSpec{ - Name: "child-job", + Name: "child-job", Command: []string{"echo", "child output"}, cfg: NewConfig(), log: NewLogger("debug", nil, os.Stdout, os.Stdout), @@ -703,3 +703,215 @@ func TestTriggeredByJobRunContext(t *testing.T) { assert.Equal(t, float64(0), parentContext["status"], "Parent job should have succeeded") assert.Contains(t, parentContext["log"], "parent output", "Parent job log should contain expected output") } + +func TestLogRetentionPeriodParsing(t *testing.T) { + tests := []struct { + name string + durationStr string + shouldError bool + expectedErr string + }{ + { + name: "valid duration - hours", + durationStr: "24 hours", + shouldError: false, + }, + { + name: "valid duration - days", + durationStr: "7 days", + shouldError: false, + }, + { + name: "valid duration - weeks", + durationStr: "2 weeks", + shouldError: false, + }, + { + name: "valid duration - months", + durationStr: "3 months", + shouldError: false, + }, + { + name: "valid duration - minutes", + durationStr: "90 minutes", + shouldError: false, + }, + { + name: "valid duration - complex", + durationStr: "1 hour and 30 minutes", + shouldError: false, + }, + { + name: "invalid duration", + durationStr: "invalid", + shouldError: true, + expectedErr: "invalid log_retention_period", + }, + { + name: "zero duration", + durationStr: "0 seconds", + shouldError: true, + expectedErr: "must be positive", + }, + { + name: "negative duration fails", + durationStr: "-1 hour", + shouldError: true, + expectedErr: "must be positive", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + yamlContent := fmt.Sprintf(` +jobs: + test-job: + command: echo "test" + cron: "* * * * *" + log_retention_period: %s +`, tc.durationStr) + + // Write to temp file + tmpfile, err := os.CreateTemp("", "test-*.yaml") + if err != nil { + t.Fatal(err) + } + defer func() { _ = os.Remove(tmpfile.Name()) }() + + if _, err := tmpfile.Write([]byte(yamlContent)); err != nil { + t.Fatal(err) + } + if err := tmpfile.Close(); err != nil { + t.Fatal(err) + } + + // Try to load schedule + log := NewLogger("debug", nil, os.Stdout) + cfg := NewConfig() + _, err = loadSchedule(log, cfg, tmpfile.Name()) + + if tc.shouldError { + assert.Error(t, err) + if tc.expectedErr != "" { + assert.Contains(t, err.Error(), tc.expectedErr) + } + } else { + assert.NoError(t, err) + } + }) + } +} + +func TestLogRetentionIntegration(t *testing.T) { + // Create test database + db, err := OpenDB(":memory:") + if err != nil { + t.Fatal(err) + } + defer func() { _ = db.Close() }() + + cfg := NewConfig() + cfg.DB = db + + // Create a job with log retention + j := &JobSpec{ + Name: "test-retention", + Command: []string{"echo", "test"}, + logRetentionDuration: 1 * time.Hour, // Retain logs for 1 hour + cfg: cfg, + log: NewLogger("debug", nil), + } + + // Insert some old log entries (2 hours ago) + oldTime := time.Now().Add(-2 * time.Hour) + _, err = db.Exec(` + INSERT INTO log (job, triggered_at, triggered_by, duration, status, message) + VALUES (?, ?, ?, ?, ?, ?) + `, j.Name, oldTime, "test", 100, 0, "old log entry 1") + assert.NoError(t, err) + + _, err = db.Exec(` + INSERT INTO log (job, triggered_at, triggered_by, duration, status, message) + VALUES (?, ?, ?, ?, ?, ?) + `, j.Name, oldTime.Add(-30*time.Minute), "test", 200, 0, "old log entry 2") + assert.NoError(t, err) + + // Insert a recent log entry (30 minutes ago) + recentTime := time.Now().Add(-30 * time.Minute) + _, err = db.Exec(` + INSERT INTO log (job, triggered_at, triggered_by, duration, status, message) + VALUES (?, ?, ?, ?, ?, ?) + `, j.Name, recentTime, "test", 150, 0, "recent log entry") + assert.NoError(t, err) + + // Check initial count + var initialCount int + err = db.Get(&initialCount, "SELECT COUNT(*) FROM log WHERE job = ?", j.Name) + assert.NoError(t, err) + assert.Equal(t, 3, initialCount) + + // Run log cleanup + j.cleanupOldLogs() + + // Check final count - should have deleted the 2 old entries, kept the recent one + var finalCount int + err = db.Get(&finalCount, "SELECT COUNT(*) FROM log WHERE job = ?", j.Name) + assert.NoError(t, err) + assert.Equal(t, 1, finalCount) + + // Verify the remaining entry is the recent one + var remainingMessage string + err = db.Get(&remainingMessage, "SELECT message FROM log WHERE job = ?", j.Name) + assert.NoError(t, err) + assert.Equal(t, "recent log entry", remainingMessage) +} + +func TestLogRetentionNoDuration(t *testing.T) { + // Test that cleanup doesn't run when no duration is set + db, err := OpenDB(":memory:") + if err != nil { + t.Fatal(err) + } + defer func() { _ = db.Close() }() + + cfg := NewConfig() + cfg.DB = db + + j := &JobSpec{ + Name: "test-no-retention", + Command: []string{"echo", "test"}, + // No logRetentionDuration set + cfg: cfg, + log: NewLogger("debug", nil), + } + + // Insert a log entry + _, err = db.Exec(` + INSERT INTO log (job, triggered_at, triggered_by, duration, status, message) + VALUES (?, ?, ?, ?, ?, ?) + `, j.Name, time.Now().Add(-2*time.Hour), "test", 100, 0, "old log entry") + assert.NoError(t, err) + + // Run log cleanup - should do nothing + j.cleanupOldLogs() + + // Verify entry still exists + var count int + err = db.Get(&count, "SELECT COUNT(*) FROM log WHERE job = ?", j.Name) + assert.NoError(t, err) + assert.Equal(t, 1, count) +} + +func TestLogRetentionNoDB(t *testing.T) { + // Test that cleanup doesn't crash when no DB connection + j := &JobSpec{ + Name: "test-no-db", + Command: []string{"echo", "test"}, + logRetentionDuration: 1 * time.Hour, + cfg: NewConfig(), // No DB set + log: NewLogger("debug", nil), + } + + // This should not panic + j.cleanupOldLogs() +} diff --git a/pkg/schedule.go b/pkg/schedule.go index ec263ee..f21f1fe 100644 --- a/pkg/schedule.go +++ b/pkg/schedule.go @@ -12,9 +12,23 @@ import ( "gopkg.in/yaml.v3" + "github.com/pawelszydlo/humanize" "github.com/rs/zerolog" ) +var ( + humanizerOnce sync.Once + humanizerInst *humanize.Humanizer + humanizerErr error +) + +func getHumanizer() (*humanize.Humanizer, error) { + humanizerOnce.Do(func() { + humanizerInst, humanizerErr = humanize.New("en") + }) + return humanizerInst, humanizerErr +} + // Schedule defines specs of a job schedule. type Schedule struct { Jobs map[string]*JobSpec `yaml:"jobs" json:"jobs"` @@ -152,6 +166,22 @@ func (s *Schedule) initialize() error { return err } + // parse log retention period if specified + if v.LogRetentionPeriod != "" { + humanizer, err := getHumanizer() + if err != nil { + return fmt.Errorf("failed to create humanizer: %w", err) + } + duration, err := humanizer.ParseDuration(v.LogRetentionPeriod) + if err != nil { + return fmt.Errorf("invalid log_retention_period for job '%s': %w", k, err) + } + if duration <= 0 { + return fmt.Errorf("log_retention_period for job '%s' must be positive", k) + } + v.logRetentionDuration = duration + } + // init nextTick if err := v.setNextTick(s.now(), true); err != nil { return err diff --git a/testdata/readme_example.yaml b/testdata/readme_example.yaml index 4477c3a..94dbe52 100644 --- a/testdata/readme_example.yaml +++ b/testdata/readme_example.yaml @@ -19,6 +19,7 @@ jobs: command: this fails cron: "* * * * *" retries: 3 + log_retention_period: 7 days on_error: notify_webhook: # notify something on error - https://webhook.site/4b732eb4-ba10-4a84-8f6b-30167b2f2762 From 4dfb61441cb75ca24a58635fe5e728dec325316b Mon Sep 17 00:00:00 2001 From: Francois Botha Date: Fri, 13 Feb 2026 12:58:46 +0200 Subject: [PATCH 4/5] Add log retention period on schedule level too --- pkg/job_test.go | 116 +++++++++++++ pkg/schedule.go | 37 ++++- pkg/schedule_test.go | 312 +++++++++++++++++++++++++++++++++++ testdata/readme_example.yaml | 3 +- 4 files changed, 459 insertions(+), 9 deletions(-) diff --git a/pkg/job_test.go b/pkg/job_test.go index 6b9a79e..f314b87 100644 --- a/pkg/job_test.go +++ b/pkg/job_test.go @@ -10,6 +10,7 @@ import ( "net/http" "net/http/httptest" "os" + "strings" "testing" "time" @@ -915,3 +916,118 @@ func TestLogRetentionNoDB(t *testing.T) { // This should not panic j.cleanupOldLogs() } + +func TestGlobalLogRetentionInheritance(t *testing.T) { + tests := []struct { + name string + globalDuration string + jobDuration string + expectedJobDuration time.Duration + shouldError bool + expectedErr string + }{ + { + name: "global only, job inherits", + globalDuration: "30 days", + jobDuration: "", + expectedJobDuration: 30 * 24 * time.Hour, + shouldError: false, + }, + { + name: "job overrides global", + globalDuration: "30 days", + jobDuration: "7 days", + expectedJobDuration: 7 * 24 * time.Hour, + shouldError: false, + }, + { + name: "no global, no job", + globalDuration: "", + jobDuration: "", + expectedJobDuration: 0, + shouldError: false, + }, + { + name: "invalid global duration", + globalDuration: "invalid", + jobDuration: "", + expectedJobDuration: 0, + shouldError: true, + expectedErr: "invalid log_retention_period for schedule", + }, + { + name: "valid global, invalid job", + globalDuration: "30 days", + jobDuration: "invalid", + expectedJobDuration: 0, + shouldError: true, + expectedErr: "invalid log_retention_period for job", + }, + { + name: "job overrides with zero duration", + globalDuration: "30 days", + jobDuration: "0 seconds", + expectedJobDuration: 0, + shouldError: true, + expectedErr: "must be positive", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + // Build YAML content + yamlContent := fmt.Sprintf(` +tz_location: UTC +log_retention_period: %s +jobs: + test-job: + command: echo "test" + cron: "* * * * *" +`, tc.globalDuration) + + // Add job-specific retention if specified + if tc.jobDuration != "" { + yamlContent = strings.Replace(yamlContent, " cron: \"* * * * *\"", + fmt.Sprintf(" cron: \"* * * * *\"\n log_retention_period: %s", tc.jobDuration), 1) + } + + // Write to temp file + tmpfile, err := os.CreateTemp("", "test-global-*.yaml") + if err != nil { + t.Fatal(err) + } + defer func() { _ = os.Remove(tmpfile.Name()) }() + + if _, err := tmpfile.Write([]byte(yamlContent)); err != nil { + t.Fatal(err) + } + if err := tmpfile.Close(); err != nil { + t.Fatal(err) + } + + // Try to load schedule + log := NewLogger("debug", nil, os.Stdout) + cfg := NewConfig() + schedule, err := loadSchedule(log, cfg, tmpfile.Name()) + + if tc.shouldError { + assert.Error(t, err) + if tc.expectedErr != "" { + assert.Contains(t, err.Error(), tc.expectedErr) + } + } else { + assert.NoError(t, err) + // Check that job has the expected duration + job := schedule.Jobs["test-job"] + assert.NotNil(t, job) + assert.Equal(t, tc.expectedJobDuration, job.logRetentionDuration) + + // Check schedule has global duration if specified and valid + if tc.globalDuration != "" && tc.globalDuration != "invalid" && !tc.shouldError { + // Schedule should have a non-zero duration + assert.Greater(t, schedule.logRetentionDuration, time.Duration(0)) + } + } + }) + } +} diff --git a/pkg/schedule.go b/pkg/schedule.go index f21f1fe..2e4b4d5 100644 --- a/pkg/schedule.go +++ b/pkg/schedule.go @@ -31,14 +31,16 @@ func getHumanizer() (*humanize.Humanizer, error) { // Schedule defines specs of a job schedule. type Schedule struct { - Jobs map[string]*JobSpec `yaml:"jobs" json:"jobs"` - OnSuccess OnEvent `yaml:"on_success,omitempty" json:"on_success,omitempty"` - OnError OnEvent `yaml:"on_error,omitempty" json:"on_error,omitempty"` - OnRetriesExhausted OnEvent `yaml:"on_retries_exhausted,omitempty" json:"on_retries_exhausted,omitempty"` - TZLocation string `yaml:"tz_location,omitempty" json:"tz_location,omitempty"` - loc *time.Location - log zerolog.Logger - cfg Config + Jobs map[string]*JobSpec `yaml:"jobs" json:"jobs"` + OnSuccess OnEvent `yaml:"on_success,omitempty" json:"on_success,omitempty"` + OnError OnEvent `yaml:"on_error,omitempty" json:"on_error,omitempty"` + OnRetriesExhausted OnEvent `yaml:"on_retries_exhausted,omitempty" json:"on_retries_exhausted,omitempty"` + TZLocation string `yaml:"tz_location,omitempty" json:"tz_location,omitempty"` + LogRetentionPeriod string `yaml:"log_retention_period,omitempty" json:"log_retention_period,omitempty"` + loc *time.Location + log zerolog.Logger + cfg Config + logRetentionDuration time.Duration } func (s *Schedule) Run() { @@ -146,6 +148,22 @@ func (s *Schedule) initialize() error { } s.loc = loc + // parse global log retention period if specified + if s.LogRetentionPeriod != "" { + humanizer, err := getHumanizer() + if err != nil { + return fmt.Errorf("failed to create humanizer: %w", err) + } + duration, err := humanizer.ParseDuration(s.LogRetentionPeriod) + if err != nil { + return fmt.Errorf("invalid log_retention_period for schedule: %w", err) + } + if duration <= 0 { + return fmt.Errorf("log_retention_period for schedule must be positive") + } + s.logRetentionDuration = duration + } + for k, v := range s.Jobs { // check if trigger references exist triggerJobs := append(v.OnSuccess.TriggerJob, v.OnError.TriggerJob...) @@ -180,6 +198,9 @@ func (s *Schedule) initialize() error { return fmt.Errorf("log_retention_period for job '%s' must be positive", k) } v.logRetentionDuration = duration + } else if s.logRetentionDuration > 0 { + // inherit from global schedule setting + v.logRetentionDuration = s.logRetentionDuration } // init nextTick diff --git a/pkg/schedule_test.go b/pkg/schedule_test.go index e04de0b..7ec112d 100644 --- a/pkg/schedule_test.go +++ b/pkg/schedule_test.go @@ -165,3 +165,315 @@ jobs: // because jobs can overlap (8 seconds runtime with 3-second jobs starting every second) assert.Greater(t, concurrentStarts, 1, "Expected more than 1 start for concurrent job") } + +func TestScheduleGlobalLogRetentionPeriod(t *testing.T) { + tests := []struct { + name string + yamlContent string + shouldError bool + expectedErr string + expectDuration time.Duration + }{ + { + name: "valid global retention period - days", + yamlContent: ` +tz_location: UTC +log_retention_period: 30 days +jobs: + test_job: + command: echo "test" + cron: "* * * * *" +`, + shouldError: false, + expectDuration: 30 * 24 * time.Hour, + }, + { + name: "valid global retention period - weeks", + yamlContent: ` +tz_location: UTC +log_retention_period: 2 weeks +jobs: + test_job: + command: echo "test" + cron: "* * * * *" +`, + shouldError: false, + expectDuration: 14 * 24 * time.Hour, + }, + { + name: "valid global retention period - months", + yamlContent: ` +tz_location: UTC +log_retention_period: 3 months +jobs: + test_job: + command: echo "test" + cron: "* * * * *" +`, + shouldError: false, + expectDuration: 90 * 24 * time.Hour, // 3 months ≈ 90 days + }, + { + name: "no global retention period", + yamlContent: ` +tz_location: UTC +jobs: + test_job: + command: echo "test" + cron: "* * * * *" +`, + shouldError: false, + expectDuration: 0, + }, + { + name: "invalid global retention period", + yamlContent: ` +tz_location: UTC +log_retention_period: invalid +jobs: + test_job: + command: echo "test" + cron: "* * * * *" +`, + shouldError: true, + expectedErr: "invalid log_retention_period for schedule", + }, + { + name: "zero global retention period", + yamlContent: ` +tz_location: UTC +log_retention_period: 0 seconds +jobs: + test_job: + command: echo "test" + cron: "* * * * *" +`, + shouldError: true, + expectedErr: "must be positive", + }, + { + name: "negative global retention period", + yamlContent: ` +tz_location: UTC +log_retention_period: -1 hour +jobs: + test_job: + command: echo "test" + cron: "* * * * *" +`, + shouldError: true, + expectedErr: "must be positive", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + // Write test schedule to temp file + tmpFile, err := os.CreateTemp("", "test_schedule_*.yaml") + if err != nil { + t.Fatal(err) + } + defer func() { _ = os.Remove(tmpFile.Name()) }() + + if _, err := tmpFile.WriteString(tc.yamlContent); err != nil { + t.Fatal(err) + } + _ = tmpFile.Close() + + // Load the schedule + logger := NewLogger("debug", nil, os.Stdout) + schedule, err := loadSchedule(logger, Config{}, tmpFile.Name()) + + if tc.shouldError { + assert.Error(t, err) + if tc.expectedErr != "" { + assert.Contains(t, err.Error(), tc.expectedErr) + } + } else { + assert.NoError(t, err) + assert.Equal(t, tc.expectDuration, schedule.logRetentionDuration) + + // Verify schedule fields are set correctly + assert.NotNil(t, schedule.Jobs) + assert.NotNil(t, schedule.loc) + + // Check that job exists + job, exists := schedule.Jobs["test_job"] + assert.True(t, exists) + assert.NotNil(t, job) + } + }) + } +} + +func TestScheduleGlobalLogRetentionPeriodInheritance(t *testing.T) { + tests := []struct { + name string + yamlContent string + jobName string + expectDuration time.Duration + }{ + { + name: "job inherits global retention period", + yamlContent: ` +tz_location: UTC +log_retention_period: 30 days +jobs: + job1: + command: echo "test1" + cron: "* * * * *" + job2: + command: echo "test2" + cron: "0 * * * *" +`, + jobName: "job1", + expectDuration: 30 * 24 * time.Hour, + }, + { + name: "job overrides global retention period", + yamlContent: ` +tz_location: UTC +log_retention_period: 30 days +jobs: + job1: + command: echo "test1" + cron: "* * * * *" + log_retention_period: 7 days + job2: + command: echo "test2" + cron: "0 * * * *" +`, + jobName: "job1", + expectDuration: 7 * 24 * time.Hour, + }, + { + name: "mixed inheritance and overrides", + yamlContent: ` +tz_location: UTC +log_retention_period: 30 days +jobs: + inherits_job: + command: echo "inherits" + cron: "* * * * *" + overrides_job: + command: echo "overrides" + cron: "0 * * * *" + log_retention_period: 14 days +`, + jobName: "inherits_job", + expectDuration: 30 * 24 * time.Hour, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + // Write test schedule to temp file + tmpFile, err := os.CreateTemp("", "test_schedule_*.yaml") + if err != nil { + t.Fatal(err) + } + defer func() { _ = os.Remove(tmpFile.Name()) }() + + if _, err := tmpFile.WriteString(tc.yamlContent); err != nil { + t.Fatal(err) + } + _ = tmpFile.Close() + + // Load the schedule + logger := NewLogger("debug", nil, os.Stdout) + schedule, err := loadSchedule(logger, Config{}, tmpFile.Name()) + assert.NoError(t, err) + + // Check specific job + job, exists := schedule.Jobs[tc.jobName] + assert.True(t, exists) + assert.NotNil(t, job) + assert.Equal(t, tc.expectDuration, job.logRetentionDuration) + + // Verify schedule has global duration + if strings.Contains(tc.yamlContent, "log_retention_period:") && + !strings.Contains(tc.yamlContent, "log_retention_period: invalid") { + assert.Greater(t, schedule.logRetentionDuration, time.Duration(0)) + } + }) + } +} + +func TestScheduleGlobalLogRetentionPeriodWithExistingFeatures(t *testing.T) { + // Test that global retention period doesn't break existing schedule features + testScheduleYAML := ` +tz_location: UTC +log_retention_period: 30 days +jobs: + foo: + command: date + cron: "* * * * *" + on_success: + trigger_job: + - bar + bar: + command: + - echo + - $foo + env: + foo: bar + cron: "* * * * *" + coffee: + command: this fails + cron: "* * * * *" + retries: 3 + log_retention_period: 7 days + on_error: + notify_webhook: + - https://example.com/webhook +` + + // Write test schedule to temp file + tmpFile, err := os.CreateTemp("", "test_schedule_*.yaml") + if err != nil { + t.Fatal(err) + } + defer func() { _ = os.Remove(tmpFile.Name()) }() + + if _, err := tmpFile.WriteString(testScheduleYAML); err != nil { + t.Fatal(err) + } + _ = tmpFile.Close() + + // Load the schedule + logger := NewLogger("debug", nil, os.Stdout) + schedule, err := loadSchedule(logger, Config{}, tmpFile.Name()) + assert.NoError(t, err) + + // Verify schedule has global duration + assert.Equal(t, 30*24*time.Hour, schedule.logRetentionDuration) + + // Verify jobs are loaded correctly + assert.Len(t, schedule.Jobs, 3) + + // Check specific jobs + fooJob, exists := schedule.Jobs["foo"] + assert.True(t, exists) + assert.Equal(t, 30*24*time.Hour, fooJob.logRetentionDuration) // Inherits global + + barJob, exists := schedule.Jobs["bar"] + assert.True(t, exists) + assert.Equal(t, 30*24*time.Hour, barJob.logRetentionDuration) // Inherits global + + coffeeJob, exists := schedule.Jobs["coffee"] + assert.True(t, exists) + assert.Equal(t, 7*24*time.Hour, coffeeJob.logRetentionDuration) // Overrides global + + // Verify job triggers are set up + assert.Contains(t, fooJob.OnSuccess.TriggerJob, "bar") + + // Verify environment variables + assert.Equal(t, "bar", string(barJob.Env["foo"])) + + // Verify retries + assert.Equal(t, 3, coffeeJob.Retries) + + // Verify webhooks + assert.Len(t, coffeeJob.OnError.NotifyWebhook, 1) + assert.Equal(t, "https://example.com/webhook", coffeeJob.OnError.NotifyWebhook[0]) +} diff --git a/testdata/readme_example.yaml b/testdata/readme_example.yaml index 94dbe52..6115b78 100644 --- a/testdata/readme_example.yaml +++ b/testdata/readme_example.yaml @@ -1,4 +1,5 @@ tz_location: Europe/Brussels # optionally set timezone to adhere to +log_retention_period: 30 days # global default for all jobs (optional) jobs: foo: command: date @@ -19,7 +20,7 @@ jobs: command: this fails cron: "* * * * *" retries: 3 - log_retention_period: 7 days + log_retention_period: 7 days # overrides global setting on_error: notify_webhook: # notify something on error - https://webhook.site/4b732eb4-ba10-4a84-8f6b-30167b2f2762 From fe285fc8931d59fd3705839f5e945c07369c53ff Mon Sep 17 00:00:00 2001 From: Francois Botha Date: Fri, 13 Feb 2026 13:04:27 +0200 Subject: [PATCH 5/5] Add documentation --- docs/content/en/docs/log-retention.md | 63 +++++++++++++++++++++++++++ docs/data/en/docs/sidebar.yml | 1 + 2 files changed, 64 insertions(+) create mode 100644 docs/content/en/docs/log-retention.md diff --git a/docs/content/en/docs/log-retention.md b/docs/content/en/docs/log-retention.md new file mode 100644 index 0000000..bd7c158 --- /dev/null +++ b/docs/content/en/docs/log-retention.md @@ -0,0 +1,63 @@ +--- +title: Log Retention +--- + +# Log Retention + +Cheek supports automatic cleanup of old log entries to prevent database bloat. Configure retention periods at the schedule level (global default) or job level (overrides). + +## Global Configuration + +Set a default retention period for all jobs: + +```yaml +tz_location: UTC +log_retention_period: 30 days +jobs: + job1: + command: echo "hello" + cron: "* * * * *" +``` + +## Job-Level Configuration + +Jobs can override the global setting: + +```yaml +tz_location: UTC +log_retention_period: 30 days # global default +jobs: + important_job: + command: critical-task + cron: "0 * * * *" + log_retention_period: 7 days # override +``` + +## Inheritance Rules + +- If a job specifies `log_retention_period`, it uses that duration +- If not specified, job inherits from global schedule setting +- If neither is set, no automatic cleanup occurs + +## Supported Duration Formats + +Uses human-readable formats: +- Days: `30 days`, `1 day` +- Weeks: `2 weeks`, `1 week` +- Months: `3 months`, `1 month` +- Hours: `24 hours`, `1 hour` +- Minutes: `90 minutes`, `30 minutes` +- Complex: `1 hour and 30 minutes` + +## Validation + +- Must be positive duration (> 0) +- Invalid formats are rejected at startup +- Zero or negative values cause configuration errors + +## How It Works + +- Cleanup runs automatically after each job execution +- Deletes log entries older than the configured duration +- Uses efficient database queries with indexes +- Only affects jobs with retention configured \ No newline at end of file diff --git a/docs/data/en/docs/sidebar.yml b/docs/data/en/docs/sidebar.yml index db2606c..8be4066 100644 --- a/docs/data/en/docs/sidebar.yml +++ b/docs/data/en/docs/sidebar.yml @@ -11,6 +11,7 @@ - title: WebUI - title: Events - title: Job Flow + - title: Log Retention - title: 📚 Various pages: