Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,11 @@ jobs:
fetch-depth: '0'
- uses: actions/setup-go@v5
with:
go-version: "^1.20"
go-version: "^1.25"
- name: golangci-lint
uses: golangci/golangci-lint-action@v8
with:
version: v2.1
version: v2.9.0

tests:
runs-on: ubuntu-latest
Expand All @@ -26,7 +26,7 @@ jobs:
fetch-depth: '0'
- uses: actions/setup-go@v5
with:
go-version: "^1.20"
go-version: "^1.25"
- name: run tests on cmd
run: go test ./cmd
- name: run tests on pkg
Expand Down Expand Up @@ -86,7 +86,7 @@ jobs:
## setup go
- uses: actions/setup-go@v5
with:
go-version: "^1.22"
go-version: "^1.25"
## above is fine to get latest for now, also save a copy with short sha
- id: vars
run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
Expand Down
63 changes: 63 additions & 0 deletions docs/content/en/docs/log-retention.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
---
title: Log Retention
---

# Log Retention

Cheek supports automatic cleanup of old log entries to prevent database bloat. Configure retention periods at the schedule level (global default) or job level (overrides).

## Global Configuration

Set a default retention period for all jobs:

```yaml
tz_location: UTC
log_retention_period: 30 days
jobs:
job1:
command: echo "hello"
cron: "* * * * *"
```

## Job-Level Configuration

Jobs can override the global setting:

```yaml
tz_location: UTC
log_retention_period: 30 days # global default
jobs:
important_job:
command: critical-task
cron: "0 * * * *"
log_retention_period: 7 days # override
```

## Inheritance Rules

- If a job specifies `log_retention_period`, it uses that duration
- If not specified, job inherits from global schedule setting
- If neither is set, no automatic cleanup occurs

## Supported Duration Formats

Uses human-readable formats:
- Days: `30 days`, `1 day`
- Weeks: `2 weeks`, `1 week`
- Months: `3 months`, `1 month`
- Hours: `24 hours`, `1 hour`
- Minutes: `90 minutes`, `30 minutes`
- Complex: `1 hour and 30 minutes`

## Validation

- Must be positive duration (> 0)
- Invalid formats are rejected at startup
- Zero or negative values cause configuration errors

## How It Works

- Cleanup runs automatically after each job execution
- Deletes log entries older than the configured duration
- Uses efficient database queries with indexes
- Only affects jobs with retention configured
1 change: 1 addition & 0 deletions docs/data/en/docs/sidebar.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
- title: WebUI
- title: Events
- title: Job Flow
- title: Log Retention

- title: 📚 Various
pages:
Expand Down
7 changes: 3 additions & 4 deletions go.mod
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
module github.com/bart6114/cheek

go 1.22

toolchain go1.22.4
go 1.25.6

require github.com/stretchr/testify v1.10.0

Expand All @@ -16,6 +14,7 @@ require (
require (
github.com/jmoiron/sqlx v1.4.0
github.com/julienschmidt/httprouter v1.3.0
github.com/pawelszydlo/humanize v0.0.0-20260212182641-ba25cf34090b
github.com/spf13/cobra v1.9.1
github.com/spf13/viper v1.20.1
)
Expand Down Expand Up @@ -49,6 +48,6 @@ require (
github.com/subosito/gotenv v1.6.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/sys v0.29.0 // indirect
golang.org/x/text v0.21.0 // indirect
golang.org/x/text v0.34.0 // indirect
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
)
6 changes: 4 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,8 @@ github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/mattn/go-sqlite3 v1.14.28 h1:ThEiQrnbtumT+QMknw63Befp/ce/nUPgBPMlRFEum7A=
github.com/mattn/go-sqlite3 v1.14.28/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/pawelszydlo/humanize v0.0.0-20260212182641-ba25cf34090b h1:jtxaEjubJjJkWdYZYsCrrbp39n6PFRPf94agT3nfqy8=
github.com/pawelszydlo/humanize v0.0.0-20260212182641-ba25cf34090b/go.mod h1:rx2jFGgY1C4LL7MSlavPTNRbUTgXHIXgNpGKuPHqaCM=
github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M=
github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
Expand Down Expand Up @@ -87,8 +89,8 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU=
golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
Expand Down
6 changes: 6 additions & 0 deletions pkg/db.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,12 @@ func InitDB(db *sqlx.DB) error {
return fmt.Errorf("create log table: %w", err)
}

// Create index for efficient log retention cleanup queries
_, err = db.Exec(`CREATE INDEX IF NOT EXISTS idx_log_job_triggered_at ON log (job, triggered_at)`)
if err != nil {
return fmt.Errorf("create log index: %w", err)
}

// Perform cleanup to remove old, non-conforming records
_, err = db.Exec(`
DELETE FROM log
Expand Down
33 changes: 27 additions & 6 deletions pkg/job.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,12 +37,14 @@ type JobSpec struct {
Cron string `yaml:"cron,omitempty" json:"cron,omitempty"`
Command stringArray `yaml:"command" json:"command"`

OnSuccess OnEvent `yaml:"on_success,omitempty" json:"on_success,omitempty"`
OnError OnEvent `yaml:"on_error,omitempty" json:"on_error,omitempty"`
OnSuccess OnEvent `yaml:"on_success,omitempty" json:"on_success,omitempty"`
OnError OnEvent `yaml:"on_error,omitempty" json:"on_error,omitempty"`
OnRetriesExhausted OnEvent `yaml:"on_retries_exhausted,omitempty" json:"on_retries_exhausted,omitempty"`

Name string `json:"name"`
Retries int `yaml:"retries,omitempty" json:"retries,omitempty"`
LogRetentionPeriod string `yaml:"log_retention_period,omitempty" json:"log_retention_period,omitempty"`
logRetentionDuration time.Duration `yaml:"-"`
Env map[string]secret `yaml:"env,omitempty"`
WorkingDirectory string `yaml:"working_directory,omitempty" json:"working_directory,omitempty"`
DisableConcurrentExecution bool `yaml:"disable_concurrent_execution,omitempty" json:"disable_concurrent_execution,omitempty"`
Expand All @@ -64,8 +66,8 @@ func (secret) MarshalText() ([]byte, error) {

// JobRun holds information about a job execution.
type JobRun struct {
LogEntryId int `json:"id,omitempty" db:"id"`
Status *int `json:"status,omitempty" db:"status,omitempty"`
LogEntryId int `json:"id,omitempty" db:"id"`
Status *int `json:"status,omitempty" db:"status,omitempty"`
logBuf bytes.Buffer
Log string `json:"log" db:"message"`
Name string `json:"name" db:"job"`
Expand Down Expand Up @@ -135,6 +137,8 @@ func (j *JobSpec) finalize(jr *JobRun) {
if j.cfg.DB == nil {
j.Runs = append(j.Runs, *jr)
}
// cleanup old log entries if configured
j.cleanupOldLogs()
// launch on_events
j.OnEvent(jr)
}
Expand Down Expand Up @@ -208,7 +212,6 @@ func (j *JobSpec) execCommandWithRetry(ctx context.Context, trigger string, pare
return jr
}


func (j *JobSpec) now() time.Time {
// defer for if schedule doesn't exist, allows for easy testing
if j.globalSchedule != nil {
Expand Down Expand Up @@ -319,7 +322,6 @@ func (j *JobSpec) execCommand(ctx context.Context, jr JobRun, trigger string) Jo
return jr
}


func (j *JobSpec) loadLogFromDb(id int) (JobRun, error) {
var jr JobRun
if j.cfg.DB == nil {
Expand Down Expand Up @@ -372,6 +374,25 @@ func (j *JobSpec) loadRunsFromDb(nruns int, includeLogs bool) {
j.Runs = jrs
}

func (j *JobSpec) cleanupOldLogs() {
if j.logRetentionDuration <= 0 || j.cfg.DB == nil {
return
}

cutoffTime := time.Now().Add(-j.logRetentionDuration)

result, err := j.cfg.DB.Exec("DELETE FROM log WHERE job = ? AND triggered_at < ?", j.Name, cutoffTime)
if err != nil {
j.log.Warn().Str("job", j.Name).Err(err).Msg("Failed to cleanup old logs")
return
}

rowsAffected, err := result.RowsAffected()
if err == nil && rowsAffected > 0 {
j.log.Info().Str("job", j.Name).Int64("rows_deleted", rowsAffected).Msg("Cleaned up old log entries")
}
}

func (j *JobSpec) setNextTick(refTime time.Time, includeRefTime bool) error {
if j.Cron != "" {
t, err := gronx.NextTickAfter(j.Cron, refTime, includeRefTime)
Expand Down
Loading