diff --git a/docs/assets/css/docs/article.css b/docs/assets/css/docs/article.css new file mode 100644 index 0000000..04a477d --- /dev/null +++ b/docs/assets/css/docs/article.css @@ -0,0 +1,335 @@ +#article { + padding: 8px 16px; +} + +#article-header { + font-size: 3em; + font-weight: 400; + margin-bottom: 1em; + color: var(--color2) +} + +#article-content h1, +#article-content h2, +#article-content h3, +#article-content h4, +#article-content h5, +#article-content h6 { + line-height: 1em; + font-weight: 400; + margin: 2.6em 0 .1em; + color: var(--color2) +} + +#article-content h1 { + font-size: 1.8em +} + +#article-content h2 { + font-size: 1.5em +} + +#article-content h3 { + font-size: 1.3em +} + +#article-content h4 { + font-size: 1.1em +} + +#article-content .highlight, +#article-content blockquote, +#article-content dl, +#article-content iframe, +#article-content ol, +#article-content p, +#article-content table, +#article-content ul { + margin-top: 1em; + line-height: 1.8rem; + letter-spacing: -.1px; +} + +#article-content blockquote p { + margin: 1em 0 +} + +#article-content blockquote dl, +#article-content blockquote ol, +#article-content blockquote ul { + margin: 0 1em 1em 1em +} + +#article-content a { + color: var(--color-anchor); + text-decoration: none +} + +#article-content a:hover { + color: var(--color-hover); + text-decoration: underline +} + +@media print { + #article-content a { + color: #355265; + text-decoration: underline + } + + #article-content a:after { + content: " (" attr(href) ")"; + font-size: 80% + } +} + +#article-content strong, #article-content b, #article-content table th { + font-weight: 600 +} + +#article-content em { + font-style: italic +} + +#article-content dl, +#article-content ol, +#article-content ul { + margin-left: 20px +} + +#article-content dl dl, +#article-content dl ol, +#article-content dl ul, +#article-content ol dl, +#article-content ol ol, +#article-content ol ul, +#article-content ul dl, +#article-content ul ol, +#article-content ul ul { + margin-top: 0; + margin-bottom: 0 +} + +#article-content ul { + list-style: disc +} + +#article-content ol { + list-style: decimal +} + +#article-content dl { + list-style: square +} + +#article-content li > ul { + list-style: circle +} + +#article-content li > ol { + list-style: lower-alpha +} + +#article-content li p { + margin: 0 +} + +#article-content li .highlight, +#article-content li blockquote, +#article-content li iframe, +#article-content li table { + margin: 1em 0 +} + +#article-content img, +#article-content video { + max-width: 100%; + border-radius: 4px +} + +#article-content blockquote { + padding: 8px 12px; + position: relative; + background: var(--background-fg); + border-left: 4px solid var(--border-color); + border-radius: 6px; +} + +#article-content blockquote footer { + margin: 1em 0; + font-style: italic +} + +#article-content blockquote footer cite:before { + content: "—"; + padding: 0 .3em +} + +#article-content blockquote footer cite a { + color: var(--border-color); +} + +#article-content code, #article-content pre { + font-family: var(--font-family-code); +} + +#article-content h1 code, +#article-content h2 code, +#article-content h3 code, +#article-content h4 code, +#article-content h5 code, +#article-content h6 code, +#article-content p code, +#article-content blockquote code, +#article-content ul code, +#article-content ol code, +#article-content dl code, +#article-content table code { + background: var(--chroma-base01); + padding: 4px; + border-radius: 4px; + font-size: .9em; +} + +#article-content pre:not(.chroma) { + color: var(--chroma-base05); + font-size: .9em; + line-height: 1.8; + letter-spacing: -.1px; + background-color: var(--chroma-base00); + border-radius: 6px; + padding: 16px 24px; + overflow-x: auto; + margin-top: 1em; +} + +#article-content blockquote code { + background: var(--background-fg2); + opacity: .8; +} + +#article-content blockquote .chroma, #article-content blockquote pre:not(.chroma) { + background: var(--background-fg2); + margin-bottom: 1em; +} + +#article-content blockquote .chroma code, #article-content blockquote pre:not(.chroma) code { + padding: 0; +} + +#article-content table { + max-width: 100%; + border: 1px solid var(--border-color) +} + +#article-content table td, +#article-content table th { + padding: 5px 15px +} + +#article-content table tr:nth-child(2n) { + background: var(--background-fg) +} + +#article-footer { + display: grid; + grid-template-columns: 1fr 1fr; + padding-top: 20px; +} + +#article-last-updated, #article-prev-link, #article-next-link { + display: flex; + align-items: center; + padding: 12px 0; +} + +#article-last-updated { + grid-column: 1 / 3; + justify-content: center; + color: var(--color3); +} + +#article-prev-link, #article-next-link { + color: var(--color-anchor); +} + +#article-prev-link:hover, #article-next-link:hover { + color: var(--color-hover); + font-weight: 600; + font-size: 98%; +} + +#article-next-link { + justify-content: flex-end; +} + +#article-prev-link .icon { + padding-right: 6px; +} + +#article-next-link .icon { + padding-left: 6px; +} + +@media (max-width: 767px) { + #article-next-link[data-first-page="true"] { + grid-column: 2/ 3; + } +} + +@media (min-width: 768px) { + #article { + padding: 16px 24px; + } + + #article-footer { + display: grid; + grid-template-columns: repeat(3, 1fr); + } + + #article-prev-link { + grid-column: 1/ 2; + grid-row: 1; + } + + #article-last-updated { + grid-column: 2 / 3; + } + + #article-next-link { + grid-column: 3 / 4; + } +} + +@media (min-width: 1024px) { + #article { + padding: 24px 32px; + } +} + +@media (min-width: 1281px) { + #article { + padding: 32px 40px; + } +} + +@media (min-width: 1920px) { + #article { + padding: 40px 48px; + } + + #article-content { + width: 90%; + } +} + +@media (min-width: 2560px) { + #article-content { + width: 85%; + } +} + +@media (min-width: 3840px) { + #article-content { + width: 80%; + } +} diff --git a/docs/content/en/docs/configuration.md b/docs/content/en/docs/configuration.md index 24bc862..594ea6a 100644 --- a/docs/content/en/docs/configuration.md +++ b/docs/content/en/docs/configuration.md @@ -23,6 +23,7 @@ jobs: - $foo env: # you can pass env variables foo: bar + encoding: utf-8 # specify output encoding if job produces non-UTF-8 text other_workingdir: command: pwd working_directory: ../testdata # specify the working directory of the job @@ -38,6 +39,9 @@ jobs: - https://webhook.site/048ff47f-9ef5-43fb-9375-a795a8c5cbf5 notify_discord_webhook: # notify discord via a discord compatible webhook - https://discord.com/api/webhooks/user/token + legacy_system: + command: legacy-app.exe + encoding: windows-1252 # handle Windows-specific encoding for legacy applications ``` ## Configuration Options @@ -46,10 +50,42 @@ All configuration options are available by checking out `cheek --help` or the he Configuration can be passed as flags to the `cheek` CLI directly. All configuration flags are also possible to set via environment variables. The following environment variables are available, they will override the default and/or set value of their similarly named CLI flags (without the prefix): `CHEEK_PORT`, `CHEEK_SUPPRESSLOGS`, `CHEEK_LOGLEVEL`, `CHEEK_PRETTY`, `CHEEK_HOMEDIR`. +## Output Encoding + +By default, cheek expects job output to be UTF-8 encoded. If your jobs produce output in a different encoding, unsupported characters may display as fallback Unicode "tofu" characters (□) or become corrupted. This problem most commonly occurs on Windows systems, legacy applications, or when working with applications that output text in regional character encodings such as Chinese, Japanese, or Korean. + +To handle non-UTF-8 output correctly, you can specify the `encoding` parameter at the job level to tell cheek how to properly decode the job's output. + +### Supported Encodings + +cheek supports the following output encodings: + +**UTF-8 Compatible** (no transformation needed): +- `utf-8`, `utf8` - UTF-8 Unicode encoding (default) +- `ascii` - ASCII encoding + +**Chinese Encodings**: +- `gbk`, `gb2312`, `cp936` - Simplified Chinese (GBK) +- `gb18030` - Extended Chinese encoding standard +- `big5` - Traditional Chinese + +**Japanese Encodings**: +- `shift-jis`, `shiftjis`, `sjis` - Shift JIS encoding +- `euc-jp` - Extended Unix Code for Japanese + +**Korean Encodings**: +- `euc-kr` - Extended Unix Code for Korean + +**Western Encodings**: +- `iso-8859-1`, `latin1` - Western European (Latin-1) +- `windows-1252`, `cp1252` - Windows Western European + ## Important Notes - If your `command` requires arguments, please make sure to pass them as an array like in the `bar` job example above - You can set `tz_location` if the system time of where you run your service is not to your liking +- The `encoding` parameter is specified per job, allowing different jobs in the same schedule to use different encodings +- If you see garbled characters or □ symbols in job output, consider setting the appropriate `encoding` parameter - The configuration structure should be self-explanatory, but if it's not, please create an [issue](https://github.com/bart6114/cheek/issues) ## Running cheek diff --git a/docs/data/en/docs/sidebar.yml b/docs/data/en/docs/sidebar.yml index db2606c..7456d1a 100644 --- a/docs/data/en/docs/sidebar.yml +++ b/docs/data/en/docs/sidebar.yml @@ -5,7 +5,7 @@ - title: Minimal Example - title: Configuration -- title: ⚙️ Core Features +- title: ⚙️ Core pages: - title: Scheduler - title: WebUI diff --git a/pkg/job.go b/pkg/job.go index 954cf03..d26d994 100644 --- a/pkg/job.go +++ b/pkg/job.go @@ -8,11 +8,19 @@ import ( "io" "os" "os/exec" + "strings" "sync" "time" "github.com/adhocore/gronx" "github.com/rs/zerolog" + "golang.org/x/text/encoding" + "golang.org/x/text/encoding/charmap" + "golang.org/x/text/encoding/japanese" + "golang.org/x/text/encoding/korean" + "golang.org/x/text/encoding/simplifiedchinese" + "golang.org/x/text/encoding/traditionalchinese" + "golang.org/x/text/transform" "gopkg.in/yaml.v3" ) @@ -37,8 +45,8 @@ type JobSpec struct { Cron string `yaml:"cron,omitempty" json:"cron,omitempty"` Command stringArray `yaml:"command" json:"command"` - OnSuccess OnEvent `yaml:"on_success,omitempty" json:"on_success,omitempty"` - OnError OnEvent `yaml:"on_error,omitempty" json:"on_error,omitempty"` + OnSuccess OnEvent `yaml:"on_success,omitempty" json:"on_success,omitempty"` + OnError OnEvent `yaml:"on_error,omitempty" json:"on_error,omitempty"` OnRetriesExhausted OnEvent `yaml:"on_retries_exhausted,omitempty" json:"on_retries_exhausted,omitempty"` Name string `json:"name"` @@ -46,6 +54,7 @@ type JobSpec struct { Env map[string]secret `yaml:"env,omitempty"` WorkingDirectory string `yaml:"working_directory,omitempty" json:"working_directory,omitempty"` DisableConcurrentExecution bool `yaml:"disable_concurrent_execution,omitempty" json:"disable_concurrent_execution,omitempty"` + Encoding string `yaml:"encoding,omitempty" json:"encoding,omitempty"` globalSchedule *Schedule Runs []JobRun `json:"runs" yaml:"-"` @@ -64,8 +73,8 @@ func (secret) MarshalText() ([]byte, error) { // JobRun holds information about a job execution. type JobRun struct { - LogEntryId int `json:"id,omitempty" db:"id"` - Status *int `json:"status,omitempty" db:"status,omitempty"` + LogEntryId int `json:"id,omitempty" db:"id"` + Status *int `json:"status,omitempty" db:"status,omitempty"` logBuf bytes.Buffer Log string `json:"log" db:"message"` Name string `json:"name" db:"job"` @@ -208,6 +217,38 @@ func (j *JobSpec) execCommandWithRetry(ctx context.Context, trigger string, pare return jr } +// getEncodingTransformer returns the appropriate encoding transformer for the given encoding name +// Returns (nil, nil) for UTF-8 compatible encodings (no transformation needed) +// Returns (transformer, nil) for supported encodings that need transformation +// Returns (nil, error) for unsupported encodings +func getEncodingTransformer(encodingName string) (encoding.Encoding, error) { + if encodingName == "" { + return nil, nil // No transformation needed + } + + switch strings.ToLower(encodingName) { + case "utf-8", "utf8", "ascii": + return nil, nil // No transformation needed + case "gbk", "gb2312", "cp936": + return simplifiedchinese.GBK, nil + case "gb18030": + return simplifiedchinese.GB18030, nil + case "big5": + return traditionalchinese.Big5, nil + case "shift-jis", "shiftjis", "sjis": + return japanese.ShiftJIS, nil + case "euc-jp": + return japanese.EUCJP, nil + case "euc-kr": + return korean.EUCKR, nil + case "iso-8859-1", "latin1": + return charmap.ISO8859_1, nil + case "windows-1252", "cp1252": + return charmap.Windows1252, nil + default: + return nil, fmt.Errorf("unsupported encoding: %s", encodingName) + } +} func (j *JobSpec) now() time.Time { // defer for if schedule doesn't exist, allows for easy testing @@ -249,11 +290,25 @@ func (j *JobSpec) execCommand(ctx context.Context, jr JobRun, trigger string) Jo cmd.Dir = j.WorkingDirectory var w io.Writer - switch j.cfg.SuppressLogs { - case true: - w = &jr.logBuf - default: - w = io.MultiWriter(os.Stdout, &jr.logBuf) + baseWriter := func() io.Writer { + switch j.cfg.SuppressLogs { + case true: + return &jr.logBuf + default: + return io.MultiWriter(os.Stdout, &jr.logBuf) + } + }() + + // Apply encoding transformation if specified for job + w = baseWriter // Default to base writer + + if j.Encoding != "" { + if enc, err := getEncodingTransformer(j.Encoding); err != nil { + j.log.Warn().Str("job", j.Name).Str("encoding", j.Encoding).Err(err).Msg("Unsupported encoding specified, falling back to UTF-8") + } else if enc != nil { + w = transform.NewWriter(baseWriter, enc.NewDecoder()) + j.log.Debug().Str("job", j.Name).Str("encoding", j.Encoding).Msg("Applying encoding transformation") + } } // Merge stdout and stderr to same writer @@ -319,7 +374,6 @@ func (j *JobSpec) execCommand(ctx context.Context, jr JobRun, trigger string) Jo return jr } - func (j *JobSpec) loadLogFromDb(id int) (JobRun, error) { var jr JobRun if j.cfg.DB == nil { diff --git a/pkg/job_test.go b/pkg/job_test.go index 88d1728..47df57d 100644 --- a/pkg/job_test.go +++ b/pkg/job_test.go @@ -85,7 +85,7 @@ func TestSpecialCron(t *testing.T) { t.Fatal(err) } - jobRun := JobRun{} // Create a JobRun instance + jobRun := JobRun{} // Create a JobRun instance jr := j.execCommand(context.Background(), jobRun, "test") // Pass JobRun instance and "test" assert.Equal(t, *jr.Status, 0) } @@ -134,7 +134,7 @@ env: t.Fatal("should contain foo") } - jobRun := JobRun{} // Create a JobRun instance + jobRun := JobRun{} // Create a JobRun instance jr := j.execCommand(context.Background(), jobRun, "test") // Pass JobRun instance and "test" jr.flushLogBuffer() @@ -154,7 +154,7 @@ func TestStdErrOut(t *testing.T) { cfg: cfg, } - jobRun := JobRun{} // Create a JobRun instance + jobRun := JobRun{} // Create a JobRun instance jr := j.execCommand(context.Background(), jobRun, "test") // Pass JobRun instance and "test" jr.flushLogBuffer() assert.Contains(t, jr.Log, "stdout") @@ -173,7 +173,7 @@ func TestFailingLog(t *testing.T) { cfg: cfg, } - jobRun := JobRun{} // Create a JobRun instance + jobRun := JobRun{} // Create a JobRun instance jr := j.execCommand(context.Background(), jobRun, "test") // Pass JobRun instance and "test" jr.flushLogBuffer() assert.Contains(t, jr.Log, "this fails") @@ -186,7 +186,7 @@ func TestJobRunNoCommand(t *testing.T) { cfg: NewConfig(), } - jobRun := JobRun{} // Create a JobRun instance + jobRun := JobRun{} // Create a JobRun instance jr := j.execCommand(context.Background(), jobRun, "test") // Pass JobRun instance and "test" assert.NotEqual(t, jr.Status, 0) } @@ -201,7 +201,7 @@ func TestJobNonZero(t *testing.T) { cfg: NewConfig(), } - jobRun := JobRun{} // Create a JobRun instance + jobRun := JobRun{} // Create a JobRun instance jr := j.execCommand(context.Background(), jobRun, "test") // Pass JobRun instance and "test" assert.NotEqual(t, jr.Status, 0) } @@ -246,7 +246,7 @@ func TestOnEventWebhook(t *testing.T) { NotifyWebhook: []string{testServer.URL}, }, } - jobRun := JobRun{} // Create a JobRun instance + jobRun := JobRun{} // Create a JobRun instance jr := j.execCommand(context.Background(), jobRun, "test") // Pass JobRun instance and "test" j.OnEvent(&jr) } @@ -277,7 +277,7 @@ func TestStringArray(t *testing.T) { } j.cfg = NewConfig() - jobRun := JobRun{} // Create a JobRun instance + jobRun := JobRun{} // Create a JobRun instance jr := j.execCommand(context.Background(), jobRun, "test") // Pass JobRun instance and "test" jr.flushLogBuffer() @@ -477,7 +477,7 @@ func TestRetryContextInWebhooks(t *testing.T) { // Check the final payload (should be the retries exhausted one) finalPayload := webhookPayloads[len(webhookPayloads)-1] - + // Verify retry context fields are present assert.Equal(t, float64(1), finalPayload["retry_attempt"], "Final retry attempt should be 1") assert.Equal(t, true, finalPayload["retries_exhausted"], "retries_exhausted should be true") @@ -524,7 +524,7 @@ func TestJobWithBashEval(t *testing.T) { j.log = log j.cfg = cfg - jobRun := JobRun{} // Create a JobRun instance + jobRun := JobRun{} // Create a JobRun instance jr := j.execCommand(context.Background(), jobRun, "test") // Pass JobRun instance and "test" jr.flushLogBuffer() @@ -653,7 +653,7 @@ func TestTriggeredByJobRunContext(t *testing.T) { // Create a child job that will be triggered by the parent childJob := &JobSpec{ - Name: "child-job", + Name: "child-job", Command: []string{"echo", "child output"}, cfg: NewConfig(), log: NewLogger("debug", nil, os.Stdout, os.Stdout), @@ -703,3 +703,75 @@ func TestTriggeredByJobRunContext(t *testing.T) { assert.Equal(t, float64(0), parentContext["status"], "Parent job should have succeeded") assert.Contains(t, parentContext["log"], "parent output", "Parent job log should contain expected output") } + +func TestGetEncodingTransformer(t *testing.T) { + testCases := []struct { + name string + encoding string + expectError bool + expectTransformer bool + }{ + {"Empty string", "", false, false}, + {"UTF-8 uppercase", "UTF-8", false, false}, + {"UTF-8 lowercase", "utf-8", false, false}, + {"UTF8 no dash", "utf8", false, false}, + {"ASCII", "ascii", false, false}, + {"GBK lowercase", "gbk", false, true}, + {"GBK uppercase", "GBK", false, true}, + {"GB18030", "gb18030", false, true}, + {"Big5", "big5", false, true}, + {"Shift-JIS", "shift-jis", false, true}, + {"EUC-KR", "euc-kr", false, true}, + {"Windows-1252", "windows-1252", false, true}, + {"Unsupported encoding", "invalid-encoding", true, false}, + {"Another unsupported", "made-up-encoding", true, false}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + transformer, err := getEncodingTransformer(tc.encoding) + + if tc.expectError { + assert.Error(t, err, "Expected error for encoding %s", tc.encoding) + assert.Nil(t, transformer, "Expected nil transformer for unsupported encoding %s", tc.encoding) + assert.Contains(t, err.Error(), tc.encoding, "Error should mention the unsupported encoding") + } else { + assert.NoError(t, err, "Expected no error for encoding %s", tc.encoding) + if tc.expectTransformer { + assert.NotNil(t, transformer, "Expected transformer for encoding %s", tc.encoding) + } else { + assert.Nil(t, transformer, "Expected no transformer for UTF-8 compatible encoding %s", tc.encoding) + } + } + }) + } +} + +func TestJobWithEncoding(t *testing.T) { + cfg := NewConfig() + cfg.SuppressLogs = true + + // Create a schedule without encoding (encoding is now at job level) + schedule := &Schedule{ + Jobs: make(map[string]*JobSpec), + loc: time.UTC, + } + + j := &JobSpec{ + Name: "encoding-test", + Command: []string{"echo", "test output"}, + Encoding: "utf-8", // encoding now specified at job level + cfg: cfg, + log: NewLogger("debug", nil, os.Stdout, os.Stdout), + globalSchedule: schedule, + } + + schedule.Jobs["encoding-test"] = j + + jobRun := JobRun{} + jr := j.execCommand(context.Background(), jobRun, "test") + jr.flushLogBuffer() + + assert.Equal(t, StatusOK, *jr.Status, "Job should succeed with UTF-8 encoding") + assert.Contains(t, jr.Log, "test output", "Job output should contain expected text") +} diff --git a/testdata/encoding_example.yaml b/testdata/encoding_example.yaml new file mode 100644 index 0000000..05b2aaa --- /dev/null +++ b/testdata/encoding_example.yaml @@ -0,0 +1,31 @@ +tz_location: Europe/Brussels +jobs: + utf8_job: + command: echo "UTF-8 text output" + cron: "0 */6 * * *" + encoding: utf-8 + + windows_job: + command: echo "Windows-1252 encoded output" + cron: "0 */12 * * *" + encoding: windows-1252 + + chinese_job: + command: echo "GBK encoded Chinese text" + encoding: gbk + + japanese_job: + command: echo "Shift-JIS encoded Japanese text" + encoding: shift-jis + retries: 2 + on_error: + notify_webhook: + - https://webhook.site/encoding-error + + mixed_encoding_trigger: + command: echo "This job triggers different encoding jobs" + cron: "0 8 * * *" + on_success: + trigger_job: + - utf8_job + - chinese_job \ No newline at end of file