diff --git a/config/scheduler/config.yaml b/config/scheduler/config.yaml index 8b00809c565..1fd970e21f4 100644 --- a/config/scheduler/config.yaml +++ b/config/scheduler/config.yaml @@ -107,6 +107,9 @@ scheduling: maximumPerQueueSchedulingBurst: 1000 maxJobSchedulingContextsPerExecutor: 10000 maxRetries: 3 + retryPolicy: + enabled: false + globalMaxRetries: 5 dominantResourceFairnessResourcesToConsider: - "cpu" - "memory" diff --git a/internal/common/errormatch/doc.go b/internal/common/errormatch/doc.go new file mode 100644 index 00000000000..0db421bf275 --- /dev/null +++ b/internal/common/errormatch/doc.go @@ -0,0 +1,13 @@ +// Package errormatch provides types and functions for matching job failure +// signals: exit codes, termination messages, and Kubernetes conditions. +// +// [ExitCodeMatcher] supports In/NotIn set membership against container exit +// codes. Exit code 0 never matches. [RegexMatcher] holds a pattern string +// that callers compile at construction time and pass to [MatchPattern]. +// +// Pod-level condition constants ([ConditionOOMKilled], [ConditionEvicted], +// [ConditionDeadlineExceeded], [ConditionAppError]) and the [KnownConditions] +// map are provided for config validation. Non-pod conditions +// ([ConditionPreempted], [ConditionLeaseReturned]) are also defined here for +// use by the retry engine but are not included in [KnownConditions]. +package errormatch diff --git a/internal/common/errormatch/match.go b/internal/common/errormatch/match.go new file mode 100644 index 00000000000..03c937768a8 --- /dev/null +++ b/internal/common/errormatch/match.go @@ -0,0 +1,33 @@ +package errormatch + +import "regexp" + +// MatchExitCode returns true if the exit code matches the matcher. +// Exit code 0 never matches (successful containers are not failures). +func MatchExitCode(matcher *ExitCodeMatcher, exitCode int32) bool { + if matcher == nil || exitCode == 0 { + return false + } + switch matcher.Operator { + case ExitCodeOperatorIn: + for _, v := range matcher.Values { + if exitCode == v { + return true + } + } + case ExitCodeOperatorNotIn: + for _, v := range matcher.Values { + if exitCode == v { + return false + } + } + return true + } + return false +} + +// MatchPattern returns true if the value matches the compiled regex. +// Empty values never match. +func MatchPattern(re *regexp.Regexp, value string) bool { + return value != "" && re.MatchString(value) +} diff --git a/internal/common/errormatch/match_test.go b/internal/common/errormatch/match_test.go new file mode 100644 index 00000000000..c168bee384d --- /dev/null +++ b/internal/common/errormatch/match_test.go @@ -0,0 +1,89 @@ +package errormatch + +import ( + "regexp" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestMatchExitCode(t *testing.T) { + tests := map[string]struct { + matcher *ExitCodeMatcher + exitCode int32 + expected bool + }{ + "In matches": { + matcher: &ExitCodeMatcher{Operator: ExitCodeOperatorIn, Values: []int32{74, 75}}, + exitCode: 74, + expected: true, + }, + "In does not match": { + matcher: &ExitCodeMatcher{Operator: ExitCodeOperatorIn, Values: []int32{74, 75}}, + exitCode: 1, + expected: false, + }, + "NotIn matches when code absent": { + matcher: &ExitCodeMatcher{Operator: ExitCodeOperatorNotIn, Values: []int32{1, 2}}, + exitCode: 42, + expected: true, + }, + "NotIn does not match when code present": { + matcher: &ExitCodeMatcher{Operator: ExitCodeOperatorNotIn, Values: []int32{1, 2}}, + exitCode: 1, + expected: false, + }, + "exit code 0 never matches In": { + matcher: &ExitCodeMatcher{Operator: ExitCodeOperatorIn, Values: []int32{0}}, + exitCode: 0, + expected: false, + }, + "exit code 0 never matches NotIn": { + matcher: &ExitCodeMatcher{Operator: ExitCodeOperatorNotIn, Values: []int32{1}}, + exitCode: 0, + expected: false, + }, + "nil matcher returns false": { + matcher: nil, + exitCode: 1, + expected: false, + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + assert.Equal(t, tc.expected, MatchExitCode(tc.matcher, tc.exitCode)) + }) + } +} + +func TestMatchPattern(t *testing.T) { + tests := map[string]struct { + pattern string + value string + expected bool + }{ + "matches": { + pattern: "(?i)cuda.*error", + value: "CUDA memory error on device 0", + expected: true, + }, + "does not match": { + pattern: "(?i)cuda.*error", + value: "segfault", + expected: false, + }, + "empty value never matches": { + pattern: ".*", + value: "", + expected: false, + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + re := regexp.MustCompile(tc.pattern) + assert.Equal(t, tc.expected, MatchPattern(re, tc.value)) + }) + } +} diff --git a/internal/common/errormatch/types.go b/internal/common/errormatch/types.go new file mode 100644 index 00000000000..3ea88e7bf73 --- /dev/null +++ b/internal/common/errormatch/types.go @@ -0,0 +1,44 @@ +package errormatch + +// ExitCodeOperator is a set membership operator: In or NotIn. +type ExitCodeOperator string + +const ( + ExitCodeOperatorIn ExitCodeOperator = "In" + ExitCodeOperatorNotIn ExitCodeOperator = "NotIn" +) + +// ExitCodeMatcher specifies an operator and a set of exit code values. +type ExitCodeMatcher struct { + Operator ExitCodeOperator `yaml:"operator"` + Values []int32 `yaml:"values"` +} + +// RegexMatcher specifies a regex pattern as a string. +type RegexMatcher struct { + Pattern string `yaml:"pattern"` +} + +// Condition constants derived from KubernetesReason (pod-level conditions). +const ( + ConditionOOMKilled = "OOMKilled" + ConditionEvicted = "Evicted" + ConditionDeadlineExceeded = "DeadlineExceeded" + ConditionAppError = "AppError" +) + +// Condition constants for non-pod error types (used by the retry engine). +const ( + ConditionPreempted = "Preempted" + ConditionLeaseReturned = "LeaseReturned" +) + +// KnownConditions is the set of pod-level condition strings accepted by the +// executor error categorizer. Non-pod conditions (Preempted, LeaseReturned) +// are excluded because they are not observable from Kubernetes pod status. +var KnownConditions = map[string]bool{ + ConditionOOMKilled: true, + ConditionEvicted: true, + ConditionDeadlineExceeded: true, + ConditionAppError: true, +} diff --git a/internal/executor/categorizer/classifier.go b/internal/executor/categorizer/classifier.go new file mode 100644 index 00000000000..2ebf05993ad --- /dev/null +++ b/internal/executor/categorizer/classifier.go @@ -0,0 +1,249 @@ +package categorizer + +import ( + "fmt" + "regexp" + + v1 "k8s.io/api/core/v1" + + "github.com/armadaproject/armada/internal/common/errormatch" +) + +type category struct { + name string + rules []rule +} + +type rule struct { + containerName string + onExitCodes *errormatch.ExitCodeMatcher + onTerminationMessage *regexp.Regexp + onConditions []string +} + +// Classifier evaluates pods against a set of category rules and returns +// the names of all matching categories. +type Classifier struct { + categories []category +} + +// NewClassifier validates config and compiles regex patterns. +// Returns an error if any regex is invalid, a condition is unknown, +// or an exit code matcher has an invalid operator. +func NewClassifier(configs []CategoryConfig) (*Classifier, error) { + categories := make([]category, 0, len(configs)) + seen := make(map[string]bool, len(configs)) + for _, cfg := range configs { + if cfg.Name == "" { + return nil, fmt.Errorf("category config must have a name") + } + if seen[cfg.Name] { + return nil, fmt.Errorf("duplicate category name %q", cfg.Name) + } + seen[cfg.Name] = true + if len(cfg.Rules) == 0 { + return nil, fmt.Errorf("category %q must have at least one rule", cfg.Name) + } + cat := category{name: cfg.Name} + for i, rule := range cfg.Rules { + r, err := buildRule(rule) + if err != nil { + return nil, fmt.Errorf("category %q rule %d: %w", cfg.Name, i, err) + } + cat.rules = append(cat.rules, r) + } + categories = append(categories, cat) + } + return &Classifier{categories: categories}, nil +} + +func buildRule(cfg CategoryRule) (rule, error) { + matcherCount := 0 + if len(cfg.OnConditions) > 0 { + matcherCount++ + } + if cfg.OnExitCodes != nil { + matcherCount++ + } + if cfg.OnTerminationMessage != nil { + matcherCount++ + } + if matcherCount == 0 { + return rule{}, fmt.Errorf("rule must specify one of onConditions, onExitCodes, or onTerminationMessage") + } + if matcherCount > 1 { + return rule{}, fmt.Errorf("rule must specify only one of onConditions, onExitCodes, or onTerminationMessage") + } + + for _, cond := range cfg.OnConditions { + if !errormatch.KnownConditions[cond] { + return rule{}, fmt.Errorf("unknown condition %q, valid values: %s, %s, %s, %s", + cond, errormatch.ConditionOOMKilled, errormatch.ConditionEvicted, errormatch.ConditionDeadlineExceeded, errormatch.ConditionAppError) + } + } + + if cfg.OnExitCodes != nil { + switch cfg.OnExitCodes.Operator { + case errormatch.ExitCodeOperatorIn, errormatch.ExitCodeOperatorNotIn: + // valid + default: + return rule{}, fmt.Errorf("invalid exit code operator %q, must be %q or %q", + cfg.OnExitCodes.Operator, errormatch.ExitCodeOperatorIn, errormatch.ExitCodeOperatorNotIn) + } + if len(cfg.OnExitCodes.Values) == 0 { + return rule{}, fmt.Errorf("exit code matcher requires at least one value") + } + } + + var compiledRegex *regexp.Regexp + if cfg.OnTerminationMessage != nil { + re, err := regexp.Compile(cfg.OnTerminationMessage.Pattern) + if err != nil { + return rule{}, fmt.Errorf("invalid regex %q: %w", cfg.OnTerminationMessage.Pattern, err) + } + compiledRegex = re + } + + return rule{ + containerName: cfg.ContainerName, + onExitCodes: cfg.OnExitCodes, + onConditions: cfg.OnConditions, + onTerminationMessage: compiledRegex, + }, nil +} + +// Classify returns the names of all categories that match the given pod. +// Returns nil if the receiver is nil, the pod is nil, or no categories match. +func (c *Classifier) Classify(pod *v1.Pod) []string { + if c == nil || pod == nil { + return nil + } + containers := failedContainers(pod) + podReason := pod.Status.Reason + + var matched []string + for _, cat := range c.categories { + if categoryMatches(cat, containers, podReason) { + matched = append(matched, cat.name) + } + } + return matched +} + +func categoryMatches(cat category, containers []containerInfo, podReason string) bool { + for _, rule := range cat.rules { + if ruleMatches(rule, containers, podReason) { + return true + } + } + return false +} + +// ruleMatches evaluates a single rule. When containerName is set, only that +// container is considered. It checks the first non-nil matcher: +// conditions > exit codes > termination message. +func ruleMatches(r rule, containers []containerInfo, podReason string) bool { + filtered := containers + if r.containerName != "" { + filtered = filterByName(containers, r.containerName) + } + if len(r.onConditions) > 0 { + return matchesCondition(r.onConditions, filtered, podReason) + } + if r.onExitCodes != nil { + return matchesExitCodes(r.onExitCodes, filtered) + } + if r.onTerminationMessage != nil { + return matchesTerminationMessage(r.onTerminationMessage, filtered) + } + return false +} + +func filterByName(containers []containerInfo, name string) []containerInfo { + var result []containerInfo + for _, c := range containers { + if c.name == name { + result = append(result, c) + } + } + return result +} + +func matchesCondition(conditions []string, containers []containerInfo, podReason string) bool { + for _, cond := range conditions { + switch cond { + case errormatch.ConditionOOMKilled: + for _, c := range containers { + if c.reason == errormatch.ConditionOOMKilled { + return true + } + } + case errormatch.ConditionAppError: + for _, c := range containers { + if c.reason == errormatch.ConditionAppError { + return true + } + } + case errormatch.ConditionEvicted: + if podReason == errormatch.ConditionEvicted { + return true + } + case errormatch.ConditionDeadlineExceeded: + if podReason == errormatch.ConditionDeadlineExceeded { + return true + } + } + } + return false +} + +func matchesExitCodes(matcher *errormatch.ExitCodeMatcher, containers []containerInfo) bool { + for _, c := range containers { + if errormatch.MatchExitCode(matcher, c.exitCode) { + return true + } + } + return false +} + +func matchesTerminationMessage(re *regexp.Regexp, containers []containerInfo) bool { + for _, c := range containers { + if errormatch.MatchPattern(re, c.terminationMessage) { + return true + } + } + return false +} + +// containerInfo holds the failure-relevant fields from a terminated container. +type containerInfo struct { + name string + exitCode int32 + reason string + terminationMessage string +} + +// failedContainers extracts failure info from all terminated containers +// (both regular and init containers) that exited with a non-zero code. +// Containers that exited successfully (exit code 0) are excluded to avoid +// false-positive matches on their termination messages or reasons. +func failedContainers(pod *v1.Pod) []containerInfo { + var result []containerInfo + for _, statuses := range [2][]v1.ContainerStatus{ + pod.Status.ContainerStatuses, + pod.Status.InitContainerStatuses, + } { + for _, cs := range statuses { + if cs.State.Terminated == nil || cs.State.Terminated.ExitCode == 0 { + continue + } + result = append(result, containerInfo{ + name: cs.Name, + exitCode: cs.State.Terminated.ExitCode, + reason: cs.State.Terminated.Reason, + terminationMessage: cs.State.Terminated.Message, + }) + } + } + return result +} diff --git a/internal/executor/categorizer/classifier_test.go b/internal/executor/categorizer/classifier_test.go new file mode 100644 index 00000000000..6cea4647f0f --- /dev/null +++ b/internal/executor/categorizer/classifier_test.go @@ -0,0 +1,366 @@ +package categorizer + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + v1 "k8s.io/api/core/v1" + + "github.com/armadaproject/armada/internal/common/errormatch" +) + +func TestClassify(t *testing.T) { + tests := map[string]struct { + configs []CategoryConfig + pod *v1.Pod + categories []string + }{ + "OOM condition matches": { + configs: []CategoryConfig{ + {Name: "oom", Rules: []CategoryRule{ + {OnConditions: []string{errormatch.ConditionOOMKilled}}, + }}, + }, + pod: podWithTerminatedContainer(137, errormatch.ConditionOOMKilled, ""), + categories: []string{"oom"}, + }, + "AppError condition matches": { + configs: []CategoryConfig{ + {Name: "app_error", Rules: []CategoryRule{ + {OnConditions: []string{errormatch.ConditionAppError}}, + }}, + }, + pod: podWithTerminatedContainer(1, errormatch.ConditionAppError, ""), + categories: []string{"app_error"}, + }, + "exit code In matches": { + configs: []CategoryConfig{ + {Name: "cuda_error", Rules: []CategoryRule{ + {OnExitCodes: &errormatch.ExitCodeMatcher{Operator: errormatch.ExitCodeOperatorIn, Values: []int32{74, 75}}}, + }}, + }, + pod: podWithTerminatedContainer(74, "Error", ""), + categories: []string{"cuda_error"}, + }, + "exit code NotIn matches": { + configs: []CategoryConfig{ + {Name: "unexpected", Rules: []CategoryRule{ + {OnExitCodes: &errormatch.ExitCodeMatcher{Operator: errormatch.ExitCodeOperatorNotIn, Values: []int32{1, 2}}}, + }}, + }, + pod: podWithTerminatedContainer(42, "Error", ""), + categories: []string{"unexpected"}, + }, + "termination message regex matches": { + configs: []CategoryConfig{ + {Name: "gpu_error", Rules: []CategoryRule{ + {OnTerminationMessage: &errormatch.RegexMatcher{Pattern: "(?i)cuda.*error"}}, + }}, + }, + pod: podWithTerminatedContainer(1, "Error", "CUDA memory error on device 0"), + categories: []string{"gpu_error"}, + }, + "multiple categories can match": { + configs: []CategoryConfig{ + {Name: "oom", Rules: []CategoryRule{ + {OnConditions: []string{errormatch.ConditionOOMKilled}}, + }}, + {Name: "high_exit", Rules: []CategoryRule{ + {OnExitCodes: &errormatch.ExitCodeMatcher{Operator: errormatch.ExitCodeOperatorIn, Values: []int32{137}}}, + }}, + }, + pod: podWithTerminatedContainer(137, errormatch.ConditionOOMKilled, ""), + categories: []string{"oom", "high_exit"}, + }, + "no match returns nil": { + configs: []CategoryConfig{ + {Name: "oom", Rules: []CategoryRule{ + {OnConditions: []string{errormatch.ConditionOOMKilled}}, + }}, + }, + pod: podWithTerminatedContainer(1, "Error", "normal failure"), + categories: nil, + }, + "nil pod returns nil": { + configs: []CategoryConfig{ + {Name: "oom", Rules: []CategoryRule{ + {OnConditions: []string{errormatch.ConditionOOMKilled}}, + }}, + }, + pod: nil, + categories: nil, + }, + "init container is checked": { + configs: []CategoryConfig{ + {Name: "init_fail", Rules: []CategoryRule{ + {OnExitCodes: &errormatch.ExitCodeMatcher{Operator: errormatch.ExitCodeOperatorIn, Values: []int32{1}}}, + }}, + }, + pod: &v1.Pod{ + Status: v1.PodStatus{ + Phase: v1.PodFailed, + InitContainerStatuses: []v1.ContainerStatus{ + { + Name: "init", + State: v1.ContainerState{ + Terminated: &v1.ContainerStateTerminated{ExitCode: 1, Reason: "Error"}, + }, + }, + }, + }, + }, + categories: []string{"init_fail"}, + }, + "evicted condition matches": { + configs: []CategoryConfig{ + {Name: "evicted", Rules: []CategoryRule{ + {OnConditions: []string{errormatch.ConditionEvicted}}, + }}, + }, + pod: &v1.Pod{ + Status: v1.PodStatus{ + Phase: v1.PodFailed, + Reason: errormatch.ConditionEvicted, + }, + }, + categories: []string{"evicted"}, + }, + "deadline exceeded condition matches": { + configs: []CategoryConfig{ + {Name: "timeout", Rules: []CategoryRule{ + {OnConditions: []string{errormatch.ConditionDeadlineExceeded}}, + }}, + }, + pod: &v1.Pod{ + Status: v1.PodStatus{ + Phase: v1.PodFailed, + Reason: errormatch.ConditionDeadlineExceeded, + }, + }, + categories: []string{"timeout"}, + }, + "rules within category are ORed": { + configs: []CategoryConfig{ + {Name: "infra", Rules: []CategoryRule{ + {OnConditions: []string{errormatch.ConditionOOMKilled}}, + {OnExitCodes: &errormatch.ExitCodeMatcher{Operator: errormatch.ExitCodeOperatorIn, Values: []int32{137}}}, + }}, + }, + pod: podWithTerminatedContainer(137, "Error", ""), + categories: []string{"infra"}, + }, + "exit code 0 container is skipped": { + configs: []CategoryConfig{ + {Name: "msg_match", Rules: []CategoryRule{ + {OnTerminationMessage: &errormatch.RegexMatcher{Pattern: "success"}}, + }}, + }, + pod: &v1.Pod{ + Status: v1.PodStatus{ + Phase: v1.PodFailed, + ContainerStatuses: []v1.ContainerStatus{ + { + Name: "sidecar", + State: v1.ContainerState{ + Terminated: &v1.ContainerStateTerminated{ + ExitCode: 0, + Reason: "Completed", + Message: "success message", + }, + }, + }, + { + Name: "main", + State: v1.ContainerState{ + Terminated: &v1.ContainerStateTerminated{ + ExitCode: 1, + Reason: "Error", + Message: "actual failure", + }, + }, + }, + }, + }, + }, + categories: nil, + }, + "containerName targets specific container": { + configs: []CategoryConfig{ + {Name: "main_oom", Rules: []CategoryRule{ + {ContainerName: "main", OnConditions: []string{errormatch.ConditionOOMKilled}}, + }}, + }, + pod: &v1.Pod{ + Status: v1.PodStatus{ + Phase: v1.PodFailed, + ContainerStatuses: []v1.ContainerStatus{ + {Name: "sidecar", State: v1.ContainerState{ + Terminated: &v1.ContainerStateTerminated{ExitCode: 137, Reason: errormatch.ConditionOOMKilled}, + }}, + {Name: "main", State: v1.ContainerState{ + Terminated: &v1.ContainerStateTerminated{ExitCode: 1, Reason: "Error"}, + }}, + }, + }, + }, + categories: nil, // sidecar OOMs but main does not, rule targets main only + }, + "containerName matches when container matches": { + configs: []CategoryConfig{ + {Name: "main_error", Rules: []CategoryRule{ + {ContainerName: "main", OnExitCodes: &errormatch.ExitCodeMatcher{Operator: errormatch.ExitCodeOperatorIn, Values: []int32{42}}}, + }}, + }, + pod: &v1.Pod{ + Status: v1.PodStatus{ + Phase: v1.PodFailed, + ContainerStatuses: []v1.ContainerStatus{ + {Name: "main", State: v1.ContainerState{ + Terminated: &v1.ContainerStateTerminated{ExitCode: 42, Reason: "Error"}, + }}, + }, + }, + }, + categories: []string{"main_error"}, + }, + "no containerName matches any container": { + configs: []CategoryConfig{ + {Name: "any_oom", Rules: []CategoryRule{ + {OnConditions: []string{errormatch.ConditionOOMKilled}}, + }}, + }, + pod: &v1.Pod{ + Status: v1.PodStatus{ + Phase: v1.PodFailed, + ContainerStatuses: []v1.ContainerStatus{ + {Name: "sidecar", State: v1.ContainerState{ + Terminated: &v1.ContainerStateTerminated{ExitCode: 137, Reason: errormatch.ConditionOOMKilled}, + }}, + }, + }, + }, + categories: []string{"any_oom"}, // no containerName, matches sidecar + }, + "empty config returns nil": { + configs: nil, + pod: podWithTerminatedContainer(1, "Error", ""), + categories: nil, + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + classifier, err := NewClassifier(tc.configs) + require.NoError(t, err) + result := classifier.Classify(tc.pod) + assert.Equal(t, tc.categories, result) + }) + } +} + +func TestNewClassifier_ValidationErrors(t *testing.T) { + tests := map[string]struct { + configs []CategoryConfig + errContains string + }{ + "empty name": { + configs: []CategoryConfig{ + {Name: "", Rules: []CategoryRule{ + {OnConditions: []string{errormatch.ConditionOOMKilled}}, + }}, + }, + errContains: "must have a name", + }, + "empty rule": { + configs: []CategoryConfig{{Name: "bad", Rules: []CategoryRule{{}}}}, + errContains: "must specify one of", + }, + "multiple matchers in rule": { + configs: []CategoryConfig{ + {Name: "bad", Rules: []CategoryRule{ + { + OnConditions: []string{errormatch.ConditionOOMKilled}, + OnExitCodes: &errormatch.ExitCodeMatcher{Operator: errormatch.ExitCodeOperatorIn, Values: []int32{137}}, + }, + }}, + }, + errContains: "must specify only one of", + }, + "unknown condition": { + configs: []CategoryConfig{ + {Name: "bad", Rules: []CategoryRule{ + {OnConditions: []string{"NotARealCondition"}}, + }}, + }, + errContains: "unknown condition", + }, + "invalid exit code operator": { + configs: []CategoryConfig{ + {Name: "bad", Rules: []CategoryRule{ + {OnExitCodes: &errormatch.ExitCodeMatcher{Operator: "Equals", Values: []int32{1}}}, + }}, + }, + errContains: "invalid exit code operator", + }, + "empty exit code values": { + configs: []CategoryConfig{ + {Name: "bad", Rules: []CategoryRule{ + {OnExitCodes: &errormatch.ExitCodeMatcher{Operator: errormatch.ExitCodeOperatorIn, Values: nil}}, + }}, + }, + errContains: "requires at least one value", + }, + "invalid regex": { + configs: []CategoryConfig{ + {Name: "bad", Rules: []CategoryRule{ + {OnTerminationMessage: &errormatch.RegexMatcher{Pattern: "[invalid"}}, + }}, + }, + errContains: "invalid regex", + }, + "empty rules": { + configs: []CategoryConfig{{Name: "empty", Rules: nil}}, + errContains: "must have at least one rule", + }, + "duplicate category name": { + configs: []CategoryConfig{ + {Name: "oom", Rules: []CategoryRule{ + {OnConditions: []string{errormatch.ConditionOOMKilled}}, + }}, + {Name: "oom", Rules: []CategoryRule{ + {OnExitCodes: &errormatch.ExitCodeMatcher{Operator: errormatch.ExitCodeOperatorIn, Values: []int32{137}}}, + }}, + }, + errContains: "duplicate category name", + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + _, err := NewClassifier(tc.configs) + require.Error(t, err) + assert.Contains(t, err.Error(), tc.errContains) + }) + } +} + +func podWithTerminatedContainer(exitCode int32, reason, message string) *v1.Pod { + return &v1.Pod{ + Status: v1.PodStatus{ + Phase: v1.PodFailed, + ContainerStatuses: []v1.ContainerStatus{ + { + Name: "main", + State: v1.ContainerState{ + Terminated: &v1.ContainerStateTerminated{ + ExitCode: exitCode, + Reason: reason, + Message: message, + }, + }, + }, + }, + }, + } +} diff --git a/internal/executor/categorizer/doc.go b/internal/executor/categorizer/doc.go new file mode 100644 index 00000000000..a287d246807 --- /dev/null +++ b/internal/executor/categorizer/doc.go @@ -0,0 +1,54 @@ +// Package categorizer classifies pod failures into named categories based +// on configurable rules. It runs at the executor, where full Kubernetes pod +// status is available. The resulting category names are included in the +// FailureInfo proto attached to error events. +// +// # Configuration +// +// Categories are defined in the executor config under application.errorCategories. +// Each category has a name and one or more rules. Rules within a category are +// OR'd: if any rule matches, the category is included. A pod can match multiple +// categories. +// +// Each rule uses exactly one matcher: +// - OnConditions: matches Kubernetes failure signals (OOMKilled, Evicted, DeadlineExceeded) +// - OnExitCodes: matches non-zero container exit codes using In/NotIn set operators +// - OnTerminationMessage: matches container termination messages against a regex +// +// Exit code 0 is always skipped. Both regular and init containers are checked. +// +// # Example +// +// application: +// errorCategories: +// - name: oom +// rules: +// - onConditions: ["OOMKilled"] +// - name: cuda_error +// rules: +// - onExitCodes: +// operator: In +// values: [74, 75] +// - onTerminationMessage: +// pattern: "(?i)cuda.*error" +// - name: transient_infra +// rules: +// - onConditions: ["Evicted"] +// - onExitCodes: +// operator: In +// values: [137] +// +// # Validation +// +// [NewClassifier] validates all config upfront: unknown condition strings, +// invalid exit code operators, empty value lists, and invalid regexes all +// return errors at construction time. +// +// # Usage +// +// classifier, err := categorizer.NewClassifier(config.ErrorCategories) +// if err != nil { +// // handle invalid config +// } +// categories := classifier.Classify(pod) // returns []string{"oom", "cuda_error"} or nil +package categorizer diff --git a/internal/executor/categorizer/types.go b/internal/executor/categorizer/types.go new file mode 100644 index 00000000000..84bc5a6f07d --- /dev/null +++ b/internal/executor/categorizer/types.go @@ -0,0 +1,22 @@ +package categorizer + +import "github.com/armadaproject/armada/internal/common/errormatch" + +// CategoryConfig defines a named error category with rules that match against +// pod failure signals. When any rule matches, the category name is included +// in the FailureInfo.categories field of the error event. +type CategoryConfig struct { + Name string `yaml:"name"` + Rules []CategoryRule `yaml:"rules"` +} + +// CategoryRule defines a single matching condition. Exactly one matcher must +// be set per rule (validated by NewClassifier). Rules within a category are OR'd. +// When ContainerName is set, only failures from that container are considered. +// When empty, failures from any container can match (default). +type CategoryRule struct { + ContainerName string `yaml:"containerName,omitempty"` + OnExitCodes *errormatch.ExitCodeMatcher `yaml:"onExitCodes,omitempty"` + OnTerminationMessage *errormatch.RegexMatcher `yaml:"onTerminationMessage,omitempty"` + OnConditions []string `yaml:"onConditions,omitempty"` +} diff --git a/internal/executor/configuration/types.go b/internal/executor/configuration/types.go index bc502a06940..e3ae0a3722f 100644 --- a/internal/executor/configuration/types.go +++ b/internal/executor/configuration/types.go @@ -7,6 +7,7 @@ import ( profilingconfig "github.com/armadaproject/armada/internal/common/profiling/configuration" armadaresource "github.com/armadaproject/armada/internal/common/resource" + "github.com/armadaproject/armada/internal/executor/categorizer" "github.com/armadaproject/armada/internal/executor/configuration/podchecks" "github.com/armadaproject/armada/pkg/client" ) @@ -26,6 +27,10 @@ type ApplicationConfiguration struct { // MaxLeasedJobs is the maximum jobs the executor should have in Leased state ay any one time (i.e jobs not submitted to kubernetes) // It is largely used to calculate how many new jobs to request from the scheduler MaxLeasedJobs int + // ErrorCategories defines category rules for classifying pod failures. + // Each category has a name and rules that match against exit codes, + // termination messages, or failure conditions. Empty means no categorization. + ErrorCategories []categorizer.CategoryConfig `yaml:"errorCategories"` } type PodDefaults struct { @@ -90,7 +95,8 @@ type KubernetesConfiguration struct { // When adding a fictional allocation to ensure resources allocated to non-Armada pods is at least // MinimumResourcesMarkedAllocatedToNonArmadaPodsPerNode, those resources are marked allocated at this priority. MinimumResourcesMarkedAllocatedToNonArmadaPodsPerNodePriority int32 - PodKillTimeout time.Duration + + PodKillTimeout time.Duration } type EtcdConfiguration struct { diff --git a/internal/executor/util/pod_status.go b/internal/executor/util/pod_status.go index 8f4320ebfca..b4670ed118f 100644 --- a/internal/executor/util/pod_status.go +++ b/internal/executor/util/pod_status.go @@ -5,18 +5,13 @@ import ( v1 "k8s.io/api/core/v1" + "github.com/armadaproject/armada/internal/common/errormatch" "github.com/armadaproject/armada/internal/common/util" "github.com/armadaproject/armada/pkg/armadaevents" ) var imagePullBackOffStatesSet = util.StringListToSet([]string{"ImagePullBackOff", "ErrImagePull"}) -const ( - oomKilledReason = "OOMKilled" - evictedReason = "Evicted" - deadlineExceeded = "DeadlineExceeded" -) - // TODO: Need to detect pod preemption. So that job failed events can include a string indicating a pod was preempted. // We need this so that whatever system submitted the job knows the job was preempted. @@ -46,10 +41,10 @@ func ExtractPodFailedReason(pod *v1.Pod) string { } func ExtractPodFailureCause(pod *v1.Pod) armadaevents.KubernetesReason { - if pod.Status.Reason == evictedReason { + if pod.Status.Reason == errormatch.ConditionEvicted { return armadaevents.KubernetesReason_Evicted } - if pod.Status.Reason == deadlineExceeded { + if pod.Status.Reason == errormatch.ConditionDeadlineExceeded { return armadaevents.KubernetesReason_DeadlineExceeded } @@ -114,8 +109,30 @@ func ExtractFailedPodContainerStatuses(pod *v1.Pod, clusterId string) []*armadae return returnStatuses } +// ExtractFailureInfo builds a FailureInfo proto from pod status and category labels. +// It extracts the exit code and termination message from the first failed container. +func ExtractFailureInfo(pod *v1.Pod, categories []string) *armadaevents.FailureInfo { + info := &armadaevents.FailureInfo{ + Categories: categories, + } + + if pod != nil { + // Extract exit code, termination message, and container name from the first failed container. + for _, cs := range GetPodContainerStatuses(pod) { + if cs.State.Terminated != nil && cs.State.Terminated.ExitCode != 0 { + info.ExitCode = cs.State.Terminated.ExitCode + info.TerminationMessage = cs.State.Terminated.Message + info.ContainerName = cs.Name + break + } + } + } + + return info +} + func isOom(containerStatus v1.ContainerStatus) bool { - return containerStatus.State.Terminated != nil && containerStatus.State.Terminated.Reason == oomKilledReason + return containerStatus.State.Terminated != nil && containerStatus.State.Terminated.Reason == errormatch.ConditionOOMKilled } type PodStartupStatus int diff --git a/internal/executor/util/pod_status_test.go b/internal/executor/util/pod_status_test.go index e42aa1e03b9..44701da4c58 100644 --- a/internal/executor/util/pod_status_test.go +++ b/internal/executor/util/pod_status_test.go @@ -146,6 +146,53 @@ func TestExtractFailedPodContainerStatuses(t *testing.T) { assert.Equal(t, containerStatuses[0].KubernetesReason, armadaevents.KubernetesReason_AppError) } +func TestExtractFailureInfo(t *testing.T) { + tests := map[string]struct { + pod *v1.Pod + categories []string + expectedExitCode int32 + expectedTermMsg string + expectedContainerName string + }{ + "OOM pod": { + pod: oomPod, + categories: []string{"oom"}, + expectedExitCode: 137, + expectedContainerName: "custom-error", + }, + "evicted pod": { + pod: evictedPod, + expectedExitCode: 0, + }, + "deadline exceeded pod": { + pod: deadlineExceededPod, + expectedExitCode: 0, + }, + "custom error pod": { + pod: customErrorPod, + expectedExitCode: 1, + expectedTermMsg: "Custom error", + expectedContainerName: "custom-error", + }, + "nil pod": { + pod: nil, + expectedExitCode: 0, + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + info := ExtractFailureInfo(tc.pod, tc.categories) + assert.Equal(t, tc.expectedExitCode, info.ExitCode) + assert.Equal(t, tc.categories, info.Categories) + if tc.expectedTermMsg != "" { + assert.Equal(t, tc.expectedTermMsg, info.TerminationMessage) + } + assert.Equal(t, tc.expectedContainerName, info.ContainerName) + }) + } +} + func createOomContainerStatus() v1.ContainerStatus { return v1.ContainerStatus{ Name: "custom-error", diff --git a/internal/scheduler/configuration/configuration.go b/internal/scheduler/configuration/configuration.go index d456e5822ef..bcb70863c66 100644 --- a/internal/scheduler/configuration/configuration.go +++ b/internal/scheduler/configuration/configuration.go @@ -281,6 +281,8 @@ type SchedulingConfig struct { MaximumPerQueueSchedulingBurst int `validate:"gt=0"` // Maximum number of times a job is retried before considered failed. MaxRetries uint + // RetryPolicy controls the policy-based retry engine (disabled by default). + RetryPolicy RetryPolicyConfig // List of resource names, e.g., []string{"cpu", "memory"}, to consider when computing DominantResourceFairness costs. // Dominant resource fairness is the algorithm used to assign a cost to jobs and queues. DominantResourceFairnessResourcesToConsider []string diff --git a/internal/scheduler/configuration/retry.go b/internal/scheduler/configuration/retry.go new file mode 100644 index 00000000000..75129082cdb --- /dev/null +++ b/internal/scheduler/configuration/retry.go @@ -0,0 +1,9 @@ +package configuration + +// RetryPolicyConfig controls the scheduler's retry policy behavior. +type RetryPolicyConfig struct { + // Enabled controls whether the retry policy engine is active. + Enabled bool `yaml:"enabled"` + // GlobalMaxRetries is the hard upper limit on retries across all policies. + GlobalMaxRetries uint `yaml:"globalMaxRetries"` +} diff --git a/internal/scheduler/retry/engine.go b/internal/scheduler/retry/engine.go new file mode 100644 index 00000000000..772869bc892 --- /dev/null +++ b/internal/scheduler/retry/engine.go @@ -0,0 +1,76 @@ +package retry + +import ( + "fmt" + + "github.com/armadaproject/armada/pkg/armadaevents" +) + +// Pre-allocated reason strings to avoid per-call allocations in the hot path. +var reasonForAction = map[Action]string{ + ActionFail: "matched rule: Fail", + ActionRetry: "matched rule: Retry", +} + +const reasonDefault = "no rule matched, using default action" + +// Engine evaluates retry policies against job run errors to decide whether +// a job should be retried or permanently failed. +type Engine struct { + globalMaxRetries uint +} + +// NewEngine creates a retry engine with the given hard upper limit on retries. +func NewEngine(globalMaxRetries uint) *Engine { + return &Engine{globalMaxRetries: globalMaxRetries} +} + +// Evaluate applies the policy rules to the given error and returns a retry decision. +// +// Parameters: +// - policy: the retry policy to evaluate (must not be nil) +// - runError: the error from the failed run (may be nil) +// - failureCount: how many times this job has already failed under this policy +// - totalRuns: the total number of runs for this job across all policies +func (e *Engine) Evaluate(policy *Policy, runError *armadaevents.Error, failureCount uint32, totalRuns uint) Result { + if runError == nil { + return Result{ShouldRetry: false, Reason: "no error information available"} + } + + if e.globalMaxRetries > 0 && totalRuns >= e.globalMaxRetries { + return Result{ + ShouldRetry: false, + Reason: fmt.Sprintf("global max retries exceeded (%d/%d)", totalRuns, e.globalMaxRetries), + } + } + + fi := runError.GetFailureInfo() + condition := extractCondition(runError) + exitCode := extractExitCode(runError, fi) + terminationMessage := extractTerminationMessage(runError, fi) + categories := extractCategories(fi) + + matched := matchRules(policy.Rules, condition, exitCode, terminationMessage, categories) + + action := policy.DefaultAction + matchDesc := reasonDefault + if matched != nil { + action = matched.Action + matchDesc = reasonForAction[matched.Action] + } + + if action == ActionFail { + return Result{ShouldRetry: false, Reason: matchDesc} + } + + // Action is Retry. Check the policy-level retry limit. + // RetryLimit 0 means unlimited (subject to global cap). + if policy.RetryLimit > 0 && failureCount >= policy.RetryLimit { + return Result{ + ShouldRetry: false, + Reason: fmt.Sprintf("policy retry limit exceeded (%d/%d)", failureCount, policy.RetryLimit), + } + } + + return Result{ShouldRetry: true, Reason: matchDesc} +} diff --git a/internal/scheduler/retry/engine_test.go b/internal/scheduler/retry/engine_test.go new file mode 100644 index 00000000000..f6a1c7abf5c --- /dev/null +++ b/internal/scheduler/retry/engine_test.go @@ -0,0 +1,552 @@ +package retry + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/armadaproject/armada/internal/common/errormatch" + "github.com/armadaproject/armada/pkg/armadaevents" +) + +func makeOOMError() *armadaevents.Error { + return &armadaevents.Error{ + Reason: &armadaevents.Error_PodError{ + PodError: &armadaevents.PodError{ + KubernetesReason: armadaevents.KubernetesReason_OOM, + ContainerErrors: []*armadaevents.ContainerError{{ExitCode: 137, Message: "OOMKilled"}}, + }, + }, + FailureInfo: &armadaevents.FailureInfo{ + ExitCode: 137, + TerminationMessage: "OOMKilled", + Categories: []string{"infrastructure"}, + }, + } +} + +func makeAppError(exitCode int32, message string) *armadaevents.Error { + return &armadaevents.Error{ + Reason: &armadaevents.Error_PodError{ + PodError: &armadaevents.PodError{ + KubernetesReason: armadaevents.KubernetesReason_AppError, + ContainerErrors: []*armadaevents.ContainerError{{ExitCode: exitCode, Message: message}}, + }, + }, + FailureInfo: &armadaevents.FailureInfo{ + ExitCode: exitCode, + TerminationMessage: message, + }, + } +} + +func compilePolicy(t *testing.T, p *Policy) *Policy { + t.Helper() + require.NoError(t, CompileRules(p.Rules)) + return p +} + +func TestEngine_Evaluate(t *testing.T) { + tests := map[string]struct { + globalMax uint + policy *Policy + runError *armadaevents.Error + failureCount uint32 + totalRuns uint + expected Result + }{ + "condition match OOMKilled, action Fail": { + globalMax: 10, + policy: &Policy{ + Name: "test", + DefaultAction: ActionRetry, + Rules: []Rule{ + {Action: ActionFail, OnConditions: []string{errormatch.ConditionOOMKilled}}, + }, + }, + runError: makeOOMError(), + failureCount: 0, + totalRuns: 1, + expected: Result{ShouldRetry: false, Reason: "matched rule: Fail"}, + }, + "condition match Evicted, action Retry": { + globalMax: 10, + policy: &Policy{ + Name: "test", + DefaultAction: ActionFail, + Rules: []Rule{ + {Action: ActionRetry, OnConditions: []string{errormatch.ConditionEvicted}}, + }, + }, + runError: &armadaevents.Error{ + Reason: &armadaevents.Error_PodError{ + PodError: &armadaevents.PodError{KubernetesReason: armadaevents.KubernetesReason_Evicted}, + }, + }, + failureCount: 0, + totalRuns: 1, + expected: Result{ShouldRetry: true, Reason: "matched rule: Retry"}, + }, + "condition match DeadlineExceeded": { + globalMax: 10, + policy: &Policy{ + Name: "test", + DefaultAction: ActionRetry, + Rules: []Rule{ + {Action: ActionFail, OnConditions: []string{errormatch.ConditionDeadlineExceeded}}, + }, + }, + runError: &armadaevents.Error{ + Reason: &armadaevents.Error_PodError{ + PodError: &armadaevents.PodError{KubernetesReason: armadaevents.KubernetesReason_DeadlineExceeded}, + }, + }, + failureCount: 0, + totalRuns: 1, + expected: Result{ShouldRetry: false, Reason: "matched rule: Fail"}, + }, + "condition match Preempted": { + globalMax: 10, + policy: &Policy{ + Name: "test", + DefaultAction: ActionFail, + Rules: []Rule{ + {Action: ActionRetry, OnConditions: []string{errormatch.ConditionPreempted}}, + }, + }, + runError: &armadaevents.Error{ + Reason: &armadaevents.Error_JobRunPreemptedError{ + JobRunPreemptedError: &armadaevents.JobRunPreemptedError{}, + }, + }, + failureCount: 0, + totalRuns: 1, + expected: Result{ShouldRetry: true, Reason: "matched rule: Retry"}, + }, + "condition match LeaseReturned": { + globalMax: 10, + policy: &Policy{ + Name: "test", + DefaultAction: ActionFail, + Rules: []Rule{ + {Action: ActionRetry, OnConditions: []string{errormatch.ConditionLeaseReturned}}, + }, + }, + runError: &armadaevents.Error{ + Reason: &armadaevents.Error_PodLeaseReturned{ + PodLeaseReturned: &armadaevents.PodLeaseReturned{}, + }, + }, + failureCount: 0, + totalRuns: 1, + expected: Result{ShouldRetry: true, Reason: "matched rule: Retry"}, + }, + "condition match AppError": { + globalMax: 10, + policy: &Policy{ + Name: "test", + DefaultAction: ActionRetry, + Rules: []Rule{ + {Action: ActionFail, OnConditions: []string{errormatch.ConditionAppError}}, + }, + }, + runError: makeAppError(1, "crash"), + failureCount: 0, + totalRuns: 1, + expected: Result{ShouldRetry: false, Reason: "matched rule: Fail"}, + }, + "exit code In match": { + globalMax: 10, + policy: &Policy{ + Name: "test", + DefaultAction: ActionRetry, + Rules: []Rule{ + { + Action: ActionFail, + OnExitCodes: &errormatch.ExitCodeMatcher{Operator: errormatch.ExitCodeOperatorIn, Values: []int32{42, 43}}, + }, + }, + }, + runError: makeAppError(42, ""), + failureCount: 0, + totalRuns: 1, + expected: Result{ShouldRetry: false, Reason: "matched rule: Fail"}, + }, + "exit code NotIn match": { + globalMax: 10, + policy: &Policy{ + Name: "test", + DefaultAction: ActionFail, + Rules: []Rule{ + { + Action: ActionRetry, + OnExitCodes: &errormatch.ExitCodeMatcher{Operator: errormatch.ExitCodeOperatorNotIn, Values: []int32{42}}, + }, + }, + }, + runError: makeAppError(1, ""), + failureCount: 0, + totalRuns: 1, + expected: Result{ShouldRetry: true, Reason: "matched rule: Retry"}, + }, + "termination message regex match": { + globalMax: 10, + policy: &Policy{ + Name: "test", + DefaultAction: ActionRetry, + Rules: []Rule{ + { + Action: ActionFail, + OnTerminationMessage: &errormatch.RegexMatcher{Pattern: "(?i)cuda.*error"}, + }, + }, + }, + runError: makeAppError(1, "CUDA memory error on device 0"), + failureCount: 0, + totalRuns: 1, + expected: Result{ShouldRetry: false, Reason: "matched rule: Fail"}, + }, + "termination message regex no match": { + globalMax: 10, + policy: &Policy{ + Name: "test", + DefaultAction: ActionRetry, + Rules: []Rule{ + { + Action: ActionFail, + OnTerminationMessage: &errormatch.RegexMatcher{Pattern: "(?i)cuda.*error"}, + }, + }, + }, + runError: makeAppError(1, "segfault"), + failureCount: 0, + totalRuns: 1, + expected: Result{ShouldRetry: true, Reason: "no rule matched, using default action"}, + }, + "category match with overlap": { + globalMax: 10, + policy: &Policy{ + Name: "test", + DefaultAction: ActionRetry, + Rules: []Rule{ + {Action: ActionFail, OnCategories: []string{"gpu", "network"}}, + }, + }, + runError: &armadaevents.Error{ + Reason: &armadaevents.Error_PodError{ + PodError: &armadaevents.PodError{KubernetesReason: armadaevents.KubernetesReason_AppError}, + }, + FailureInfo: &armadaevents.FailureInfo{Categories: []string{"gpu", "transient"}}, + }, + failureCount: 0, + totalRuns: 1, + expected: Result{ShouldRetry: false, Reason: "matched rule: Fail"}, + }, + "category match without overlap": { + globalMax: 10, + policy: &Policy{ + Name: "test", + DefaultAction: ActionRetry, + Rules: []Rule{ + {Action: ActionFail, OnCategories: []string{"network"}}, + }, + }, + runError: &armadaevents.Error{ + Reason: &armadaevents.Error_PodError{ + PodError: &armadaevents.PodError{KubernetesReason: armadaevents.KubernetesReason_AppError}, + }, + FailureInfo: &armadaevents.FailureInfo{Categories: []string{"gpu", "transient"}}, + }, + failureCount: 0, + totalRuns: 1, + expected: Result{ShouldRetry: true, Reason: "no rule matched, using default action"}, + }, + "first match wins": { + globalMax: 10, + policy: &Policy{ + Name: "test", + DefaultAction: ActionFail, + Rules: []Rule{ + {Action: ActionRetry, OnConditions: []string{errormatch.ConditionAppError}}, + {Action: ActionFail, OnConditions: []string{errormatch.ConditionAppError}}, + }, + }, + runError: makeAppError(1, "crash"), + failureCount: 0, + totalRuns: 1, + expected: Result{ShouldRetry: true, Reason: "matched rule: Retry"}, + }, + "no match returns DefaultAction Fail": { + globalMax: 10, + policy: &Policy{ + Name: "test", + DefaultAction: ActionFail, + Rules: []Rule{ + {Action: ActionRetry, OnConditions: []string{errormatch.ConditionOOMKilled}}, + }, + }, + runError: makeAppError(1, ""), + failureCount: 0, + totalRuns: 1, + expected: Result{ShouldRetry: false, Reason: "no rule matched, using default action"}, + }, + "no match returns DefaultAction Retry": { + globalMax: 10, + policy: &Policy{ + Name: "test", + DefaultAction: ActionRetry, + Rules: []Rule{ + {Action: ActionFail, OnConditions: []string{errormatch.ConditionOOMKilled}}, + }, + }, + runError: makeAppError(1, ""), + failureCount: 0, + totalRuns: 1, + expected: Result{ShouldRetry: true, Reason: "no rule matched, using default action"}, + }, + "global cap exceeded": { + globalMax: 5, + policy: &Policy{ + Name: "test", + DefaultAction: ActionRetry, + }, + runError: makeAppError(1, "crash"), + failureCount: 0, + totalRuns: 5, + expected: Result{ShouldRetry: false, Reason: "global max retries exceeded (5/5)"}, + }, + "retry limit exceeded": { + globalMax: 100, + policy: &Policy{ + Name: "test", + RetryLimit: 3, + DefaultAction: ActionRetry, + }, + runError: makeAppError(1, "crash"), + failureCount: 3, + totalRuns: 3, + expected: Result{ShouldRetry: false, Reason: "policy retry limit exceeded (3/3)"}, + }, + "retry limit 0 means unlimited within global cap": { + globalMax: 100, + policy: &Policy{ + Name: "test", + RetryLimit: 0, + DefaultAction: ActionRetry, + }, + runError: makeAppError(1, "crash"), + failureCount: 50, + totalRuns: 50, + expected: Result{ShouldRetry: true, Reason: "no rule matched, using default action"}, + }, + "nil error returns fail": { + globalMax: 10, + policy: &Policy{ + Name: "test", + DefaultAction: ActionRetry, + }, + runError: nil, + failureCount: 0, + totalRuns: 1, + expected: Result{ShouldRetry: false, Reason: "no error information available"}, + }, + "nil FailureInfo falls back to ContainerError fields": { + globalMax: 10, + policy: &Policy{ + Name: "test", + DefaultAction: ActionRetry, + Rules: []Rule{ + { + Action: ActionFail, + OnExitCodes: &errormatch.ExitCodeMatcher{Operator: errormatch.ExitCodeOperatorIn, Values: []int32{42}}, + }, + }, + }, + runError: &armadaevents.Error{ + Reason: &armadaevents.Error_PodError{ + PodError: &armadaevents.PodError{ + KubernetesReason: armadaevents.KubernetesReason_AppError, + ContainerErrors: []*armadaevents.ContainerError{{ExitCode: 42, Message: "custom exit"}}, + }, + }, + }, + failureCount: 0, + totalRuns: 1, + expected: Result{ShouldRetry: false, Reason: "matched rule: Fail"}, + }, + "AND logic, all fields must match": { + globalMax: 10, + policy: &Policy{ + Name: "test", + DefaultAction: ActionRetry, + Rules: []Rule{ + { + Action: ActionFail, + OnConditions: []string{errormatch.ConditionAppError}, + OnExitCodes: &errormatch.ExitCodeMatcher{Operator: errormatch.ExitCodeOperatorIn, Values: []int32{42}}, + }, + }, + }, + // Condition matches but exit code does not + runError: makeAppError(1, ""), + failureCount: 0, + totalRuns: 1, + expected: Result{ShouldRetry: true, Reason: "no rule matched, using default action"}, + }, + "AND logic, both fields match": { + globalMax: 10, + policy: &Policy{ + Name: "test", + DefaultAction: ActionRetry, + Rules: []Rule{ + { + Action: ActionFail, + OnConditions: []string{errormatch.ConditionAppError}, + OnExitCodes: &errormatch.ExitCodeMatcher{Operator: errormatch.ExitCodeOperatorIn, Values: []int32{42}}, + }, + }, + }, + runError: makeAppError(42, ""), + failureCount: 0, + totalRuns: 1, + expected: Result{ShouldRetry: false, Reason: "matched rule: Fail"}, + }, + "empty rules returns DefaultAction": { + globalMax: 10, + policy: &Policy{ + Name: "test", + DefaultAction: ActionFail, + Rules: []Rule{}, + }, + runError: makeAppError(1, "crash"), + failureCount: 0, + totalRuns: 1, + expected: Result{ShouldRetry: false, Reason: "no rule matched, using default action"}, + }, + "globalMaxRetries 0 means unlimited": { + globalMax: 0, + policy: &Policy{ + Name: "test", + DefaultAction: ActionRetry, + }, + runError: makeAppError(1, "crash"), + failureCount: 100, + totalRuns: 100, + expected: Result{ShouldRetry: true, Reason: "no rule matched, using default action"}, + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + tc.policy = compilePolicy(t, tc.policy) + engine := NewEngine(tc.globalMax) + result := engine.Evaluate(tc.policy, tc.runError, tc.failureCount, tc.totalRuns) + assert.Equal(t, tc.expected, result) + }) + } +} + +func TestCompileRules_InvalidRegex(t *testing.T) { + rules := []Rule{ + { + Action: ActionFail, + OnTerminationMessage: &errormatch.RegexMatcher{Pattern: "[invalid"}, + }, + } + err := CompileRules(rules) + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to compile termination message pattern") +} + +func TestValidatePolicy(t *testing.T) { + tests := map[string]struct { + policy Policy + expectError string + }{ + "valid policy with Fail default": { + policy: Policy{ + Name: "test", + DefaultAction: ActionFail, + Rules: []Rule{ + {Action: ActionRetry, OnConditions: []string{"OOMKilled"}}, + }, + }, + }, + "valid policy with Retry default": { + policy: Policy{ + Name: "test", + DefaultAction: ActionRetry, + }, + }, + "empty DefaultAction rejected": { + policy: Policy{Name: "test", DefaultAction: ""}, + expectError: "DefaultAction must be", + }, + "unknown DefaultAction rejected": { + policy: Policy{Name: "test", DefaultAction: "Skip"}, + expectError: "DefaultAction must be", + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + err := ValidatePolicy(tc.policy) + if tc.expectError != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tc.expectError) + } else { + assert.NoError(t, err) + } + }) + } +} + +func TestCompileRules_Validation(t *testing.T) { + tests := map[string]struct { + rules []Rule + expectError string + }{ + "empty termination message pattern": { + rules: []Rule{ + { + Action: ActionFail, + OnTerminationMessage: &errormatch.RegexMatcher{Pattern: ""}, + }, + }, + expectError: "rule 0: OnTerminationMessage pattern must not be empty", + }, + "empty rule with no match fields": { + rules: []Rule{ + {Action: ActionFail}, + }, + expectError: "rule 0: must have at least one match field", + }, + "invalid exit code operator": { + rules: []Rule{ + { + Action: ActionFail, + OnExitCodes: &errormatch.ExitCodeMatcher{Operator: "BadOp", Values: []int32{1}}, + }, + }, + expectError: "rule 0: OnExitCodes operator must be", + }, + "empty exit code values": { + rules: []Rule{ + { + Action: ActionFail, + OnExitCodes: &errormatch.ExitCodeMatcher{Operator: errormatch.ExitCodeOperatorIn, Values: []int32{}}, + }, + }, + expectError: "rule 0: OnExitCodes values must not be empty", + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + err := CompileRules(tc.rules) + require.Error(t, err) + assert.Contains(t, err.Error(), tc.expectError) + }) + } +} diff --git a/internal/scheduler/retry/extract.go b/internal/scheduler/retry/extract.go new file mode 100644 index 00000000000..e4cd269f17b --- /dev/null +++ b/internal/scheduler/retry/extract.go @@ -0,0 +1,84 @@ +package retry + +import ( + "github.com/armadaproject/armada/internal/common/errormatch" + "github.com/armadaproject/armada/pkg/armadaevents" +) + +// extractCondition derives a human-readable condition string from the Error oneof. +// Returns empty string for unrecognized error types. +func extractCondition(err *armadaevents.Error) string { + if err == nil { + return "" + } + switch reason := err.Reason.(type) { + case *armadaevents.Error_PodError: + if reason.PodError == nil { + return "" + } + switch reason.PodError.KubernetesReason { + case armadaevents.KubernetesReason_OOM: + return errormatch.ConditionOOMKilled + case armadaevents.KubernetesReason_Evicted: + return errormatch.ConditionEvicted + case armadaevents.KubernetesReason_DeadlineExceeded: + return errormatch.ConditionDeadlineExceeded + case armadaevents.KubernetesReason_AppError: + return errormatch.ConditionAppError + default: + return "" + } + case *armadaevents.Error_JobRunPreemptedError: + return errormatch.ConditionPreempted + case *armadaevents.Error_PodLeaseReturned: + return errormatch.ConditionLeaseReturned + default: + return "" + } +} + +// extractExitCode returns the exit code from FailureInfo if available, +// falling back to the first ContainerError in a PodError. +func extractExitCode(err *armadaevents.Error, fi *armadaevents.FailureInfo) int32 { + if err == nil { + return 0 + } + if fi != nil && fi.ExitCode != 0 { + return fi.ExitCode + } + if pe := err.GetPodError(); pe != nil { + for _, ce := range pe.ContainerErrors { + if ce != nil && ce.ExitCode != 0 { + return ce.ExitCode + } + } + } + return 0 +} + +// extractTerminationMessage returns the termination message from FailureInfo if available, +// falling back to the first ContainerError.Message in a PodError. +func extractTerminationMessage(err *armadaevents.Error, fi *armadaevents.FailureInfo) string { + if err == nil { + return "" + } + if fi != nil && fi.TerminationMessage != "" { + return fi.TerminationMessage + } + if pe := err.GetPodError(); pe != nil { + for _, ce := range pe.ContainerErrors { + if ce != nil && ce.Message != "" { + return ce.Message + } + } + } + return "" +} + +// extractCategories returns category labels from FailureInfo. +func extractCategories(fi *armadaevents.FailureInfo) []string { + if fi == nil { + return nil + } + return fi.Categories +} diff --git a/internal/scheduler/retry/extract_test.go b/internal/scheduler/retry/extract_test.go new file mode 100644 index 00000000000..25e2c4f1fbd --- /dev/null +++ b/internal/scheduler/retry/extract_test.go @@ -0,0 +1,230 @@ +package retry + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/armadaproject/armada/internal/common/errormatch" + "github.com/armadaproject/armada/pkg/armadaevents" +) + +func TestExtractCondition(t *testing.T) { + tests := map[string]struct { + err *armadaevents.Error + expected string + }{ + "nil error": { + err: nil, + expected: "", + }, + "PodError OOM": { + err: &armadaevents.Error{ + Reason: &armadaevents.Error_PodError{ + PodError: &armadaevents.PodError{KubernetesReason: armadaevents.KubernetesReason_OOM}, + }, + }, + expected: errormatch.ConditionOOMKilled, + }, + "PodError Evicted": { + err: &armadaevents.Error{ + Reason: &armadaevents.Error_PodError{ + PodError: &armadaevents.PodError{KubernetesReason: armadaevents.KubernetesReason_Evicted}, + }, + }, + expected: errormatch.ConditionEvicted, + }, + "PodError DeadlineExceeded": { + err: &armadaevents.Error{ + Reason: &armadaevents.Error_PodError{ + PodError: &armadaevents.PodError{KubernetesReason: armadaevents.KubernetesReason_DeadlineExceeded}, + }, + }, + expected: errormatch.ConditionDeadlineExceeded, + }, + "PodError AppError": { + err: &armadaevents.Error{ + Reason: &armadaevents.Error_PodError{ + PodError: &armadaevents.PodError{KubernetesReason: armadaevents.KubernetesReason_AppError}, + }, + }, + expected: errormatch.ConditionAppError, + }, + "PodError nil inner": { + err: &armadaevents.Error{ + Reason: &armadaevents.Error_PodError{PodError: nil}, + }, + expected: "", + }, + "Preempted": { + err: &armadaevents.Error{ + Reason: &armadaevents.Error_JobRunPreemptedError{ + JobRunPreemptedError: &armadaevents.JobRunPreemptedError{}, + }, + }, + expected: errormatch.ConditionPreempted, + }, + "LeaseReturned": { + err: &armadaevents.Error{ + Reason: &armadaevents.Error_PodLeaseReturned{ + PodLeaseReturned: &armadaevents.PodLeaseReturned{}, + }, + }, + expected: errormatch.ConditionLeaseReturned, + }, + "unrecognized error type": { + err: &armadaevents.Error{ + Reason: &armadaevents.Error_LeaseExpired{ + LeaseExpired: &armadaevents.LeaseExpired{}, + }, + }, + expected: "", + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + assert.Equal(t, tc.expected, extractCondition(tc.err)) + }) + } +} + +func fiFromErr(err *armadaevents.Error) *armadaevents.FailureInfo { + if err == nil { + return nil + } + return err.GetFailureInfo() +} + +func TestExtractExitCode(t *testing.T) { + tests := map[string]struct { + err *armadaevents.Error + expected int32 + }{ + "nil error": { + err: nil, + expected: 0, + }, + "from FailureInfo": { + err: &armadaevents.Error{ + FailureInfo: &armadaevents.FailureInfo{ExitCode: 137}, + Reason: &armadaevents.Error_PodError{ + PodError: &armadaevents.PodError{ + ContainerErrors: []*armadaevents.ContainerError{{ExitCode: 1}}, + }, + }, + }, + expected: 137, + }, + "fallback to ContainerError": { + err: &armadaevents.Error{ + Reason: &armadaevents.Error_PodError{ + PodError: &armadaevents.PodError{ + ContainerErrors: []*armadaevents.ContainerError{{ExitCode: 42}}, + }, + }, + }, + expected: 42, + }, + "nil FailureInfo with zero exit code falls back": { + err: &armadaevents.Error{ + FailureInfo: &armadaevents.FailureInfo{ExitCode: 0}, + Reason: &armadaevents.Error_PodError{ + PodError: &armadaevents.PodError{ + ContainerErrors: []*armadaevents.ContainerError{{ExitCode: 99}}, + }, + }, + }, + expected: 99, + }, + "no exit code anywhere": { + err: &armadaevents.Error{ + Reason: &armadaevents.Error_PodError{ + PodError: &armadaevents.PodError{}, + }, + }, + expected: 0, + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + assert.Equal(t, tc.expected, extractExitCode(tc.err, fiFromErr(tc.err))) + }) + } +} + +func TestExtractTerminationMessage(t *testing.T) { + tests := map[string]struct { + err *armadaevents.Error + expected string + }{ + "nil error": { + err: nil, + expected: "", + }, + "from FailureInfo": { + err: &armadaevents.Error{ + FailureInfo: &armadaevents.FailureInfo{TerminationMessage: "CUDA error"}, + Reason: &armadaevents.Error_PodError{ + PodError: &armadaevents.PodError{ + ContainerErrors: []*armadaevents.ContainerError{{Message: "container msg"}}, + }, + }, + }, + expected: "CUDA error", + }, + "fallback to ContainerError": { + err: &armadaevents.Error{ + Reason: &armadaevents.Error_PodError{ + PodError: &armadaevents.PodError{ + ContainerErrors: []*armadaevents.ContainerError{{Message: "segfault"}}, + }, + }, + }, + expected: "segfault", + }, + "no message anywhere": { + err: &armadaevents.Error{ + Reason: &armadaevents.Error_PodError{ + PodError: &armadaevents.PodError{}, + }, + }, + expected: "", + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + assert.Equal(t, tc.expected, extractTerminationMessage(tc.err, fiFromErr(tc.err))) + }) + } +} + +func TestExtractCategories(t *testing.T) { + tests := map[string]struct { + err *armadaevents.Error + expected []string + }{ + "nil error": { + err: nil, + expected: nil, + }, + "from FailureInfo": { + err: &armadaevents.Error{ + FailureInfo: &armadaevents.FailureInfo{Categories: []string{"gpu", "transient"}}, + }, + expected: []string{"gpu", "transient"}, + }, + "nil FailureInfo": { + err: &armadaevents.Error{}, + expected: nil, + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + assert.Equal(t, tc.expected, extractCategories(fiFromErr(tc.err))) + }) + } +} diff --git a/internal/scheduler/retry/matcher.go b/internal/scheduler/retry/matcher.go new file mode 100644 index 00000000000..56dc3ebf4bc --- /dev/null +++ b/internal/scheduler/retry/matcher.go @@ -0,0 +1,44 @@ +package retry + +import ( + "slices" + + "github.com/armadaproject/armada/internal/common/errormatch" +) + +// matchRule returns true if all non-empty match fields on the rule match the given values. +// Empty/nil match fields are ignored (they match anything). +func matchRule(rule *Rule, condition string, exitCode int32, terminationMessage string, categories []string) bool { + if len(rule.OnConditions) > 0 { + if !slices.Contains(rule.OnConditions, condition) { + return false + } + } + if rule.OnExitCodes != nil { + if !errormatch.MatchExitCode(rule.OnExitCodes, exitCode) { + return false + } + } + if rule.compiledPattern != nil { + if !errormatch.MatchPattern(rule.compiledPattern, terminationMessage) { + return false + } + } + if len(rule.OnCategories) > 0 { + if !slices.ContainsFunc(rule.OnCategories, func(c string) bool { return slices.Contains(categories, c) }) { + return false + } + } + return true +} + +// matchRules iterates rules in order and returns a pointer to the first +// matching rule. Returns nil if no rule matches. +func matchRules(rules []Rule, condition string, exitCode int32, terminationMessage string, categories []string) *Rule { + for i := range rules { + if matchRule(&rules[i], condition, exitCode, terminationMessage, categories) { + return &rules[i] + } + } + return nil +} diff --git a/internal/scheduler/retry/types.go b/internal/scheduler/retry/types.go new file mode 100644 index 00000000000..de077b05ce0 --- /dev/null +++ b/internal/scheduler/retry/types.go @@ -0,0 +1,103 @@ +package retry + +import ( + "fmt" + "regexp" + + "github.com/armadaproject/armada/internal/common/errormatch" +) + +// Action is the decision made by the retry engine. +type Action string + +const ( + ActionFail Action = "Fail" + ActionRetry Action = "Retry" +) + +// Policy is the internal representation used by the engine. +// Converted from the api.RetryPolicy proto at cache refresh time (in a later PR). +type Policy struct { + Name string + RetryLimit uint32 + DefaultAction Action + Rules []Rule +} + +// Rule defines a single matching rule within a policy. +// All non-nil match fields must match for the rule to apply (AND logic across fields). +// OnCategories uses OR within the list: the rule matches if the job has any of the listed categories. +type Rule struct { + Action Action + OnConditions []string + OnExitCodes *errormatch.ExitCodeMatcher + OnTerminationMessage *errormatch.RegexMatcher + OnCategories []string + + // compiledPattern holds the pre-compiled regex from OnTerminationMessage. + // Populated by CompileRules; nil when OnTerminationMessage is nil. + compiledPattern *regexp.Regexp +} + +// Result is the output of the retry engine evaluation. +type Result struct { + ShouldRetry bool + Reason string +} + +// ValidatePolicy checks that a policy has valid fields. +func ValidatePolicy(p Policy) error { + if p.DefaultAction != ActionFail && p.DefaultAction != ActionRetry { + return fmt.Errorf("DefaultAction must be %q or %q, got %q", ActionFail, ActionRetry, p.DefaultAction) + } + return CompileRules(p.Rules) +} + +// CompileRules validates and compiles regex patterns in all rules so they are ready for matching. +// Call this once when policies are loaded, not on every evaluation. +func CompileRules(rules []Rule) error { + for i := range rules { + if err := validateRule(i, rules[i]); err != nil { + return err + } + if rules[i].OnTerminationMessage != nil { + re, err := regexp.Compile(rules[i].OnTerminationMessage.Pattern) + if err != nil { + return fmt.Errorf("failed to compile termination message pattern %q in rule %d: %w", + rules[i].OnTerminationMessage.Pattern, i, err) + } + rules[i].compiledPattern = re + } + } + return nil +} + +// validateRule checks that a rule has at least one match field and that +// configured matchers have valid parameters. +func validateRule(index int, rule Rule) error { + hasConditions := len(rule.OnConditions) > 0 + hasExitCodes := rule.OnExitCodes != nil + hasTermMsg := rule.OnTerminationMessage != nil + hasCategories := len(rule.OnCategories) > 0 + + if !hasConditions && !hasExitCodes && !hasTermMsg && !hasCategories { + return fmt.Errorf("rule %d: must have at least one match field (OnConditions, OnExitCodes, OnTerminationMessage, or OnCategories)", index) + } + + if hasExitCodes { + op := rule.OnExitCodes.Operator + if op != errormatch.ExitCodeOperatorIn && op != errormatch.ExitCodeOperatorNotIn { + return fmt.Errorf("rule %d: OnExitCodes operator must be %q or %q, got %q", + index, errormatch.ExitCodeOperatorIn, errormatch.ExitCodeOperatorNotIn, op) + } + if len(rule.OnExitCodes.Values) == 0 { + return fmt.Errorf("rule %d: OnExitCodes values must not be empty", index) + } + } + + if hasTermMsg && rule.OnTerminationMessage.Pattern == "" { + return fmt.Errorf("rule %d: OnTerminationMessage pattern must not be empty", index) + } + + return nil +} diff --git a/pkg/armadaevents/events.pb.go b/pkg/armadaevents/events.pb.go index 84f2a69c0d0..80ce783fb49 100644 --- a/pkg/armadaevents/events.pb.go +++ b/pkg/armadaevents/events.pb.go @@ -2406,6 +2406,9 @@ type Error struct { // *Error_JobRejected // *Error_ReconciliationError Reason isError_Reason `protobuf_oneof:"reason"` + // Structured failure metadata (exit code, condition, categories). + // Set by the executor for pod failures; nil for non-pod errors. + FailureInfo *FailureInfo `protobuf:"bytes,15,opt,name=failure_info,json=failureInfo,proto3" json:"failureInfo,omitempty"` } func (m *Error) Reset() { *m = Error{} } @@ -2584,6 +2587,13 @@ func (m *Error) GetReconciliationError() *ReconciliationError { return nil } +func (m *Error) GetFailureInfo() *FailureInfo { + if m != nil { + return m.FailureInfo + } + return nil +} + // XXX_OneofWrappers is for the internal use of the proto package. func (*Error) XXX_OneofWrappers() []interface{} { return []interface{}{ @@ -3331,6 +3341,81 @@ func (m *ReconciliationError) GetMessage() string { return "" } +// Structured failure metadata extracted from pod status at the executor. +// Attached to Error messages to enable observability in Lookout and +// category-based filtering. +type FailureInfo struct { + // Exit code of the first failed container (0 if no container terminated with failure). + ExitCode int32 `protobuf:"varint,1,opt,name=exit_code,json=exitCode,proto3" json:"exitCode,omitempty"` + // Termination message from the first failed container. + TerminationMessage string `protobuf:"bytes,2,opt,name=termination_message,json=terminationMessage,proto3" json:"terminationMessage,omitempty"` + // Executor-assigned category labels, matched from ErrorCategories config. + Categories []string `protobuf:"bytes,3,rep,name=categories,proto3" json:"categories,omitempty"` + // Name of the container that exit_code and termination_message were extracted from. + ContainerName string `protobuf:"bytes,4,opt,name=container_name,json=containerName,proto3" json:"containerName,omitempty"` +} + +func (m *FailureInfo) Reset() { *m = FailureInfo{} } +func (m *FailureInfo) String() string { return proto.CompactTextString(m) } +func (*FailureInfo) ProtoMessage() {} +func (*FailureInfo) Descriptor() ([]byte, []int) { + return fileDescriptor_6aab92ca59e015f8, []int{40} +} +func (m *FailureInfo) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *FailureInfo) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_FailureInfo.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *FailureInfo) XXX_Merge(src proto.Message) { + xxx_messageInfo_FailureInfo.Merge(m, src) +} +func (m *FailureInfo) XXX_Size() int { + return m.Size() +} +func (m *FailureInfo) XXX_DiscardUnknown() { + xxx_messageInfo_FailureInfo.DiscardUnknown(m) +} + +var xxx_messageInfo_FailureInfo proto.InternalMessageInfo + +func (m *FailureInfo) GetExitCode() int32 { + if m != nil { + return m.ExitCode + } + return 0 +} + +func (m *FailureInfo) GetTerminationMessage() string { + if m != nil { + return m.TerminationMessage + } + return "" +} + +func (m *FailureInfo) GetCategories() []string { + if m != nil { + return m.Categories + } + return nil +} + +func (m *FailureInfo) GetContainerName() string { + if m != nil { + return m.ContainerName + } + return "" +} + // Message to indicate that a JobRun has been preempted. type JobRunPreempted struct { PreemptedJobId string `protobuf:"bytes,5,opt,name=preempted_job_id,json=preemptedJobId,proto3" json:"preemptedJobId,omitempty"` @@ -3342,7 +3427,7 @@ func (m *JobRunPreempted) Reset() { *m = JobRunPreempted{} } func (m *JobRunPreempted) String() string { return proto.CompactTextString(m) } func (*JobRunPreempted) ProtoMessage() {} func (*JobRunPreempted) Descriptor() ([]byte, []int) { - return fileDescriptor_6aab92ca59e015f8, []int{40} + return fileDescriptor_6aab92ca59e015f8, []int{41} } func (m *JobRunPreempted) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3404,7 +3489,7 @@ func (m *PartitionMarker) Reset() { *m = PartitionMarker{} } func (m *PartitionMarker) String() string { return proto.CompactTextString(m) } func (*PartitionMarker) ProtoMessage() {} func (*PartitionMarker) Descriptor() ([]byte, []int) { - return fileDescriptor_6aab92ca59e015f8, []int{41} + return fileDescriptor_6aab92ca59e015f8, []int{42} } func (m *PartitionMarker) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3457,7 +3542,7 @@ func (m *JobRunPreemptionRequested) Reset() { *m = JobRunPreemptionReque func (m *JobRunPreemptionRequested) String() string { return proto.CompactTextString(m) } func (*JobRunPreemptionRequested) ProtoMessage() {} func (*JobRunPreemptionRequested) Descriptor() ([]byte, []int) { - return fileDescriptor_6aab92ca59e015f8, []int{42} + return fileDescriptor_6aab92ca59e015f8, []int{43} } func (m *JobRunPreemptionRequested) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3510,7 +3595,7 @@ func (m *JobPreemptionRequested) Reset() { *m = JobPreemptionRequested{} func (m *JobPreemptionRequested) String() string { return proto.CompactTextString(m) } func (*JobPreemptionRequested) ProtoMessage() {} func (*JobPreemptionRequested) Descriptor() ([]byte, []int) { - return fileDescriptor_6aab92ca59e015f8, []int{43} + return fileDescriptor_6aab92ca59e015f8, []int{44} } func (m *JobPreemptionRequested) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3563,7 +3648,7 @@ func (m *JobValidated) Reset() { *m = JobValidated{} } func (m *JobValidated) String() string { return proto.CompactTextString(m) } func (*JobValidated) ProtoMessage() {} func (*JobValidated) Descriptor() ([]byte, []int) { - return fileDescriptor_6aab92ca59e015f8, []int{44} + return fileDescriptor_6aab92ca59e015f8, []int{45} } func (m *JobValidated) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3617,7 +3702,7 @@ func (m *JobRunCancelled) Reset() { *m = JobRunCancelled{} } func (m *JobRunCancelled) String() string { return proto.CompactTextString(m) } func (*JobRunCancelled) ProtoMessage() {} func (*JobRunCancelled) Descriptor() ([]byte, []int) { - return fileDescriptor_6aab92ca59e015f8, []int{45} + return fileDescriptor_6aab92ca59e015f8, []int{46} } func (m *JobRunCancelled) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3711,6 +3796,7 @@ func init() { proto.RegisterType((*GangJobUnschedulable)(nil), "armadaevents.GangJobUnschedulable") proto.RegisterType((*JobRejected)(nil), "armadaevents.JobRejected") proto.RegisterType((*ReconciliationError)(nil), "armadaevents.ReconciliationError") + proto.RegisterType((*FailureInfo)(nil), "armadaevents.FailureInfo") proto.RegisterType((*JobRunPreempted)(nil), "armadaevents.JobRunPreempted") proto.RegisterType((*PartitionMarker)(nil), "armadaevents.PartitionMarker") proto.RegisterType((*JobRunPreemptionRequested)(nil), "armadaevents.JobRunPreemptionRequested") @@ -3722,239 +3808,245 @@ func init() { func init() { proto.RegisterFile("pkg/armadaevents/events.proto", fileDescriptor_6aab92ca59e015f8) } var fileDescriptor_6aab92ca59e015f8 = []byte{ - // 3697 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe4, 0x3b, 0x4b, 0x6f, 0x1b, 0xd7, - 0xd5, 0x1e, 0xbe, 0x79, 0x28, 0x51, 0xf4, 0xd5, 0xc3, 0xb4, 0x12, 0x8b, 0x32, 0x9d, 0x2f, 0xb1, - 0x83, 0x84, 0x72, 0x9c, 0x2f, 0x1f, 0xf2, 0xf8, 0x90, 0x40, 0xb4, 0xe5, 0x87, 0x62, 0xd9, 0x0a, - 0x65, 0xa5, 0x6e, 0x11, 0x80, 0x19, 0x72, 0xae, 0xa8, 0xb1, 0xc8, 0x19, 0x66, 0x1e, 0x8a, 0x04, - 0x04, 0x68, 0x52, 0xa4, 0x5d, 0x67, 0x53, 0xa0, 0xc8, 0xa6, 0xd9, 0x74, 0xd1, 0x02, 0xdd, 0x14, - 0x68, 0x7f, 0x43, 0x17, 0x45, 0x91, 0x4d, 0xd1, 0x6e, 0x4a, 0x14, 0x09, 0xba, 0xe1, 0xa2, 0xbf, - 0xa1, 0xb8, 0x8f, 0x99, 0xb9, 0x77, 0x78, 0x69, 0x51, 0x8e, 0x65, 0x38, 0xc8, 0x4a, 0x9a, 0xf3, - 0xbc, 0x73, 0xcf, 0xb9, 0x67, 0xce, 0x39, 0xf7, 0x10, 0xce, 0xf5, 0xf7, 0x3a, 0x2b, 0xba, 0xd3, - 0xd3, 0x0d, 0x1d, 0xef, 0x63, 0xcb, 0x73, 0x57, 0xd8, 0x9f, 0x5a, 0xdf, 0xb1, 0x3d, 0x1b, 0x4d, - 0x89, 0xa8, 0xc5, 0xea, 0xde, 0xeb, 0x6e, 0xcd, 0xb4, 0x57, 0xf4, 0xbe, 0xb9, 0xd2, 0xb6, 0x1d, - 0xbc, 0xb2, 0xff, 0xca, 0x4a, 0x07, 0x5b, 0xd8, 0xd1, 0x3d, 0x6c, 0x30, 0x8e, 0xc5, 0x8b, 0x02, - 0x8d, 0x85, 0xbd, 0x8f, 0x6d, 0x67, 0xcf, 0xb4, 0x3a, 0x2a, 0xca, 0x4a, 0xc7, 0xb6, 0x3b, 0x5d, - 0xbc, 0x42, 0x9f, 0x5a, 0xfe, 0xce, 0x8a, 0x67, 0xf6, 0xb0, 0xeb, 0xe9, 0xbd, 0x3e, 0x27, 0xf8, - 0xdf, 0x48, 0x54, 0x4f, 0x6f, 0xef, 0x9a, 0x16, 0x76, 0x0e, 0x57, 0xe8, 0x7a, 0xfb, 0xe6, 0x8a, - 0x83, 0x5d, 0xdb, 0x77, 0xda, 0x78, 0x44, 0xec, 0x9b, 0xa6, 0xe5, 0x61, 0xc7, 0xd2, 0xbb, 0x2b, - 0x6e, 0x7b, 0x17, 0x1b, 0x7e, 0x17, 0x3b, 0xd1, 0x7f, 0x76, 0xeb, 0x01, 0x6e, 0x7b, 0xee, 0x08, - 0x80, 0xf1, 0x56, 0xff, 0x39, 0x0f, 0xd3, 0x6b, 0xe4, 0x5d, 0xb7, 0xf0, 0x47, 0x3e, 0xb6, 0xda, - 0x18, 0x5d, 0x82, 0xf4, 0x47, 0x3e, 0xf6, 0x71, 0x59, 0x5b, 0xd6, 0x2e, 0xe6, 0xeb, 0xb3, 0xc3, - 0x41, 0x65, 0x86, 0x02, 0x5e, 0xb2, 0x7b, 0xa6, 0x87, 0x7b, 0x7d, 0xef, 0xb0, 0xc1, 0x28, 0xd0, - 0x9b, 0x30, 0xf5, 0xc0, 0x6e, 0x35, 0x5d, 0xec, 0x35, 0x2d, 0xbd, 0x87, 0xcb, 0x09, 0xca, 0x51, - 0x1e, 0x0e, 0x2a, 0x73, 0x0f, 0xec, 0xd6, 0x16, 0xf6, 0xee, 0xe8, 0x3d, 0x91, 0x0d, 0x22, 0x28, - 0x7a, 0x19, 0xb2, 0xbe, 0x8b, 0x9d, 0xa6, 0x69, 0x94, 0x93, 0x94, 0x6d, 0x6e, 0x38, 0xa8, 0x94, - 0x08, 0xe8, 0x96, 0x21, 0xb0, 0x64, 0x18, 0x04, 0xbd, 0x04, 0x99, 0x8e, 0x63, 0xfb, 0x7d, 0xb7, - 0x9c, 0x5a, 0x4e, 0x06, 0xd4, 0x0c, 0x22, 0x52, 0x33, 0x08, 0xba, 0x0b, 0x19, 0x66, 0xc0, 0x72, - 0x7a, 0x39, 0x79, 0xb1, 0x70, 0xe5, 0x7c, 0x4d, 0xb4, 0x6a, 0x4d, 0x7a, 0x61, 0xf6, 0xc4, 0x04, - 0x32, 0xbc, 0x28, 0x90, 0xfb, 0xc1, 0x9f, 0x66, 0x21, 0x4d, 0xe9, 0xd0, 0xbb, 0x90, 0x6d, 0x3b, - 0x98, 0xec, 0x7e, 0x19, 0x2d, 0x6b, 0x17, 0x0b, 0x57, 0x16, 0x6b, 0xcc, 0xaa, 0xb5, 0xc0, 0xaa, - 0xb5, 0x7b, 0x81, 0x55, 0xeb, 0xf3, 0xc3, 0x41, 0xe5, 0x34, 0x27, 0x17, 0xa4, 0x06, 0x12, 0xd0, - 0x26, 0xe4, 0x5d, 0xbf, 0xd5, 0x33, 0xbd, 0x75, 0xbb, 0x45, 0xf7, 0xbb, 0x70, 0xe5, 0x8c, 0xbc, - 0xd4, 0xad, 0x00, 0x5d, 0x3f, 0x33, 0x1c, 0x54, 0x66, 0x43, 0xea, 0x48, 0xda, 0xcd, 0x53, 0x8d, - 0x48, 0x08, 0xda, 0x85, 0x19, 0x07, 0xf7, 0x1d, 0xd3, 0x76, 0x4c, 0xcf, 0x74, 0x31, 0x91, 0x9b, - 0xa0, 0x72, 0xcf, 0xc9, 0x72, 0x1b, 0x32, 0x51, 0xfd, 0xdc, 0x70, 0x50, 0x39, 0x1b, 0xe3, 0x94, - 0x74, 0xc4, 0xc5, 0x22, 0x0f, 0x50, 0x0c, 0xb4, 0x85, 0x3d, 0x6a, 0xcb, 0xc2, 0x95, 0xe5, 0x87, - 0x2a, 0xdb, 0xc2, 0x5e, 0x7d, 0x79, 0x38, 0xa8, 0x3c, 0x3b, 0xca, 0x2f, 0xa9, 0x54, 0xc8, 0x47, - 0x5d, 0x28, 0x89, 0x50, 0x83, 0xbc, 0x60, 0x8a, 0xea, 0x5c, 0x1a, 0xaf, 0x93, 0x50, 0xd5, 0x97, - 0x86, 0x83, 0xca, 0x62, 0x9c, 0x57, 0xd2, 0x37, 0x22, 0x99, 0xd8, 0xa7, 0xad, 0x5b, 0x6d, 0xdc, - 0x25, 0x6a, 0xd2, 0x2a, 0xfb, 0x5c, 0x0d, 0xd0, 0xcc, 0x3e, 0x21, 0xb5, 0x6c, 0x9f, 0x10, 0x8c, - 0x3e, 0x80, 0xa9, 0xf0, 0x81, 0xec, 0x57, 0x86, 0xfb, 0x90, 0x5a, 0x28, 0xd9, 0xa9, 0xc5, 0xe1, - 0xa0, 0xb2, 0x20, 0xf2, 0x48, 0xa2, 0x25, 0x69, 0x91, 0xf4, 0x2e, 0xdb, 0x99, 0xec, 0x78, 0xe9, - 0x8c, 0x42, 0x94, 0xde, 0x1d, 0xdd, 0x11, 0x49, 0x1a, 0x91, 0x4e, 0x0e, 0xb0, 0xdf, 0x6e, 0x63, - 0x6c, 0x60, 0xa3, 0x9c, 0x53, 0x49, 0x5f, 0x17, 0x28, 0x98, 0x74, 0x91, 0x47, 0x96, 0x2e, 0x62, - 0xc8, 0x5e, 0x3f, 0xb0, 0x5b, 0x6b, 0x8e, 0x63, 0x3b, 0x6e, 0x39, 0xaf, 0xda, 0xeb, 0xf5, 0x00, - 0xcd, 0xf6, 0x3a, 0xa4, 0x96, 0xf7, 0x3a, 0x04, 0xf3, 0xf5, 0x36, 0x7c, 0xeb, 0x36, 0xd6, 0x5d, - 0x6c, 0x94, 0x61, 0xcc, 0x7a, 0x43, 0x8a, 0x70, 0xbd, 0x21, 0x64, 0x64, 0xbd, 0x21, 0x06, 0x19, - 0x50, 0x64, 0xcf, 0xab, 0xae, 0x6b, 0x76, 0x2c, 0x6c, 0x94, 0x0b, 0x54, 0xfe, 0xb3, 0x2a, 0xf9, - 0x01, 0x4d, 0xfd, 0xd9, 0xe1, 0xa0, 0x52, 0x96, 0xf9, 0x24, 0x1d, 0x31, 0x99, 0xe8, 0x43, 0x98, - 0x66, 0x90, 0x86, 0x6f, 0x59, 0xa6, 0xd5, 0x29, 0x4f, 0x51, 0x25, 0xcf, 0xa8, 0x94, 0x70, 0x92, - 0xfa, 0x33, 0xc3, 0x41, 0xe5, 0x8c, 0xc4, 0x25, 0xa9, 0x90, 0x05, 0x92, 0x88, 0xc1, 0x00, 0x91, - 0x61, 0xa7, 0x55, 0x11, 0x63, 0x5d, 0x26, 0x62, 0x11, 0x23, 0xc6, 0x29, 0x47, 0x8c, 0x18, 0x32, - 0xb2, 0x07, 0x37, 0x72, 0x71, 0xbc, 0x3d, 0xb8, 0x9d, 0x05, 0x7b, 0x28, 0x4c, 0x2d, 0x49, 0x43, - 0x9f, 0x6a, 0x30, 0xef, 0x7a, 0xba, 0x65, 0xe8, 0x5d, 0xdb, 0xc2, 0xb7, 0xac, 0x8e, 0x83, 0x5d, - 0xf7, 0x96, 0xb5, 0x63, 0x97, 0x4b, 0x54, 0xcf, 0x85, 0x58, 0x60, 0x55, 0x91, 0xd6, 0x2f, 0x0c, - 0x07, 0x95, 0x8a, 0x52, 0x8a, 0xa4, 0x59, 0xad, 0x08, 0x1d, 0xc0, 0x6c, 0xf0, 0x91, 0xde, 0xf6, - 0xcc, 0xae, 0xe9, 0xea, 0x9e, 0x69, 0x5b, 0xe5, 0xd3, 0x54, 0xff, 0xf9, 0x78, 0x7c, 0x1a, 0x21, - 0xac, 0x9f, 0x1f, 0x0e, 0x2a, 0xe7, 0x14, 0x12, 0x24, 0xdd, 0x2a, 0x15, 0x91, 0x11, 0x37, 0x1d, - 0x4c, 0x08, 0xb1, 0x51, 0x9e, 0x1d, 0x6f, 0xc4, 0x90, 0x48, 0x34, 0x62, 0x08, 0x54, 0x19, 0x31, - 0x44, 0x12, 0x4d, 0x7d, 0xdd, 0xf1, 0x4c, 0xa2, 0x76, 0x43, 0x77, 0xf6, 0xb0, 0x53, 0x9e, 0x53, - 0x69, 0xda, 0x94, 0x89, 0x98, 0xa6, 0x18, 0xa7, 0xac, 0x29, 0x86, 0x44, 0x5f, 0x68, 0x20, 0x2f, - 0xcd, 0xb4, 0xad, 0x06, 0xf9, 0x68, 0xbb, 0xe4, 0xf5, 0xe6, 0xa9, 0xd2, 0x17, 0x1e, 0xf2, 0x7a, - 0x22, 0x79, 0xfd, 0x85, 0xe1, 0xa0, 0x72, 0x61, 0xac, 0x34, 0x69, 0x21, 0xe3, 0x95, 0xa2, 0xfb, - 0x50, 0x20, 0x48, 0x4c, 0xd3, 0x1f, 0xa3, 0xbc, 0x40, 0xd7, 0x70, 0x76, 0x74, 0x0d, 0x9c, 0xa0, - 0x7e, 0x76, 0x38, 0xa8, 0xcc, 0x0b, 0x1c, 0x92, 0x1e, 0x51, 0x14, 0xfa, 0x5c, 0x03, 0xe2, 0xe8, - 0xaa, 0x37, 0x3d, 0x43, 0xb5, 0x3c, 0x37, 0xa2, 0x45, 0xf5, 0x9a, 0xcf, 0x0d, 0x07, 0x95, 0x65, - 0xb5, 0x1c, 0x49, 0xf7, 0x18, 0x5d, 0x91, 0x1f, 0x85, 0x1f, 0x89, 0x72, 0x79, 0xbc, 0x1f, 0x85, - 0x44, 0xa2, 0x1f, 0x85, 0x40, 0x95, 0x1f, 0x85, 0x48, 0x1e, 0x0c, 0xde, 0xd7, 0xbb, 0xa6, 0x41, - 0x93, 0xa9, 0xb3, 0x63, 0x82, 0x41, 0x48, 0x11, 0x06, 0x83, 0x10, 0x32, 0x12, 0x0c, 0x22, 0xda, - 0x2c, 0xa4, 0xa9, 0x88, 0xea, 0x97, 0x79, 0x98, 0x55, 0x1c, 0x35, 0x84, 0x61, 0x3a, 0x38, 0x47, - 0x4d, 0x93, 0x04, 0x89, 0xa4, 0x6a, 0x97, 0xdf, 0xf5, 0x5b, 0xd8, 0xb1, 0xb0, 0x87, 0xdd, 0x40, - 0x06, 0x8d, 0x12, 0x74, 0x25, 0x8e, 0x00, 0x11, 0x72, 0xbb, 0x29, 0x11, 0x8e, 0xbe, 0xd4, 0xa0, - 0xdc, 0xd3, 0x0f, 0x9a, 0x01, 0xd0, 0x6d, 0xee, 0xd8, 0x4e, 0xb3, 0x8f, 0x1d, 0xd3, 0x36, 0x68, - 0x26, 0x5b, 0xb8, 0xf2, 0xff, 0x47, 0xc6, 0x85, 0xda, 0x86, 0x7e, 0x10, 0x80, 0xdd, 0xeb, 0xb6, - 0xb3, 0x49, 0xd9, 0xd7, 0x2c, 0xcf, 0x39, 0x64, 0x01, 0xab, 0xa7, 0xc2, 0x0b, 0x6b, 0x9a, 0x57, - 0x12, 0xa0, 0x5f, 0x6a, 0xb0, 0xe0, 0xd9, 0x9e, 0xde, 0x6d, 0xb6, 0xfd, 0x9e, 0xdf, 0xd5, 0x3d, - 0x73, 0x1f, 0x37, 0x7d, 0x57, 0xef, 0x60, 0x9e, 0x36, 0xbf, 0x75, 0xf4, 0xd2, 0xee, 0x11, 0xfe, - 0xab, 0x21, 0xfb, 0x36, 0xe1, 0x66, 0x2b, 0xab, 0x0e, 0x07, 0x95, 0x25, 0x4f, 0x81, 0x16, 0x16, - 0x36, 0xa7, 0xc2, 0xa3, 0x17, 0x21, 0x43, 0xca, 0x0a, 0xd3, 0xa0, 0xd9, 0x11, 0x2f, 0x41, 0x1e, - 0xd8, 0x2d, 0xa9, 0x30, 0x48, 0x53, 0x00, 0xa1, 0x75, 0x7c, 0x8b, 0xd0, 0x66, 0x23, 0x5a, 0xc7, - 0xb7, 0x64, 0x5a, 0x0a, 0xa0, 0xc6, 0xd0, 0xf7, 0x3b, 0x6a, 0x63, 0xe4, 0x26, 0x35, 0xc6, 0xea, - 0x7e, 0xe7, 0xa1, 0xc6, 0xd0, 0x55, 0x78, 0xd1, 0x18, 0x4a, 0x82, 0xc5, 0xaf, 0x34, 0x58, 0x1c, - 0x6f, 0x67, 0x74, 0x01, 0x92, 0x7b, 0xf8, 0x90, 0xd7, 0x64, 0xa7, 0x87, 0x83, 0xca, 0xf4, 0x1e, - 0x3e, 0x14, 0xa4, 0x12, 0x2c, 0xfa, 0x31, 0xa4, 0xf7, 0xf5, 0xae, 0x8f, 0x79, 0xca, 0x5f, 0xab, - 0xb1, 0x72, 0xb2, 0x26, 0x96, 0x93, 0xb5, 0xfe, 0x5e, 0x87, 0x00, 0x6a, 0xc1, 0x2e, 0xd4, 0xde, - 0xf3, 0x75, 0xcb, 0x33, 0xbd, 0x43, 0xb6, 0x77, 0x54, 0x80, 0xb8, 0x77, 0x14, 0xf0, 0x66, 0xe2, - 0x75, 0x6d, 0xf1, 0xd7, 0x1a, 0x9c, 0x1d, 0x6b, 0xef, 0xa7, 0x62, 0x85, 0x64, 0x13, 0xc7, 0xdb, - 0xe7, 0x69, 0x58, 0xe2, 0x7a, 0x2a, 0xa7, 0x95, 0x12, 0xeb, 0xa9, 0x5c, 0xa2, 0x94, 0xac, 0xfe, - 0x21, 0x03, 0xf9, 0xb0, 0xc0, 0x43, 0x37, 0xa1, 0x64, 0x60, 0xc3, 0xef, 0x77, 0xcd, 0x36, 0xf5, - 0x34, 0xe2, 0xd4, 0xac, 0xa2, 0xa6, 0xd1, 0x55, 0xc2, 0x49, 0xee, 0x3d, 0x13, 0x43, 0xa1, 0x2b, - 0x90, 0xe3, 0x85, 0xcc, 0x21, 0x8d, 0x6b, 0xd3, 0xf5, 0x85, 0xe1, 0xa0, 0x82, 0x02, 0x98, 0xc0, - 0x1a, 0xd2, 0xa1, 0x06, 0x00, 0xeb, 0x0c, 0x6c, 0x60, 0x4f, 0xe7, 0x25, 0x55, 0x59, 0x3e, 0x0d, - 0x77, 0x43, 0x3c, 0xab, 0xf1, 0x23, 0x7a, 0xb1, 0xc6, 0x8f, 0xa0, 0xe8, 0x03, 0x80, 0x9e, 0x6e, - 0x5a, 0x8c, 0x8f, 0xd7, 0x4f, 0xd5, 0x71, 0x11, 0x76, 0x23, 0xa4, 0x64, 0xd2, 0x23, 0x4e, 0x51, - 0x7a, 0x04, 0x45, 0x77, 0x21, 0xcb, 0x7b, 0x19, 0xe5, 0x0c, 0x3d, 0xbc, 0x4b, 0xe3, 0x44, 0x73, - 0xb1, 0xb4, 0x1a, 0xe7, 0x2c, 0x62, 0x35, 0xce, 0x41, 0x64, 0xdb, 0xba, 0xe6, 0x0e, 0xf6, 0xcc, - 0x1e, 0xa6, 0xd1, 0x84, 0x6f, 0x5b, 0x00, 0x13, 0xb7, 0x2d, 0x80, 0xa1, 0xd7, 0x01, 0x74, 0x6f, - 0xc3, 0x76, 0xbd, 0xbb, 0x56, 0x1b, 0xd3, 0x8a, 0x28, 0xc7, 0x96, 0x1f, 0x41, 0xc5, 0xe5, 0x47, - 0x50, 0xf4, 0x16, 0x14, 0xfa, 0xfc, 0x0b, 0xdc, 0xea, 0x62, 0x5a, 0xf1, 0xe4, 0x58, 0xc2, 0x20, - 0x80, 0x05, 0x5e, 0x91, 0x1a, 0xdd, 0x80, 0x99, 0xb6, 0x6d, 0xb5, 0x7d, 0xc7, 0xc1, 0x56, 0xfb, - 0x70, 0x4b, 0xdf, 0xc1, 0xb4, 0xba, 0xc9, 0x31, 0x57, 0x89, 0xa1, 0x44, 0x57, 0x89, 0xa1, 0xd0, - 0x6b, 0x90, 0x0f, 0x3b, 0x43, 0xb4, 0x80, 0xc9, 0xf3, 0x46, 0x43, 0x00, 0x14, 0x98, 0x23, 0x4a, - 0xb2, 0x78, 0xd3, 0xbd, 0xc6, 0x9d, 0x0e, 0xd3, 0xa2, 0x84, 0x2f, 0x5e, 0x00, 0x8b, 0x8b, 0x17, - 0xc0, 0x42, 0x7c, 0x2f, 0x1e, 0x15, 0xdf, 0xc3, 0xe3, 0x32, 0x5d, 0x2a, 0xae, 0xa7, 0x72, 0x33, - 0xa5, 0x52, 0xf5, 0x2f, 0x1a, 0xcc, 0xa9, 0xbc, 0x26, 0xe6, 0xc1, 0xda, 0x63, 0xf1, 0xe0, 0xf7, - 0x21, 0xd7, 0xb7, 0x8d, 0xa6, 0xdb, 0xc7, 0x6d, 0x1e, 0x0f, 0x62, 0xfe, 0xbb, 0x69, 0x1b, 0x5b, - 0x7d, 0xdc, 0xfe, 0x91, 0xe9, 0xed, 0xae, 0xee, 0xdb, 0xa6, 0x71, 0xdb, 0x74, 0xb9, 0xa3, 0xf5, - 0x19, 0x46, 0x4a, 0x52, 0xb2, 0x1c, 0x58, 0xcf, 0x41, 0x86, 0x69, 0xa9, 0xfe, 0x35, 0x09, 0xa5, - 0xb8, 0xa7, 0x7e, 0x9f, 0x5e, 0x05, 0xdd, 0x87, 0xac, 0xc9, 0x6a, 0x20, 0x9e, 0x43, 0xfd, 0x8f, - 0x10, 0x31, 0x6b, 0x51, 0x43, 0xb4, 0xb6, 0xff, 0x4a, 0x8d, 0x17, 0x4b, 0x74, 0x0b, 0xa8, 0x64, - 0xce, 0x29, 0x4b, 0xe6, 0x40, 0xd4, 0x80, 0xac, 0x8b, 0x9d, 0x7d, 0xb3, 0x8d, 0x79, 0x3c, 0xaa, - 0x88, 0x92, 0xdb, 0xb6, 0x83, 0x89, 0xcc, 0x2d, 0x46, 0x12, 0xc9, 0xe4, 0x3c, 0xb2, 0x4c, 0x0e, - 0x44, 0xef, 0x43, 0xbe, 0x6d, 0x5b, 0x3b, 0x66, 0x67, 0x43, 0xef, 0xf3, 0x88, 0x74, 0x4e, 0x25, - 0xf5, 0x6a, 0x40, 0xc4, 0xfb, 0x3a, 0xc1, 0x63, 0xac, 0xaf, 0x13, 0x52, 0x45, 0x06, 0xfd, 0x4f, - 0x0a, 0x20, 0x32, 0x0e, 0x7a, 0x03, 0x0a, 0xf8, 0x00, 0xb7, 0x7d, 0xcf, 0xa6, 0xbd, 0x4e, 0x2d, - 0x6a, 0x91, 0x06, 0x60, 0xc9, 0xed, 0x21, 0x82, 0x92, 0xb3, 0x69, 0xe9, 0x3d, 0xec, 0xf6, 0xf5, - 0x76, 0xd0, 0x5b, 0xa5, 0x8b, 0x09, 0x81, 0xe2, 0xd9, 0x0c, 0x81, 0xe8, 0x79, 0x48, 0xd1, 0x6e, - 0x2c, 0x6b, 0xab, 0xa2, 0xe1, 0xa0, 0x52, 0xb4, 0xe4, 0x3e, 0x2c, 0xc5, 0xa3, 0x77, 0x60, 0x7a, - 0x2f, 0x74, 0x3c, 0xb2, 0xb6, 0x14, 0x65, 0xa0, 0xc9, 0x6d, 0x84, 0x90, 0x56, 0x37, 0x25, 0xc2, - 0xd1, 0x0e, 0x14, 0x74, 0xcb, 0xb2, 0x3d, 0xfa, 0xd9, 0x09, 0x5a, 0xad, 0x97, 0xc6, 0xb9, 0x69, - 0x6d, 0x35, 0xa2, 0x65, 0xe9, 0x12, 0x8d, 0x17, 0x82, 0x04, 0x31, 0x5e, 0x08, 0x60, 0xd4, 0x80, - 0x4c, 0x57, 0x6f, 0xe1, 0x6e, 0x10, 0xe7, 0x9f, 0x1b, 0xab, 0xe2, 0x36, 0x25, 0x63, 0xd2, 0x69, - 0x43, 0x97, 0xf1, 0x89, 0x0d, 0x5d, 0x06, 0x59, 0xdc, 0x81, 0x52, 0x7c, 0x3d, 0x93, 0xa5, 0x07, - 0x97, 0xc4, 0xf4, 0x20, 0x7f, 0x64, 0x46, 0xa2, 0x43, 0x41, 0x58, 0xd4, 0x49, 0xa8, 0xa8, 0xfe, - 0x56, 0x83, 0x39, 0xd5, 0xd9, 0x45, 0x1b, 0xc2, 0x89, 0xd7, 0x78, 0xdb, 0x48, 0xe1, 0xea, 0x9c, - 0x77, 0xcc, 0x51, 0x8f, 0x0e, 0x7a, 0x1d, 0x8a, 0x96, 0x6d, 0xe0, 0xa6, 0x4e, 0x14, 0x74, 0x4d, - 0xd7, 0x2b, 0x27, 0x68, 0x2b, 0x9e, 0xb6, 0x9b, 0x08, 0x66, 0x35, 0x40, 0x08, 0xdc, 0xd3, 0x12, - 0xa2, 0xfa, 0x31, 0xcc, 0xc4, 0x9a, 0xc1, 0x52, 0xb2, 0x92, 0x98, 0x30, 0x59, 0x89, 0xbe, 0x20, - 0xc9, 0xc9, 0xbe, 0x20, 0xd5, 0x9f, 0x27, 0xa0, 0x20, 0x54, 0xe6, 0xe8, 0x01, 0xcc, 0xf0, 0xaf, - 0x99, 0x69, 0x75, 0x58, 0x05, 0x98, 0xe0, 0x6d, 0xa2, 0x91, 0x9b, 0x92, 0x75, 0xbb, 0xb5, 0x15, - 0xd2, 0xd2, 0x02, 0x90, 0x76, 0xf1, 0x5c, 0x09, 0x26, 0x28, 0x2e, 0xca, 0x18, 0x74, 0x1f, 0x16, - 0xfc, 0x3e, 0xa9, 0x4b, 0x9b, 0x2e, 0xbf, 0x73, 0x68, 0x5a, 0x7e, 0xaf, 0x85, 0x1d, 0xba, 0xfa, - 0x34, 0xab, 0x94, 0x18, 0x45, 0x70, 0x29, 0x71, 0x87, 0xe2, 0xc5, 0x4a, 0x49, 0x85, 0x17, 0xf6, - 0x21, 0x35, 0xe1, 0x3e, 0xdc, 0x04, 0x34, 0xda, 0x8d, 0x97, 0x6c, 0xa0, 0x4d, 0x66, 0x83, 0xea, - 0x01, 0x94, 0xe2, 0x3d, 0xf6, 0x27, 0x64, 0xcb, 0x3d, 0xc8, 0x87, 0x1d, 0x72, 0xf4, 0x12, 0x64, - 0x1c, 0xac, 0xbb, 0xb6, 0xc5, 0x4f, 0x0b, 0x3d, 0xf6, 0x0c, 0x22, 0x1e, 0x7b, 0x06, 0x79, 0x04, - 0x65, 0xf7, 0x60, 0x8a, 0x6d, 0xd2, 0x75, 0xb3, 0xeb, 0x61, 0x07, 0x5d, 0x83, 0x8c, 0xeb, 0xe9, - 0x1e, 0x76, 0xcb, 0xda, 0x72, 0xf2, 0x62, 0xf1, 0xca, 0xc2, 0x68, 0xfb, 0x9b, 0xa0, 0xd9, 0x3a, - 0x18, 0xa5, 0xb8, 0x0e, 0x06, 0xa9, 0xfe, 0x4c, 0x83, 0x29, 0xb1, 0xcb, 0xff, 0x78, 0xc4, 0x1e, - 0x6f, 0x33, 0x48, 0xe0, 0x98, 0x12, 0x2f, 0x03, 0x4e, 0x6e, 0x2f, 0xc9, 0x57, 0x90, 0x5d, 0x25, - 0x34, 0x7d, 0x17, 0x3b, 0xdc, 0x5b, 0xe9, 0x57, 0x90, 0x81, 0xb7, 0x5d, 0xc9, 0xdb, 0x21, 0x82, - 0x72, 0x33, 0x90, 0xb5, 0x8a, 0x57, 0x0b, 0xa8, 0x13, 0x35, 0x70, 0xc8, 0x21, 0x73, 0x69, 0x30, - 0x9a, 0xb4, 0x81, 0x43, 0x43, 0x96, 0xc4, 0x2e, 0x86, 0x2c, 0x09, 0xf1, 0x08, 0x2e, 0xf3, 0x55, - 0x9a, 0xae, 0x35, 0xba, 0x2a, 0x88, 0xe5, 0x00, 0xc9, 0x63, 0xe4, 0x00, 0x2f, 0x43, 0x96, 0x06, - 0xdd, 0xf0, 0x88, 0x53, 0x9b, 0x10, 0x90, 0x7c, 0x4d, 0xca, 0x20, 0x0f, 0x09, 0x35, 0xe9, 0xef, - 0x18, 0x6a, 0x9a, 0x70, 0x76, 0x57, 0x77, 0x9b, 0x41, 0x70, 0x34, 0x9a, 0xba, 0xd7, 0x0c, 0xcf, - 0x7a, 0x86, 0xe6, 0xff, 0xb4, 0xf9, 0xb8, 0xab, 0xbb, 0x5b, 0x01, 0xcd, 0xaa, 0xb7, 0x39, 0x7a, - 0xf2, 0x17, 0xd4, 0x14, 0x68, 0x1b, 0xe6, 0xd5, 0xc2, 0xb3, 0x74, 0xe5, 0xb4, 0x37, 0xee, 0x3e, - 0x54, 0xf2, 0xac, 0x02, 0x8d, 0x3e, 0xd3, 0xa0, 0x4c, 0xbe, 0x82, 0x0e, 0xfe, 0xc8, 0x37, 0x1d, - 0xdc, 0x23, 0x6e, 0xd1, 0xb4, 0xf7, 0xb1, 0xd3, 0xd5, 0x0f, 0xf9, 0x35, 0xd3, 0xf9, 0xd1, 0x90, - 0xbf, 0x69, 0x1b, 0x0d, 0x81, 0x81, 0xbd, 0x5a, 0x5f, 0x06, 0xde, 0x65, 0x42, 0xc4, 0x57, 0x53, - 0x53, 0x08, 0x2e, 0x04, 0xc7, 0x68, 0x68, 0x15, 0x8e, 0x6c, 0x68, 0x3d, 0x0f, 0xa9, 0xbe, 0x6d, - 0x77, 0x69, 0xf9, 0xc5, 0x33, 0x3d, 0xf2, 0x2c, 0x66, 0x7a, 0xe4, 0x59, 0xec, 0x39, 0xac, 0xa7, - 0x72, 0xb9, 0x52, 0x9e, 0x7c, 0x0e, 0x8b, 0xf2, 0xcd, 0xd4, 0xe8, 0x81, 0x4a, 0x9e, 0xf8, 0x81, - 0x4a, 0x1d, 0x63, 0x37, 0xd2, 0x13, 0xef, 0x46, 0x66, 0xf2, 0xdd, 0xa8, 0x7e, 0x9e, 0x80, 0x69, - 0xe9, 0xf2, 0xec, 0x87, 0xb9, 0x0d, 0xbf, 0x4a, 0xc0, 0x82, 0xfa, 0x95, 0x4e, 0xa4, 0x14, 0xbd, - 0x09, 0x24, 0xa9, 0xbc, 0x15, 0x25, 0x5d, 0xf3, 0x23, 0x95, 0x28, 0xdd, 0xce, 0x20, 0x23, 0x1d, - 0xb9, 0x7f, 0x0b, 0xd8, 0xd1, 0x7d, 0x28, 0x98, 0xc2, 0x4d, 0x5f, 0x52, 0x75, 0x21, 0x23, 0xde, - 0xef, 0xb1, 0x16, 0xc5, 0x98, 0x5b, 0x3d, 0x51, 0x54, 0x3d, 0x03, 0x29, 0x92, 0x15, 0x56, 0xf7, - 0x21, 0xcb, 0x97, 0x83, 0x5e, 0x85, 0x3c, 0x8d, 0xc5, 0xb4, 0xba, 0x62, 0x29, 0x3c, 0x4d, 0x6f, - 0x08, 0x30, 0x36, 0xe9, 0x92, 0x0b, 0x60, 0xe8, 0xff, 0x00, 0x48, 0xf8, 0xe1, 0x51, 0x38, 0x41, - 0x63, 0x19, 0xad, 0xe2, 0xfa, 0xb6, 0x31, 0x12, 0x7a, 0xf3, 0x21, 0xb0, 0xfa, 0xfb, 0x04, 0x14, - 0xc4, 0xbb, 0xc5, 0x47, 0x52, 0xfe, 0x09, 0x04, 0x15, 0x76, 0x53, 0x37, 0x0c, 0xf2, 0x17, 0x07, - 0x1f, 0xca, 0x95, 0xb1, 0x9b, 0x14, 0xfc, 0xbf, 0x1a, 0x70, 0xb0, 0x7a, 0x8a, 0xce, 0x4f, 0x98, - 0x31, 0x94, 0xa0, 0xb5, 0x14, 0xc7, 0x2d, 0xee, 0xc1, 0xbc, 0x52, 0x94, 0x58, 0x05, 0xa5, 0x1f, - 0x57, 0x15, 0xf4, 0x9b, 0x34, 0xcc, 0x2b, 0xef, 0x74, 0x63, 0x1e, 0x9c, 0x7c, 0x2c, 0x1e, 0xfc, - 0x0b, 0x4d, 0xb5, 0xb3, 0xec, 0x42, 0xe7, 0x8d, 0x09, 0x2e, 0x9a, 0x1f, 0xd7, 0x1e, 0xcb, 0x6e, - 0x91, 0x7e, 0x24, 0x9f, 0xcc, 0x4c, 0xea, 0x93, 0xe8, 0x32, 0x2b, 0x28, 0xa9, 0x2e, 0x76, 0xdd, - 0x12, 0x9c, 0xd0, 0x98, 0xaa, 0x2c, 0x07, 0xa1, 0x77, 0x60, 0x3a, 0xe0, 0x60, 0x6d, 0x8c, 0x5c, - 0xd4, 0x63, 0xe0, 0x34, 0xf1, 0x4e, 0xc6, 0x94, 0x08, 0x17, 0xa2, 0x64, 0xfe, 0x18, 0x51, 0x12, - 0x8e, 0x8a, 0x92, 0x4f, 0xd4, 0x37, 0xa5, 0x50, 0x3b, 0xd0, 0x60, 0x26, 0x36, 0x4a, 0xf1, 0xbd, - 0xff, 0xe6, 0x48, 0x2f, 0xf8, 0xa9, 0x06, 0xf9, 0x70, 0x52, 0x07, 0xad, 0x42, 0x06, 0xb3, 0x69, - 0x0f, 0x16, 0x76, 0x66, 0x63, 0x93, 0x78, 0x04, 0xc7, 0x67, 0xef, 0x62, 0x03, 0x1e, 0x0d, 0xce, - 0xf8, 0x08, 0x09, 0xf8, 0x1f, 0xb5, 0x20, 0x01, 0x1f, 0x59, 0x45, 0xf2, 0xbb, 0xaf, 0xe2, 0xe4, - 0xb6, 0xee, 0xef, 0x79, 0x48, 0xd3, 0xb5, 0x90, 0x42, 0xda, 0xc3, 0x4e, 0xcf, 0xb4, 0xf4, 0x2e, - 0x75, 0xc5, 0x1c, 0x3b, 0xd5, 0x01, 0x4c, 0x3c, 0xd5, 0x01, 0x0c, 0xed, 0xc2, 0x4c, 0xd4, 0x9e, - 0xa3, 0x62, 0xd4, 0xa3, 0x7f, 0xef, 0xca, 0x44, 0xec, 0xca, 0x20, 0xc6, 0x29, 0xdf, 0xdd, 0xc7, - 0x90, 0xc8, 0x80, 0x62, 0xdb, 0xb6, 0x3c, 0xdd, 0xb4, 0xb0, 0xc3, 0x14, 0x25, 0x55, 0xa3, 0x4f, - 0x57, 0x25, 0x1a, 0xd6, 0x34, 0x91, 0xf9, 0xe4, 0xd1, 0x27, 0x19, 0x87, 0x3e, 0x84, 0xe9, 0xa0, - 0x10, 0x62, 0x4a, 0x52, 0xaa, 0xd1, 0xa7, 0x35, 0x91, 0x84, 0x1d, 0x06, 0x89, 0x4b, 0x1e, 0x7d, - 0x92, 0x50, 0xe8, 0x03, 0x98, 0xea, 0x92, 0x0a, 0x6d, 0xed, 0xa0, 0x6f, 0x3a, 0xd8, 0x50, 0x0f, - 0xe3, 0xdd, 0x16, 0x28, 0x58, 0xe0, 0x12, 0x79, 0xe4, 0x19, 0x04, 0x11, 0x43, 0xec, 0xd1, 0xd3, - 0x0f, 0x1a, 0xbe, 0xe5, 0xae, 0x1d, 0xf0, 0xc1, 0xaa, 0xac, 0xca, 0x1e, 0x1b, 0x32, 0x11, 0xb3, - 0x47, 0x8c, 0x53, 0xb6, 0x47, 0x0c, 0x89, 0x6e, 0xd3, 0xb8, 0xcc, 0x36, 0x89, 0x0d, 0xe5, 0x2d, - 0x8c, 0x24, 0x54, 0x6c, 0x7f, 0x58, 0x3b, 0x86, 0x3f, 0x49, 0x42, 0x43, 0x09, 0xa8, 0x0b, 0xa5, - 0xbe, 0x6d, 0xd0, 0xd7, 0x6e, 0x60, 0xcf, 0x77, 0x2c, 0x6c, 0xf0, 0x42, 0x69, 0x69, 0x44, 0xaa, - 0x44, 0xc5, 0x3e, 0x5f, 0x71, 0x5e, 0x79, 0xc4, 0x32, 0x8e, 0x45, 0x9f, 0xc0, 0x5c, 0x6c, 0xc4, - 0x88, 0xbd, 0x47, 0x41, 0x75, 0x45, 0xb1, 0xae, 0xa0, 0x64, 0x35, 0xad, 0x4a, 0x86, 0xa4, 0x59, - 0xa9, 0x85, 0x68, 0xef, 0xe8, 0x56, 0x67, 0xdd, 0x6e, 0x6d, 0x5b, 0xbc, 0x08, 0xd4, 0x5b, 0x5d, - 0xcc, 0xa7, 0xec, 0x62, 0xda, 0x6f, 0x28, 0x28, 0x99, 0x76, 0x95, 0x0c, 0x59, 0xbb, 0x8a, 0x22, - 0x1c, 0x27, 0x22, 0x69, 0x45, 0x38, 0x76, 0xa7, 0x1a, 0x27, 0x62, 0x04, 0xc2, 0x38, 0x11, 0x03, - 0x28, 0xc6, 0x89, 0x18, 0x82, 0x4d, 0xa2, 0xb5, 0x6d, 0xab, 0x6d, 0x76, 0x4d, 0xda, 0xe2, 0x66, - 0x9b, 0x5a, 0x54, 0x4f, 0xa2, 0x8d, 0x10, 0x06, 0x93, 0x68, 0x23, 0x88, 0xf8, 0x24, 0xda, 0x28, - 0x67, 0x2e, 0xe8, 0x21, 0xad, 0xa7, 0x72, 0xe9, 0x52, 0x66, 0x3d, 0x95, 0x83, 0x52, 0xa1, 0xfa, - 0x1e, 0xcc, 0xc4, 0xc2, 0x0e, 0x7a, 0x1b, 0xc2, 0x51, 0x99, 0x7b, 0x87, 0xfd, 0x20, 0xa7, 0x95, - 0x46, 0x6b, 0x08, 0x5c, 0x35, 0x5a, 0x43, 0xe0, 0xd5, 0x2f, 0x52, 0x90, 0x0b, 0xfc, 0xfa, 0x44, - 0xaa, 0x94, 0x15, 0xc8, 0xf6, 0xb0, 0x4b, 0xc7, 0x61, 0x12, 0x51, 0xb2, 0xc3, 0x41, 0x62, 0xb2, - 0xc3, 0x41, 0x72, 0x2e, 0x96, 0x7c, 0xa4, 0x5c, 0x2c, 0x35, 0x71, 0x2e, 0x86, 0xe9, 0x0d, 0xb0, - 0x10, 0x2f, 0x83, 0x0b, 0x98, 0x87, 0x07, 0xe1, 0xe0, 0x7e, 0x58, 0x64, 0x8c, 0xdd, 0x0f, 0x8b, - 0x28, 0xb4, 0x07, 0xa7, 0x85, 0x4b, 0x22, 0xde, 0x1d, 0x24, 0x71, 0xb2, 0x38, 0xfe, 0xba, 0xbd, - 0x41, 0xa9, 0x58, 0x34, 0xd8, 0x8b, 0x41, 0xc5, 0x64, 0x36, 0x8e, 0x23, 0x2e, 0x61, 0xe0, 0x96, - 0xdf, 0xd9, 0xe0, 0xdb, 0x9e, 0x8d, 0x5c, 0x42, 0x84, 0x8b, 0x2e, 0x21, 0xc2, 0xab, 0xff, 0x4e, - 0x40, 0x51, 0x7e, 0xdf, 0x13, 0x71, 0x8c, 0x57, 0x21, 0x8f, 0x0f, 0x4c, 0xaf, 0xd9, 0xb6, 0x0d, - 0xcc, 0x2b, 0x3a, 0x6a, 0x67, 0x02, 0xbc, 0x6a, 0x1b, 0x92, 0x9d, 0x03, 0x98, 0xe8, 0x4d, 0xc9, - 0x89, 0xbc, 0x29, 0x6a, 0xc6, 0xa6, 0x26, 0x68, 0xc6, 0x2a, 0xed, 0x94, 0x3f, 0x19, 0x3b, 0x55, - 0xbf, 0x4e, 0x40, 0x29, 0x1e, 0xfc, 0x9f, 0x8e, 0x23, 0x28, 0x9f, 0xa6, 0xe4, 0xc4, 0xa7, 0xe9, - 0x1d, 0x98, 0x26, 0x19, 0x9b, 0xee, 0x79, 0x7c, 0x7a, 0x36, 0x45, 0x93, 0x2e, 0x16, 0x8d, 0x7c, - 0x6b, 0x35, 0x80, 0x4b, 0xd1, 0x48, 0x80, 0x8f, 0xb8, 0x6e, 0xfa, 0x98, 0xae, 0xfb, 0x59, 0x02, - 0xa6, 0x37, 0x6d, 0xe3, 0x1e, 0x4b, 0xe6, 0xbc, 0xa7, 0x65, 0x3f, 0x9f, 0x64, 0x48, 0xab, 0xce, - 0xc0, 0xb4, 0x94, 0xcd, 0x55, 0x3f, 0x67, 0x7e, 0x26, 0x7f, 0x34, 0x7f, 0x78, 0xfb, 0x52, 0x84, - 0x29, 0x31, 0x09, 0xad, 0xd6, 0x61, 0x26, 0x96, 0x33, 0x8a, 0x2f, 0xa0, 0x4d, 0xf2, 0x02, 0xd5, - 0x6b, 0x30, 0xa7, 0x4a, 0xa6, 0x84, 0xa8, 0xa3, 0x4d, 0x70, 0x83, 0x74, 0x03, 0xe6, 0x54, 0x49, - 0xd1, 0xf1, 0x97, 0xf3, 0x36, 0xbf, 0x9d, 0xe5, 0xe9, 0xcb, 0xb1, 0xf9, 0xaf, 0xc3, 0xac, 0x22, - 0x8d, 0x39, 0xbe, 0x9c, 0xbf, 0x85, 0xd5, 0x79, 0x34, 0xf1, 0x7e, 0x1d, 0x4a, 0xfd, 0xe0, 0xa1, - 0xc9, 0x6b, 0x40, 0x76, 0xbc, 0x69, 0x45, 0x13, 0xe2, 0xd6, 0x63, 0xc5, 0x60, 0x51, 0xc6, 0xc8, - 0x72, 0x78, 0x7d, 0x98, 0x51, 0xc8, 0x69, 0xc4, 0x0a, 0xc5, 0xa2, 0x8c, 0x11, 0x4c, 0x94, 0x3d, - 0xda, 0x44, 0xb4, 0xbe, 0x4c, 0x93, 0xa2, 0x7c, 0x26, 0x36, 0x91, 0x8f, 0x2e, 0x43, 0x8e, 0xfe, - 0x5c, 0x2e, 0xaa, 0xac, 0xe9, 0xee, 0x50, 0x98, 0xb4, 0x80, 0x2c, 0x07, 0xa1, 0xd7, 0x20, 0x1f, - 0x0e, 0xe9, 0xf3, 0xfb, 0x5d, 0xe6, 0xbf, 0x01, 0x50, 0xf2, 0xdf, 0x00, 0xc8, 0x8b, 0xf2, 0x9f, - 0xc2, 0xd9, 0xb1, 0xe3, 0xf9, 0xc7, 0xba, 0x4b, 0x8c, 0xaa, 0xeb, 0xd4, 0xb1, 0xaa, 0xeb, 0x03, - 0x58, 0x50, 0x4f, 0xcd, 0x0b, 0xda, 0x13, 0x47, 0x6a, 0x8f, 0x76, 0x3f, 0x39, 0xe1, 0xee, 0x27, - 0xaa, 0x7b, 0xb4, 0x1d, 0x11, 0x4e, 0xa7, 0xa3, 0x4b, 0x90, 0xee, 0xdb, 0x76, 0xd7, 0xe5, 0x03, - 0x14, 0x54, 0x1d, 0x05, 0x88, 0xea, 0x28, 0xe0, 0x11, 0x9a, 0x1f, 0x7e, 0xe0, 0xc1, 0xd1, 0xac, - 0xfd, 0x13, 0xd8, 0xdd, 0x17, 0x2f, 0x43, 0x2e, 0xb8, 0xa4, 0x46, 0x00, 0x99, 0xf7, 0xb6, 0xd7, - 0xb6, 0xd7, 0xae, 0x95, 0x4e, 0xa1, 0x02, 0x64, 0x37, 0xd7, 0xee, 0x5c, 0xbb, 0x75, 0xe7, 0x46, - 0x49, 0x23, 0x0f, 0x8d, 0xed, 0x3b, 0x77, 0xc8, 0x43, 0xe2, 0xc5, 0xdb, 0xe2, 0xe0, 0x1b, 0xcf, - 0x00, 0xa7, 0x20, 0xb7, 0xda, 0xef, 0xd3, 0xc3, 0xcb, 0x78, 0xd7, 0xf6, 0x4d, 0x12, 0x11, 0x4a, - 0x1a, 0xca, 0x42, 0xf2, 0xee, 0xdd, 0x8d, 0x52, 0x02, 0xcd, 0x41, 0xe9, 0x1a, 0xd6, 0x8d, 0xae, - 0x69, 0xe1, 0x20, 0xfe, 0x95, 0x92, 0xf5, 0x07, 0x7f, 0xfe, 0x66, 0x49, 0xfb, 0xfa, 0x9b, 0x25, - 0xed, 0x5f, 0xdf, 0x2c, 0x69, 0x5f, 0x7c, 0xbb, 0x74, 0xea, 0xeb, 0x6f, 0x97, 0x4e, 0xfd, 0xe3, - 0xdb, 0xa5, 0x53, 0x3f, 0xb9, 0xdc, 0x31, 0xbd, 0x5d, 0xbf, 0x55, 0x6b, 0xdb, 0x3d, 0xfe, 0xbb, - 0xdf, 0xbe, 0x63, 0x93, 0x40, 0xc3, 0x9f, 0x56, 0xe2, 0x3f, 0x08, 0xfe, 0x5d, 0xe2, 0xdc, 0x2a, - 0x7d, 0xdc, 0x64, 0x74, 0xb5, 0x5b, 0x76, 0x8d, 0x01, 0xe8, 0x4f, 0x40, 0xdd, 0x56, 0x86, 0xfe, - 0xd4, 0xf3, 0xd5, 0xff, 0x06, 0x00, 0x00, 0xff, 0xff, 0xc1, 0x1d, 0x32, 0xc0, 0x4b, 0x3c, 0x00, - 0x00, + // 3801 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe4, 0x5b, 0xcd, 0x6f, 0x1b, 0x47, + 0x74, 0xf7, 0xf2, 0x9b, 0x8f, 0x22, 0x45, 0x8f, 0x3e, 0x4c, 0x2b, 0xb1, 0x28, 0xd3, 0x69, 0x62, + 0x07, 0x09, 0xe5, 0x38, 0x4d, 0xe1, 0x24, 0x45, 0x02, 0xd1, 0x96, 0x3f, 0x14, 0xcb, 0x96, 0x29, + 0x2b, 0x71, 0x8b, 0x00, 0xcc, 0x92, 0x3b, 0xa2, 0xd6, 0x22, 0x77, 0x99, 0xfd, 0x50, 0x24, 0x20, + 0x40, 0x93, 0x22, 0xed, 0x39, 0x40, 0x51, 0xa0, 0xc8, 0xa5, 0xb9, 0xf4, 0xd0, 0x02, 0x05, 0x8a, + 0x02, 0xed, 0xdf, 0xd0, 0x43, 0x51, 0xe4, 0x52, 0xa0, 0x97, 0x12, 0x45, 0x82, 0x5e, 0x78, 0xe8, + 0xdf, 0x50, 0xcc, 0xc7, 0xee, 0xce, 0x2c, 0x87, 0x16, 0xe5, 0x58, 0x86, 0x83, 0x9c, 0x24, 0xfe, + 0xde, 0xd7, 0xec, 0xcc, 0x9b, 0xb7, 0xef, 0xbd, 0x99, 0x85, 0x0b, 0x83, 0xfd, 0xee, 0xaa, 0xee, + 0xf4, 0x75, 0x43, 0xc7, 0x07, 0xd8, 0xf2, 0xdc, 0x55, 0xf6, 0xa7, 0x3e, 0x70, 0x6c, 0xcf, 0x46, + 0x33, 0x22, 0x69, 0xa9, 0xb6, 0x7f, 0xdd, 0xad, 0x9b, 0xf6, 0xaa, 0x3e, 0x30, 0x57, 0x3b, 0xb6, + 0x83, 0x57, 0x0f, 0xde, 0x59, 0xed, 0x62, 0x0b, 0x3b, 0xba, 0x87, 0x0d, 0x26, 0xb1, 0x74, 0x59, + 0xe0, 0xb1, 0xb0, 0xf7, 0x95, 0xed, 0xec, 0x9b, 0x56, 0x57, 0xc5, 0x59, 0xed, 0xda, 0x76, 0xb7, + 0x87, 0x57, 0xe9, 0xaf, 0xb6, 0xbf, 0xbb, 0xea, 0x99, 0x7d, 0xec, 0x7a, 0x7a, 0x7f, 0xc0, 0x19, + 0xfe, 0x30, 0x52, 0xd5, 0xd7, 0x3b, 0x7b, 0xa6, 0x85, 0x9d, 0xa3, 0x55, 0x3a, 0xde, 0x81, 0xb9, + 0xea, 0x60, 0xd7, 0xf6, 0x9d, 0x0e, 0x1e, 0x53, 0xfb, 0x81, 0x69, 0x79, 0xd8, 0xb1, 0xf4, 0xde, + 0xaa, 0xdb, 0xd9, 0xc3, 0x86, 0xdf, 0xc3, 0x4e, 0xf4, 0x9f, 0xdd, 0x7e, 0x82, 0x3b, 0x9e, 0x3b, + 0x06, 0x30, 0xd9, 0xda, 0x7f, 0x2f, 0x40, 0x71, 0x9d, 0x3c, 0xeb, 0x36, 0xfe, 0xd2, 0xc7, 0x56, + 0x07, 0xa3, 0x2b, 0x90, 0xfe, 0xd2, 0xc7, 0x3e, 0xae, 0x68, 0x2b, 0xda, 0xe5, 0x7c, 0x63, 0x6e, + 0x34, 0xac, 0xce, 0x52, 0xe0, 0x2d, 0xbb, 0x6f, 0x7a, 0xb8, 0x3f, 0xf0, 0x8e, 0x9a, 0x8c, 0x03, + 0x7d, 0x00, 0x33, 0x4f, 0xec, 0x76, 0xcb, 0xc5, 0x5e, 0xcb, 0xd2, 0xfb, 0xb8, 0x92, 0xa0, 0x12, + 0x95, 0xd1, 0xb0, 0x3a, 0xff, 0xc4, 0x6e, 0x6f, 0x63, 0xef, 0xbe, 0xde, 0x17, 0xc5, 0x20, 0x42, + 0xd1, 0xdb, 0x90, 0xf5, 0x5d, 0xec, 0xb4, 0x4c, 0xa3, 0x92, 0xa4, 0x62, 0xf3, 0xa3, 0x61, 0xb5, + 0x4c, 0xa0, 0xbb, 0x86, 0x20, 0x92, 0x61, 0x08, 0x7a, 0x0b, 0x32, 0x5d, 0xc7, 0xf6, 0x07, 0x6e, + 0x25, 0xb5, 0x92, 0x0c, 0xb8, 0x19, 0x22, 0x72, 0x33, 0x04, 0x3d, 0x80, 0x0c, 0x5b, 0xc0, 0x4a, + 0x7a, 0x25, 0x79, 0xb9, 0x70, 0xed, 0x62, 0x5d, 0x5c, 0xd5, 0xba, 0xf4, 0xc0, 0xec, 0x17, 0x53, + 0xc8, 0xe8, 0xa2, 0x42, 0xee, 0x07, 0xff, 0x3a, 0x07, 0x69, 0xca, 0x87, 0x3e, 0x81, 0x6c, 0xc7, + 0xc1, 0x64, 0xf6, 0x2b, 0x68, 0x45, 0xbb, 0x5c, 0xb8, 0xb6, 0x54, 0x67, 0xab, 0x5a, 0x0f, 0x56, + 0xb5, 0xfe, 0x28, 0x58, 0xd5, 0xc6, 0xc2, 0x68, 0x58, 0x3d, 0xcb, 0xd9, 0x05, 0xad, 0x81, 0x06, + 0xb4, 0x05, 0x79, 0xd7, 0x6f, 0xf7, 0x4d, 0x6f, 0xc3, 0x6e, 0xd3, 0xf9, 0x2e, 0x5c, 0x3b, 0x27, + 0x0f, 0x75, 0x3b, 0x20, 0x37, 0xce, 0x8d, 0x86, 0xd5, 0xb9, 0x90, 0x3b, 0xd2, 0x76, 0xe7, 0x4c, + 0x33, 0x52, 0x82, 0xf6, 0x60, 0xd6, 0xc1, 0x03, 0xc7, 0xb4, 0x1d, 0xd3, 0x33, 0x5d, 0x4c, 0xf4, + 0x26, 0xa8, 0xde, 0x0b, 0xb2, 0xde, 0xa6, 0xcc, 0xd4, 0xb8, 0x30, 0x1a, 0x56, 0xcf, 0xc7, 0x24, + 0x25, 0x1b, 0x71, 0xb5, 0xc8, 0x03, 0x14, 0x83, 0xb6, 0xb1, 0x47, 0xd7, 0xb2, 0x70, 0x6d, 0xe5, + 0xa9, 0xc6, 0xb6, 0xb1, 0xd7, 0x58, 0x19, 0x0d, 0xab, 0xaf, 0x8e, 0xcb, 0x4b, 0x26, 0x15, 0xfa, + 0x51, 0x0f, 0xca, 0x22, 0x6a, 0x90, 0x07, 0x4c, 0x51, 0x9b, 0xcb, 0x93, 0x6d, 0x12, 0xae, 0xc6, + 0xf2, 0x68, 0x58, 0x5d, 0x8a, 0xcb, 0x4a, 0xf6, 0xc6, 0x34, 0x93, 0xf5, 0xe9, 0xe8, 0x56, 0x07, + 0xf7, 0x88, 0x99, 0xb4, 0x6a, 0x7d, 0x6e, 0x04, 0x64, 0xb6, 0x3e, 0x21, 0xb7, 0xbc, 0x3e, 0x21, + 0x8c, 0x3e, 0x87, 0x99, 0xf0, 0x07, 0x99, 0xaf, 0x0c, 0xf7, 0x21, 0xb5, 0x52, 0x32, 0x53, 0x4b, + 0xa3, 0x61, 0x75, 0x51, 0x94, 0x91, 0x54, 0x4b, 0xda, 0x22, 0xed, 0x3d, 0x36, 0x33, 0xd9, 0xc9, + 0xda, 0x19, 0x87, 0xa8, 0xbd, 0x37, 0x3e, 0x23, 0x92, 0x36, 0xa2, 0x9d, 0x6c, 0x60, 0xbf, 0xd3, + 0xc1, 0xd8, 0xc0, 0x46, 0x25, 0xa7, 0xd2, 0xbe, 0x21, 0x70, 0x30, 0xed, 0xa2, 0x8c, 0xac, 0x5d, + 0xa4, 0x90, 0xb9, 0x7e, 0x62, 0xb7, 0xd7, 0x1d, 0xc7, 0x76, 0xdc, 0x4a, 0x5e, 0x35, 0xd7, 0x1b, + 0x01, 0x99, 0xcd, 0x75, 0xc8, 0x2d, 0xcf, 0x75, 0x08, 0xf3, 0xf1, 0x36, 0x7d, 0xeb, 0x1e, 0xd6, + 0x5d, 0x6c, 0x54, 0x60, 0xc2, 0x78, 0x43, 0x8e, 0x70, 0xbc, 0x21, 0x32, 0x36, 0xde, 0x90, 0x82, + 0x0c, 0x28, 0xb1, 0xdf, 0x6b, 0xae, 0x6b, 0x76, 0x2d, 0x6c, 0x54, 0x0a, 0x54, 0xff, 0xab, 0x2a, + 0xfd, 0x01, 0x4f, 0xe3, 0xd5, 0xd1, 0xb0, 0x5a, 0x91, 0xe5, 0x24, 0x1b, 0x31, 0x9d, 0xe8, 0x0b, + 0x28, 0x32, 0xa4, 0xe9, 0x5b, 0x96, 0x69, 0x75, 0x2b, 0x33, 0xd4, 0xc8, 0x2b, 0x2a, 0x23, 0x9c, + 0xa5, 0xf1, 0xca, 0x68, 0x58, 0x3d, 0x27, 0x49, 0x49, 0x26, 0x64, 0x85, 0x24, 0x62, 0x30, 0x20, + 0x5a, 0xd8, 0xa2, 0x2a, 0x62, 0x6c, 0xc8, 0x4c, 0x2c, 0x62, 0xc4, 0x24, 0xe5, 0x88, 0x11, 0x23, + 0x46, 0xeb, 0xc1, 0x17, 0xb9, 0x34, 0x79, 0x3d, 0xf8, 0x3a, 0x0b, 0xeb, 0xa1, 0x58, 0x6a, 0x49, + 0x1b, 0xfa, 0x46, 0x83, 0x05, 0xd7, 0xd3, 0x2d, 0x43, 0xef, 0xd9, 0x16, 0xbe, 0x6b, 0x75, 0x1d, + 0xec, 0xba, 0x77, 0xad, 0x5d, 0xbb, 0x52, 0xa6, 0x76, 0x2e, 0xc5, 0x02, 0xab, 0x8a, 0xb5, 0x71, + 0x69, 0x34, 0xac, 0x56, 0x95, 0x5a, 0x24, 0xcb, 0x6a, 0x43, 0xe8, 0x10, 0xe6, 0x82, 0x97, 0xf4, + 0x8e, 0x67, 0xf6, 0x4c, 0x57, 0xf7, 0x4c, 0xdb, 0xaa, 0x9c, 0xa5, 0xf6, 0x2f, 0xc6, 0xe3, 0xd3, + 0x18, 0x63, 0xe3, 0xe2, 0x68, 0x58, 0xbd, 0xa0, 0xd0, 0x20, 0xd9, 0x56, 0x99, 0x88, 0x16, 0x71, + 0xcb, 0xc1, 0x84, 0x11, 0x1b, 0x95, 0xb9, 0xc9, 0x8b, 0x18, 0x32, 0x89, 0x8b, 0x18, 0x82, 0xaa, + 0x45, 0x0c, 0x89, 0xc4, 0xd2, 0x40, 0x77, 0x3c, 0x93, 0x98, 0xdd, 0xd4, 0x9d, 0x7d, 0xec, 0x54, + 0xe6, 0x55, 0x96, 0xb6, 0x64, 0x26, 0x66, 0x29, 0x26, 0x29, 0x5b, 0x8a, 0x11, 0xd1, 0xf7, 0x1a, + 0xc8, 0x43, 0x33, 0x6d, 0xab, 0x49, 0x5e, 0xda, 0x2e, 0x79, 0xbc, 0x05, 0x6a, 0xf4, 0x8d, 0xa7, + 0x3c, 0x9e, 0xc8, 0xde, 0x78, 0x63, 0x34, 0xac, 0x5e, 0x9a, 0xa8, 0x4d, 0x1a, 0xc8, 0x64, 0xa3, + 0xe8, 0x31, 0x14, 0x08, 0x11, 0xd3, 0xf4, 0xc7, 0xa8, 0x2c, 0xd2, 0x31, 0x9c, 0x1f, 0x1f, 0x03, + 0x67, 0x68, 0x9c, 0x1f, 0x0d, 0xab, 0x0b, 0x82, 0x84, 0x64, 0x47, 0x54, 0x85, 0xbe, 0xd3, 0x80, + 0x38, 0xba, 0xea, 0x49, 0xcf, 0x51, 0x2b, 0xaf, 0x8d, 0x59, 0x51, 0x3d, 0xe6, 0x6b, 0xa3, 0x61, + 0x75, 0x45, 0xad, 0x47, 0xb2, 0x3d, 0xc1, 0x56, 0xe4, 0x47, 0xe1, 0x4b, 0xa2, 0x52, 0x99, 0xec, + 0x47, 0x21, 0x93, 0xe8, 0x47, 0x21, 0xa8, 0xf2, 0xa3, 0x90, 0xc8, 0x83, 0xc1, 0xa7, 0x7a, 0xcf, + 0x34, 0x68, 0x32, 0x75, 0x7e, 0x42, 0x30, 0x08, 0x39, 0xc2, 0x60, 0x10, 0x22, 0x63, 0xc1, 0x20, + 0xe2, 0xcd, 0x42, 0x9a, 0xaa, 0xa8, 0xfd, 0x90, 0x87, 0x39, 0xc5, 0x56, 0x43, 0x18, 0x8a, 0xc1, + 0x3e, 0x6a, 0x99, 0x24, 0x48, 0x24, 0x55, 0xb3, 0xfc, 0x89, 0xdf, 0xc6, 0x8e, 0x85, 0x3d, 0xec, + 0x06, 0x3a, 0x68, 0x94, 0xa0, 0x23, 0x71, 0x04, 0x44, 0xc8, 0xed, 0x66, 0x44, 0x1c, 0xfd, 0xa0, + 0x41, 0xa5, 0xaf, 0x1f, 0xb6, 0x02, 0xd0, 0x6d, 0xed, 0xda, 0x4e, 0x6b, 0x80, 0x1d, 0xd3, 0x36, + 0x68, 0x26, 0x5b, 0xb8, 0xf6, 0xc7, 0xc7, 0xc6, 0x85, 0xfa, 0xa6, 0x7e, 0x18, 0xc0, 0xee, 0x2d, + 0xdb, 0xd9, 0xa2, 0xe2, 0xeb, 0x96, 0xe7, 0x1c, 0xb1, 0x80, 0xd5, 0x57, 0xd1, 0x85, 0x31, 0x2d, + 0x28, 0x19, 0xd0, 0x5f, 0x6b, 0xb0, 0xe8, 0xd9, 0x9e, 0xde, 0x6b, 0x75, 0xfc, 0xbe, 0xdf, 0xd3, + 0x3d, 0xf3, 0x00, 0xb7, 0x7c, 0x57, 0xef, 0x62, 0x9e, 0x36, 0x7f, 0x78, 0xfc, 0xd0, 0x1e, 0x11, + 0xf9, 0x1b, 0xa1, 0xf8, 0x0e, 0x91, 0x66, 0x23, 0xab, 0x8d, 0x86, 0xd5, 0x65, 0x4f, 0x41, 0x16, + 0x06, 0x36, 0xaf, 0xa2, 0xa3, 0x37, 0x21, 0x43, 0xca, 0x0a, 0xd3, 0xa0, 0xd9, 0x11, 0x2f, 0x41, + 0x9e, 0xd8, 0x6d, 0xa9, 0x30, 0x48, 0x53, 0x80, 0xf0, 0x3a, 0xbe, 0x45, 0x78, 0xb3, 0x11, 0xaf, + 0xe3, 0x5b, 0x32, 0x2f, 0x05, 0xe8, 0x62, 0xe8, 0x07, 0x5d, 0xf5, 0x62, 0xe4, 0xa6, 0x5d, 0x8c, + 0xb5, 0x83, 0xee, 0x53, 0x17, 0x43, 0x57, 0xd1, 0xc5, 0xc5, 0x50, 0x32, 0x2c, 0xfd, 0xa8, 0xc1, + 0xd2, 0xe4, 0x75, 0x46, 0x97, 0x20, 0xb9, 0x8f, 0x8f, 0x78, 0x4d, 0x76, 0x76, 0x34, 0xac, 0x16, + 0xf7, 0xf1, 0x91, 0xa0, 0x95, 0x50, 0xd1, 0x9f, 0x40, 0xfa, 0x40, 0xef, 0xf9, 0x98, 0xa7, 0xfc, + 0xf5, 0x3a, 0x2b, 0x27, 0xeb, 0x62, 0x39, 0x59, 0x1f, 0xec, 0x77, 0x09, 0x50, 0x0f, 0x66, 0xa1, + 0xfe, 0xd0, 0xd7, 0x2d, 0xcf, 0xf4, 0x8e, 0xd8, 0xdc, 0x51, 0x05, 0xe2, 0xdc, 0x51, 0xe0, 0x83, + 0xc4, 0x75, 0x6d, 0xe9, 0x6f, 0x35, 0x38, 0x3f, 0x71, 0xbd, 0x5f, 0x8a, 0x11, 0x92, 0x49, 0x9c, + 0xbc, 0x3e, 0x2f, 0xc3, 0x10, 0x37, 0x52, 0x39, 0xad, 0x9c, 0xd8, 0x48, 0xe5, 0x12, 0xe5, 0x64, + 0xed, 0x9f, 0x33, 0x90, 0x0f, 0x0b, 0x3c, 0x74, 0x07, 0xca, 0x06, 0x36, 0xfc, 0x41, 0xcf, 0xec, + 0x50, 0x4f, 0x23, 0x4e, 0xcd, 0x2a, 0x6a, 0x1a, 0x5d, 0x25, 0x9a, 0xe4, 0xde, 0xb3, 0x31, 0x12, + 0xba, 0x06, 0x39, 0x5e, 0xc8, 0x1c, 0xd1, 0xb8, 0x56, 0x6c, 0x2c, 0x8e, 0x86, 0x55, 0x14, 0x60, + 0x82, 0x68, 0xc8, 0x87, 0x9a, 0x00, 0xac, 0x33, 0xb0, 0x89, 0x3d, 0x9d, 0x97, 0x54, 0x15, 0x79, + 0x37, 0x3c, 0x08, 0xe9, 0xac, 0xc6, 0x8f, 0xf8, 0xc5, 0x1a, 0x3f, 0x42, 0xd1, 0xe7, 0x00, 0x7d, + 0xdd, 0xb4, 0x98, 0x1c, 0xaf, 0x9f, 0x6a, 0x93, 0x22, 0xec, 0x66, 0xc8, 0xc9, 0xb4, 0x47, 0x92, + 0xa2, 0xf6, 0x08, 0x45, 0x0f, 0x20, 0xcb, 0x7b, 0x19, 0x95, 0x0c, 0xdd, 0xbc, 0xcb, 0x93, 0x54, + 0x73, 0xb5, 0xb4, 0x1a, 0xe7, 0x22, 0x62, 0x35, 0xce, 0x21, 0x32, 0x6d, 0x3d, 0x73, 0x17, 0x7b, + 0x66, 0x1f, 0xd3, 0x68, 0xc2, 0xa7, 0x2d, 0xc0, 0xc4, 0x69, 0x0b, 0x30, 0x74, 0x1d, 0x40, 0xf7, + 0x36, 0x6d, 0xd7, 0x7b, 0x60, 0x75, 0x30, 0xad, 0x88, 0x72, 0x6c, 0xf8, 0x11, 0x2a, 0x0e, 0x3f, + 0x42, 0xd1, 0x87, 0x50, 0x18, 0xf0, 0x37, 0x70, 0xbb, 0x87, 0x69, 0xc5, 0x93, 0x63, 0x09, 0x83, + 0x00, 0x0b, 0xb2, 0x22, 0x37, 0xba, 0x0d, 0xb3, 0x1d, 0xdb, 0xea, 0xf8, 0x8e, 0x83, 0xad, 0xce, + 0xd1, 0xb6, 0xbe, 0x8b, 0x69, 0x75, 0x93, 0x63, 0xae, 0x12, 0x23, 0x89, 0xae, 0x12, 0x23, 0xa1, + 0xf7, 0x20, 0x1f, 0x76, 0x86, 0x68, 0x01, 0x93, 0xe7, 0x8d, 0x86, 0x00, 0x14, 0x84, 0x23, 0x4e, + 0x32, 0x78, 0xd3, 0xbd, 0xc9, 0x9d, 0x0e, 0xd3, 0xa2, 0x84, 0x0f, 0x5e, 0x80, 0xc5, 0xc1, 0x0b, + 0xb0, 0x10, 0xdf, 0x4b, 0xc7, 0xc5, 0xf7, 0x70, 0xbb, 0x14, 0xcb, 0xa5, 0x8d, 0x54, 0x6e, 0xb6, + 0x5c, 0xae, 0xfd, 0xbb, 0x06, 0xf3, 0x2a, 0xaf, 0x89, 0x79, 0xb0, 0xf6, 0x5c, 0x3c, 0xf8, 0x53, + 0xc8, 0x0d, 0x6c, 0xa3, 0xe5, 0x0e, 0x70, 0x87, 0xc7, 0x83, 0x98, 0xff, 0x6e, 0xd9, 0xc6, 0xf6, + 0x00, 0x77, 0x3e, 0x33, 0xbd, 0xbd, 0xb5, 0x03, 0xdb, 0x34, 0xee, 0x99, 0x2e, 0x77, 0xb4, 0x01, + 0xa3, 0x48, 0x49, 0x4a, 0x96, 0x83, 0x8d, 0x1c, 0x64, 0x98, 0x95, 0xda, 0x7f, 0x24, 0xa1, 0x1c, + 0xf7, 0xd4, 0xdf, 0xd2, 0xa3, 0xa0, 0xc7, 0x90, 0x35, 0x59, 0x0d, 0xc4, 0x73, 0xa8, 0x3f, 0x10, + 0x22, 0x66, 0x3d, 0x6a, 0x88, 0xd6, 0x0f, 0xde, 0xa9, 0xf3, 0x62, 0x89, 0x4e, 0x01, 0xd5, 0xcc, + 0x25, 0x65, 0xcd, 0x1c, 0x44, 0x4d, 0xc8, 0xba, 0xd8, 0x39, 0x30, 0x3b, 0x98, 0xc7, 0xa3, 0xaa, + 0xa8, 0xb9, 0x63, 0x3b, 0x98, 0xe8, 0xdc, 0x66, 0x2c, 0x91, 0x4e, 0x2e, 0x23, 0xeb, 0xe4, 0x20, + 0xfa, 0x14, 0xf2, 0x1d, 0xdb, 0xda, 0x35, 0xbb, 0x9b, 0xfa, 0x80, 0x47, 0xa4, 0x0b, 0x2a, 0xad, + 0x37, 0x02, 0x26, 0xde, 0xd7, 0x09, 0x7e, 0xc6, 0xfa, 0x3a, 0x21, 0x57, 0xb4, 0xa0, 0xff, 0x97, + 0x02, 0x88, 0x16, 0x07, 0xbd, 0x0f, 0x05, 0x7c, 0x88, 0x3b, 0xbe, 0x67, 0xd3, 0x5e, 0xa7, 0x16, + 0xb5, 0x48, 0x03, 0x58, 0x72, 0x7b, 0x88, 0x50, 0xb2, 0x37, 0x2d, 0xbd, 0x8f, 0xdd, 0x81, 0xde, + 0x09, 0x7a, 0xab, 0x74, 0x30, 0x21, 0x28, 0xee, 0xcd, 0x10, 0x44, 0xaf, 0x43, 0x8a, 0x76, 0x63, + 0x59, 0x5b, 0x15, 0x8d, 0x86, 0xd5, 0x92, 0x25, 0xf7, 0x61, 0x29, 0x1d, 0x7d, 0x0c, 0xc5, 0xfd, + 0xd0, 0xf1, 0xc8, 0xd8, 0x52, 0x54, 0x80, 0x26, 0xb7, 0x11, 0x41, 0x1a, 0xdd, 0x8c, 0x88, 0xa3, + 0x5d, 0x28, 0xe8, 0x96, 0x65, 0x7b, 0xf4, 0xb5, 0x13, 0xb4, 0x5a, 0xaf, 0x4c, 0x72, 0xd3, 0xfa, + 0x5a, 0xc4, 0xcb, 0xd2, 0x25, 0x1a, 0x2f, 0x04, 0x0d, 0x62, 0xbc, 0x10, 0x60, 0xd4, 0x84, 0x4c, + 0x4f, 0x6f, 0xe3, 0x5e, 0x10, 0xe7, 0x5f, 0x9b, 0x68, 0xe2, 0x1e, 0x65, 0x63, 0xda, 0x69, 0x43, + 0x97, 0xc9, 0x89, 0x0d, 0x5d, 0x86, 0x2c, 0xed, 0x42, 0x39, 0x3e, 0x9e, 0xe9, 0xd2, 0x83, 0x2b, + 0x62, 0x7a, 0x90, 0x3f, 0x36, 0x23, 0xd1, 0xa1, 0x20, 0x0c, 0xea, 0x34, 0x4c, 0xd4, 0xfe, 0x5e, + 0x83, 0x79, 0xd5, 0xde, 0x45, 0x9b, 0xc2, 0x8e, 0xd7, 0x78, 0xdb, 0x48, 0xe1, 0xea, 0x5c, 0x76, + 0xc2, 0x56, 0x8f, 0x36, 0x7a, 0x03, 0x4a, 0x96, 0x6d, 0xe0, 0x96, 0x4e, 0x0c, 0xf4, 0x4c, 0xd7, + 0xab, 0x24, 0x68, 0x2b, 0x9e, 0xb6, 0x9b, 0x08, 0x65, 0x2d, 0x20, 0x08, 0xd2, 0x45, 0x89, 0x50, + 0xfb, 0x0a, 0x66, 0x63, 0xcd, 0x60, 0x29, 0x59, 0x49, 0x4c, 0x99, 0xac, 0x44, 0x6f, 0x90, 0xe4, + 0x74, 0x6f, 0x90, 0xda, 0x5f, 0x24, 0xa0, 0x20, 0x54, 0xe6, 0xe8, 0x09, 0xcc, 0xf2, 0xb7, 0x99, + 0x69, 0x75, 0x59, 0x05, 0x98, 0xe0, 0x6d, 0xa2, 0xb1, 0x93, 0x92, 0x0d, 0xbb, 0xbd, 0x1d, 0xf2, + 0xd2, 0x02, 0x90, 0x76, 0xf1, 0x5c, 0x09, 0x13, 0x0c, 0x97, 0x64, 0x0a, 0x7a, 0x0c, 0x8b, 0xfe, + 0x80, 0xd4, 0xa5, 0x2d, 0x97, 0x9f, 0x39, 0xb4, 0x2c, 0xbf, 0xdf, 0xc6, 0x0e, 0x1d, 0x7d, 0x9a, + 0x55, 0x4a, 0x8c, 0x23, 0x38, 0x94, 0xb8, 0x4f, 0xe9, 0x62, 0xa5, 0xa4, 0xa2, 0x0b, 0xf3, 0x90, + 0x9a, 0x72, 0x1e, 0xee, 0x00, 0x1a, 0xef, 0xc6, 0x4b, 0x6b, 0xa0, 0x4d, 0xb7, 0x06, 0xb5, 0x43, + 0x28, 0xc7, 0x7b, 0xec, 0x2f, 0x68, 0x2d, 0xf7, 0x21, 0x1f, 0x76, 0xc8, 0xd1, 0x5b, 0x90, 0x71, + 0xb0, 0xee, 0xda, 0x16, 0xdf, 0x2d, 0x74, 0xdb, 0x33, 0x44, 0xdc, 0xf6, 0x0c, 0x79, 0x06, 0x63, + 0x8f, 0x60, 0x86, 0x4d, 0xd2, 0x2d, 0xb3, 0xe7, 0x61, 0x07, 0xdd, 0x84, 0x8c, 0xeb, 0xe9, 0x1e, + 0x76, 0x2b, 0xda, 0x4a, 0xf2, 0x72, 0xe9, 0xda, 0xe2, 0x78, 0xfb, 0x9b, 0x90, 0xd9, 0x38, 0x18, + 0xa7, 0x38, 0x0e, 0x86, 0xd4, 0xfe, 0x5c, 0x83, 0x19, 0xb1, 0xcb, 0xff, 0x7c, 0xd4, 0x9e, 0x6c, + 0x32, 0x48, 0xe0, 0x98, 0x11, 0x0f, 0x03, 0x4e, 0x6f, 0x2e, 0xc9, 0x5b, 0x90, 0x1d, 0x25, 0xb4, + 0x7c, 0x17, 0x3b, 0xdc, 0x5b, 0xe9, 0x5b, 0x90, 0xc1, 0x3b, 0xae, 0xe4, 0xed, 0x10, 0xa1, 0x7c, + 0x19, 0xc8, 0x58, 0xc5, 0xa3, 0x05, 0xd4, 0x8d, 0x1a, 0x38, 0x64, 0x93, 0xb9, 0x34, 0x18, 0x4d, + 0xdb, 0xc0, 0xa1, 0x21, 0x4b, 0x12, 0x17, 0x43, 0x96, 0x44, 0x78, 0x06, 0x97, 0xf9, 0x31, 0x4d, + 0xc7, 0x1a, 0x1d, 0x15, 0xc4, 0x72, 0x80, 0xe4, 0x09, 0x72, 0x80, 0xb7, 0x21, 0x4b, 0x83, 0x6e, + 0xb8, 0xc5, 0xe9, 0x9a, 0x10, 0x48, 0x3e, 0x26, 0x65, 0xc8, 0x53, 0x42, 0x4d, 0xfa, 0x57, 0x86, + 0x9a, 0x16, 0x9c, 0xdf, 0xd3, 0xdd, 0x56, 0x10, 0x1c, 0x8d, 0x96, 0xee, 0xb5, 0xc2, 0xbd, 0x9e, + 0xa1, 0xf9, 0x3f, 0x6d, 0x3e, 0xee, 0xe9, 0xee, 0x76, 0xc0, 0xb3, 0xe6, 0x6d, 0x8d, 0xef, 0xfc, + 0x45, 0x35, 0x07, 0xda, 0x81, 0x05, 0xb5, 0xf2, 0x2c, 0x1d, 0x39, 0xed, 0x8d, 0xbb, 0x4f, 0xd5, + 0x3c, 0xa7, 0x20, 0xa3, 0x6f, 0x35, 0xa8, 0x90, 0xb7, 0xa0, 0x83, 0xbf, 0xf4, 0x4d, 0x07, 0xf7, + 0x89, 0x5b, 0xb4, 0xec, 0x03, 0xec, 0xf4, 0xf4, 0x23, 0x7e, 0xcc, 0x74, 0x71, 0x3c, 0xe4, 0x6f, + 0xd9, 0x46, 0x53, 0x10, 0x60, 0x8f, 0x36, 0x90, 0xc1, 0x07, 0x4c, 0x89, 0xf8, 0x68, 0x6a, 0x0e, + 0xc1, 0x85, 0xe0, 0x04, 0x0d, 0xad, 0xc2, 0xb1, 0x0d, 0xad, 0xd7, 0x21, 0x35, 0xb0, 0xed, 0x1e, + 0x2d, 0xbf, 0x78, 0xa6, 0x47, 0x7e, 0x8b, 0x99, 0x1e, 0xf9, 0x2d, 0xf6, 0x1c, 0x36, 0x52, 0xb9, + 0x5c, 0x39, 0x4f, 0x5e, 0x87, 0x25, 0xf9, 0x64, 0x6a, 0x7c, 0x43, 0x25, 0x4f, 0x7d, 0x43, 0xa5, + 0x4e, 0x30, 0x1b, 0xe9, 0xa9, 0x67, 0x23, 0x33, 0xfd, 0x6c, 0xd4, 0xbe, 0x4b, 0x40, 0x51, 0x3a, + 0x3c, 0xfb, 0x7d, 0x4e, 0xc3, 0xdf, 0x24, 0x60, 0x51, 0xfd, 0x48, 0xa7, 0x52, 0x8a, 0xde, 0x01, + 0x92, 0x54, 0xde, 0x8d, 0x92, 0xae, 0x85, 0xb1, 0x4a, 0x94, 0x4e, 0x67, 0x90, 0x91, 0x8e, 0x9d, + 0xbf, 0x05, 0xe2, 0xe8, 0x31, 0x14, 0x4c, 0xe1, 0xa4, 0x2f, 0xa9, 0x3a, 0x90, 0x11, 0xcf, 0xf7, + 0x58, 0x8b, 0x62, 0xc2, 0xa9, 0x9e, 0xa8, 0xaa, 0x91, 0x81, 0x14, 0xc9, 0x0a, 0x6b, 0x07, 0x90, + 0xe5, 0xc3, 0x41, 0xef, 0x42, 0x9e, 0xc6, 0x62, 0x5a, 0x5d, 0xb1, 0x14, 0x9e, 0xa6, 0x37, 0x04, + 0x8c, 0xdd, 0x74, 0xc9, 0x05, 0x18, 0xfa, 0x23, 0x00, 0x12, 0x7e, 0x78, 0x14, 0x4e, 0xd0, 0x58, + 0x46, 0xab, 0xb8, 0x81, 0x6d, 0x8c, 0x85, 0xde, 0x7c, 0x08, 0xd6, 0xfe, 0x31, 0x01, 0x05, 0xf1, + 0x6c, 0xf1, 0x99, 0x8c, 0x7f, 0x0d, 0x41, 0x85, 0xdd, 0xd2, 0x0d, 0x83, 0xfc, 0xc5, 0xc1, 0x8b, + 0x72, 0x75, 0xe2, 0x24, 0x05, 0xff, 0xaf, 0x05, 0x12, 0xac, 0x9e, 0xa2, 0xf7, 0x27, 0xcc, 0x18, + 0x49, 0xb0, 0x5a, 0x8e, 0xd3, 0x96, 0xf6, 0x61, 0x41, 0xa9, 0x4a, 0xac, 0x82, 0xd2, 0xcf, 0xab, + 0x0a, 0xfa, 0xbb, 0x34, 0x2c, 0x28, 0xcf, 0x74, 0x63, 0x1e, 0x9c, 0x7c, 0x2e, 0x1e, 0xfc, 0x97, + 0x9a, 0x6a, 0x66, 0xd9, 0x81, 0xce, 0xfb, 0x53, 0x1c, 0x34, 0x3f, 0xaf, 0x39, 0x96, 0xdd, 0x22, + 0xfd, 0x4c, 0x3e, 0x99, 0x99, 0xd6, 0x27, 0xd1, 0x55, 0x56, 0x50, 0x52, 0x5b, 0xec, 0xb8, 0x25, + 0xd8, 0xa1, 0x31, 0x53, 0x59, 0x0e, 0xa1, 0x8f, 0xa1, 0x18, 0x48, 0xb0, 0x36, 0x46, 0x2e, 0xea, + 0x31, 0x70, 0x9e, 0x78, 0x27, 0x63, 0x46, 0xc4, 0x85, 0x28, 0x99, 0x3f, 0x41, 0x94, 0x84, 0xe3, + 0xa2, 0xe4, 0x0b, 0xf5, 0x4d, 0x29, 0xd4, 0x0e, 0x35, 0x98, 0x8d, 0x5d, 0xa5, 0xf8, 0xcd, 0xbf, + 0x73, 0xa4, 0x07, 0xfc, 0x46, 0x83, 0x7c, 0x78, 0x53, 0x07, 0xad, 0x41, 0x06, 0xb3, 0xdb, 0x1e, + 0x2c, 0xec, 0xcc, 0xc5, 0x6e, 0xe2, 0x11, 0x1a, 0xbf, 0x7b, 0x17, 0xbb, 0xe0, 0xd1, 0xe4, 0x82, + 0xcf, 0x90, 0x80, 0xff, 0x8b, 0x16, 0x24, 0xe0, 0x63, 0xa3, 0x48, 0xfe, 0xfa, 0x51, 0x9c, 0xde, + 0xd4, 0xfd, 0x13, 0x40, 0x9a, 0x8e, 0x85, 0x14, 0xd2, 0x1e, 0x76, 0xfa, 0xa6, 0xa5, 0xf7, 0xa8, + 0x2b, 0xe6, 0xd8, 0xae, 0x0e, 0x30, 0x71, 0x57, 0x07, 0x18, 0xda, 0x83, 0xd9, 0xa8, 0x3d, 0x47, + 0xd5, 0xa8, 0xaf, 0xfe, 0x7d, 0x22, 0x33, 0xb1, 0x23, 0x83, 0x98, 0xa4, 0x7c, 0x76, 0x1f, 0x23, + 0x22, 0x03, 0x4a, 0x1d, 0xdb, 0xf2, 0x74, 0xd3, 0xc2, 0x0e, 0x33, 0x94, 0x54, 0x5d, 0x7d, 0xba, + 0x21, 0xf1, 0xb0, 0xa6, 0x89, 0x2c, 0x27, 0x5f, 0x7d, 0x92, 0x69, 0xe8, 0x0b, 0x28, 0x06, 0x85, + 0x10, 0x33, 0x92, 0x52, 0x5d, 0x7d, 0x5a, 0x17, 0x59, 0xd8, 0x66, 0x90, 0xa4, 0xe4, 0xab, 0x4f, + 0x12, 0x09, 0x7d, 0x0e, 0x33, 0x3d, 0x52, 0xa1, 0xad, 0x1f, 0x0e, 0x4c, 0x07, 0x1b, 0xea, 0xcb, + 0x78, 0xf7, 0x04, 0x0e, 0x16, 0xb8, 0x44, 0x19, 0xf9, 0x0e, 0x82, 0x48, 0x21, 0xeb, 0xd1, 0xd7, + 0x0f, 0x9b, 0xbe, 0xe5, 0xae, 0x1f, 0xf2, 0x8b, 0x55, 0x59, 0xd5, 0x7a, 0x6c, 0xca, 0x4c, 0x6c, + 0x3d, 0x62, 0x92, 0xf2, 0x7a, 0xc4, 0x88, 0xe8, 0x1e, 0x8d, 0xcb, 0x6c, 0x92, 0xd8, 0xa5, 0xbc, + 0xc5, 0xb1, 0x84, 0x8a, 0xcd, 0x0f, 0x6b, 0xc7, 0xf0, 0x5f, 0x92, 0xd2, 0x50, 0x03, 0xea, 0x41, + 0x79, 0x60, 0x1b, 0xf4, 0xb1, 0x9b, 0xd8, 0xf3, 0x1d, 0x0b, 0x1b, 0xbc, 0x50, 0x5a, 0x1e, 0xd3, + 0x2a, 0x71, 0xb1, 0xd7, 0x57, 0x5c, 0x56, 0xbe, 0x62, 0x19, 0xa7, 0xa2, 0xaf, 0x61, 0x3e, 0x76, + 0xc5, 0x88, 0x3d, 0x47, 0x41, 0x75, 0x44, 0xb1, 0xa1, 0xe0, 0x64, 0x35, 0xad, 0x4a, 0x87, 0x64, + 0x59, 0x69, 0x85, 0x58, 0xef, 0xea, 0x56, 0x77, 0xc3, 0x6e, 0xef, 0x58, 0xbc, 0x08, 0xd4, 0xdb, + 0x3d, 0xcc, 0x6f, 0xd9, 0xc5, 0xac, 0xdf, 0x56, 0x70, 0x32, 0xeb, 0x2a, 0x1d, 0xb2, 0x75, 0x15, + 0x47, 0x78, 0x9d, 0x88, 0xa4, 0x15, 0xe1, 0xb5, 0x3b, 0xd5, 0x75, 0x22, 0xc6, 0x20, 0x5c, 0x27, + 0x62, 0x80, 0xe2, 0x3a, 0x11, 0x23, 0xb0, 0x9b, 0x68, 0x1d, 0xdb, 0xea, 0x98, 0x3d, 0x93, 0xb6, + 0xb8, 0xd9, 0xa4, 0x96, 0xd4, 0x37, 0xd1, 0xc6, 0x18, 0x83, 0x9b, 0x68, 0x63, 0x84, 0xf8, 0x4d, + 0xb4, 0x31, 0x06, 0xf4, 0x19, 0xcc, 0xec, 0xea, 0x66, 0xcf, 0x77, 0xf8, 0xbd, 0x9a, 0x59, 0xd5, + 0x43, 0xdd, 0x62, 0x1c, 0x51, 0x4a, 0xbe, 0x1b, 0x01, 0xe2, 0x29, 0x80, 0x00, 0x37, 0x72, 0x41, + 0x73, 0x6a, 0x23, 0x95, 0x4b, 0x97, 0x33, 0x1b, 0xa9, 0x1c, 0x94, 0x0b, 0xb5, 0x87, 0x30, 0x1b, + 0x8b, 0x67, 0xe8, 0x23, 0x08, 0xef, 0xe0, 0x3c, 0x3a, 0x1a, 0x04, 0xc9, 0xb2, 0x74, 0x67, 0x87, + 0xe0, 0xaa, 0x3b, 0x3b, 0x04, 0xaf, 0x7d, 0x9f, 0x82, 0x5c, 0xb0, 0x61, 0x4e, 0xa5, 0xfc, 0x59, + 0x85, 0x6c, 0x1f, 0xbb, 0xf4, 0x9e, 0x4d, 0x22, 0xca, 0xa2, 0x38, 0x24, 0x66, 0x51, 0x1c, 0x92, + 0x93, 0xbc, 0xe4, 0x33, 0x25, 0x79, 0xa9, 0xa9, 0x93, 0x3c, 0x4c, 0x8f, 0x96, 0x85, 0x40, 0x1c, + 0x9c, 0xec, 0x3c, 0x3d, 0xba, 0x07, 0x07, 0xcf, 0xa2, 0x60, 0xec, 0xe0, 0x59, 0x24, 0xa1, 0x7d, + 0x38, 0x2b, 0x9c, 0x3e, 0xf1, 0xb6, 0x23, 0x09, 0xc0, 0xa5, 0xc9, 0xe7, 0xf8, 0x4d, 0xca, 0xc5, + 0xc2, 0xcc, 0x7e, 0x0c, 0x15, 0xb3, 0xe4, 0x38, 0x8d, 0xb8, 0x84, 0x81, 0xdb, 0x7e, 0x77, 0x93, + 0x4f, 0x7b, 0x36, 0x72, 0x09, 0x11, 0x17, 0x5d, 0x42, 0xc4, 0x6b, 0xff, 0x9b, 0x80, 0x92, 0xfc, + 0xbc, 0xa7, 0xe2, 0x18, 0xef, 0x42, 0x1e, 0x1f, 0x9a, 0x5e, 0xab, 0x63, 0x1b, 0x98, 0x97, 0x8a, + 0x74, 0x9d, 0x09, 0x78, 0xc3, 0x36, 0xa4, 0x75, 0x0e, 0x30, 0xd1, 0x9b, 0x92, 0x53, 0x79, 0x53, + 0xd4, 0xe5, 0x4d, 0x4d, 0xd1, 0xe5, 0x55, 0xae, 0x53, 0xfe, 0x74, 0xd6, 0xa9, 0xf6, 0x53, 0x02, + 0xca, 0xf1, 0xb7, 0xca, 0xcb, 0xb1, 0x05, 0xe5, 0xdd, 0x94, 0x9c, 0x7a, 0x37, 0x7d, 0x0c, 0x45, + 0x92, 0x0a, 0xea, 0x9e, 0xc7, 0xaf, 0xe5, 0xa6, 0x68, 0x36, 0xc7, 0xa2, 0x91, 0x6f, 0xad, 0x05, + 0xb8, 0x14, 0x8d, 0x04, 0x7c, 0xcc, 0x75, 0xd3, 0x27, 0x74, 0xdd, 0x6f, 0x13, 0x50, 0xdc, 0xb2, + 0x8d, 0x47, 0x2c, 0x4b, 0xf4, 0x5e, 0x96, 0xf9, 0x7c, 0x91, 0x21, 0xad, 0x36, 0x0b, 0x45, 0x29, + 0x4d, 0xac, 0x7d, 0xc7, 0xfc, 0x4c, 0x7e, 0x1b, 0xff, 0xfe, 0xe6, 0xa5, 0x04, 0x33, 0x62, 0x76, + 0x5b, 0x6b, 0xc0, 0x6c, 0x2c, 0x19, 0x15, 0x1f, 0x40, 0x9b, 0xe6, 0x01, 0x6a, 0x37, 0x61, 0x5e, + 0x95, 0xa5, 0x09, 0x51, 0x47, 0x9b, 0xe2, 0x68, 0xea, 0x36, 0xcc, 0xab, 0xb2, 0xad, 0x93, 0x0f, + 0xe7, 0x23, 0x7e, 0xec, 0xcb, 0xf3, 0xa2, 0x13, 0xcb, 0xdf, 0x82, 0x39, 0x45, 0x7e, 0x74, 0x72, + 0x3d, 0x7f, 0x95, 0x80, 0x82, 0x90, 0xf5, 0xc8, 0xa1, 0x5e, 0x9b, 0x32, 0xd4, 0x3f, 0x84, 0x39, + 0x5e, 0xed, 0xd1, 0xfb, 0x81, 0xb2, 0x67, 0xd1, 0x2f, 0xaa, 0x04, 0xf2, 0x78, 0x60, 0x40, 0xe3, + 0x54, 0x74, 0x1d, 0xa0, 0xa3, 0x7b, 0xb8, 0x6b, 0x3b, 0x26, 0x66, 0xd5, 0x71, 0x78, 0x2e, 0x17, + 0xa0, 0xf2, 0xb9, 0x5c, 0x80, 0xa2, 0x86, 0x50, 0x04, 0x32, 0x77, 0x65, 0xaf, 0x13, 0x5a, 0x82, + 0x85, 0x94, 0x98, 0xcf, 0x16, 0x25, 0x42, 0xed, 0x3f, 0xc3, 0x66, 0x48, 0xf4, 0x81, 0xc1, 0x2d, + 0x28, 0x0f, 0x82, 0x1f, 0x2d, 0x5e, 0x72, 0xb3, 0xa0, 0x47, 0x0b, 0xc8, 0x90, 0xb6, 0x11, 0xab, + 0xbd, 0x4b, 0x32, 0x45, 0xd6, 0xc3, 0xcb, 0xf1, 0x8c, 0x42, 0x4f, 0x33, 0x56, 0x97, 0x97, 0x64, + 0x8a, 0xe0, 0xb8, 0xd9, 0xe3, 0x1d, 0x97, 0x96, 0xf3, 0xe9, 0xda, 0x37, 0x1a, 0xcc, 0xc6, 0x3e, + 0x80, 0x40, 0x57, 0x21, 0x47, 0xbf, 0x4e, 0x8c, 0x1a, 0x19, 0xd4, 0x67, 0x28, 0x26, 0x0d, 0x20, + 0xcb, 0x21, 0xf4, 0x1e, 0xe4, 0xc3, 0x6f, 0x22, 0xf8, 0x71, 0x3a, 0xdb, 0xd5, 0x01, 0x28, 0xed, + 0xea, 0x00, 0xe4, 0x3d, 0x90, 0x3f, 0x83, 0xf3, 0x13, 0xbf, 0x86, 0x38, 0xd1, 0xd1, 0x6d, 0xd4, + 0xcc, 0x48, 0x9d, 0xa8, 0x99, 0x71, 0x08, 0x8b, 0xea, 0x8f, 0x14, 0x04, 0xeb, 0x89, 0x63, 0xad, + 0x47, 0xb3, 0x9f, 0x9c, 0x72, 0xf6, 0x13, 0xb5, 0x7d, 0xda, 0xfd, 0x09, 0x3f, 0x06, 0x40, 0x57, + 0x20, 0x3d, 0xb0, 0xed, 0x9e, 0xcb, 0xef, 0xab, 0x50, 0x73, 0x14, 0x10, 0xcd, 0x51, 0xe0, 0x19, + 0x7a, 0x4d, 0x7e, 0xe0, 0xc1, 0xd1, 0xa7, 0x0d, 0x2f, 0x60, 0x76, 0xdf, 0xbc, 0x0a, 0xb9, 0xe0, + 0x4e, 0x00, 0x02, 0xc8, 0x3c, 0xdc, 0x59, 0xdf, 0x59, 0xbf, 0x59, 0x3e, 0x83, 0x0a, 0x90, 0xdd, + 0x5a, 0xbf, 0x7f, 0xf3, 0xee, 0xfd, 0xdb, 0x65, 0x8d, 0xfc, 0x68, 0xee, 0xdc, 0xbf, 0x4f, 0x7e, + 0x24, 0xde, 0xbc, 0x27, 0xde, 0x33, 0xe4, 0x79, 0xf1, 0x0c, 0xe4, 0xd6, 0x06, 0x03, 0x1a, 0xd2, + 0x98, 0xec, 0xfa, 0x81, 0x49, 0xe2, 0x64, 0x59, 0x43, 0x59, 0x48, 0x3e, 0x78, 0xb0, 0x59, 0x4e, + 0xa0, 0x79, 0x28, 0xdf, 0xc4, 0xba, 0xd1, 0x33, 0x2d, 0x1c, 0xbc, 0x15, 0xca, 0xc9, 0xc6, 0x93, + 0x7f, 0xfb, 0x79, 0x59, 0xfb, 0xe9, 0xe7, 0x65, 0xed, 0x7f, 0x7e, 0x5e, 0xd6, 0xbe, 0xff, 0x65, + 0xf9, 0xcc, 0x4f, 0xbf, 0x2c, 0x9f, 0xf9, 0xaf, 0x5f, 0x96, 0xcf, 0xfc, 0xe9, 0xd5, 0xae, 0xe9, + 0xed, 0xf9, 0xed, 0x7a, 0xc7, 0xee, 0xf3, 0xcf, 0xac, 0x07, 0x8e, 0x4d, 0xc2, 0x2f, 0xff, 0xb5, + 0x1a, 0xff, 0xfe, 0xfa, 0x1f, 0x12, 0x17, 0xd6, 0xe8, 0xcf, 0x2d, 0xc6, 0x57, 0xbf, 0x6b, 0xd7, + 0x19, 0x40, 0xbf, 0xb8, 0x75, 0xdb, 0x19, 0xfa, 0x65, 0xed, 0xbb, 0xff, 0x1f, 0x00, 0x00, 0xff, + 0xff, 0x1e, 0xad, 0x40, 0xef, 0xba, 0x3d, 0x00, 0x00, } func (m *EventSequence) Marshal() (dAtA []byte, err error) { @@ -6147,6 +6239,18 @@ func (m *Error) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.FailureInfo != nil { + { + size, err := m.FailureInfo.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintEvents(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x7a + } if m.Reason != nil { { size := m.Reason.Size() @@ -6937,6 +7041,57 @@ func (m *ReconciliationError) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *FailureInfo) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *FailureInfo) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *FailureInfo) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.ContainerName) > 0 { + i -= len(m.ContainerName) + copy(dAtA[i:], m.ContainerName) + i = encodeVarintEvents(dAtA, i, uint64(len(m.ContainerName))) + i-- + dAtA[i] = 0x22 + } + if len(m.Categories) > 0 { + for iNdEx := len(m.Categories) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.Categories[iNdEx]) + copy(dAtA[i:], m.Categories[iNdEx]) + i = encodeVarintEvents(dAtA, i, uint64(len(m.Categories[iNdEx]))) + i-- + dAtA[i] = 0x1a + } + } + if len(m.TerminationMessage) > 0 { + i -= len(m.TerminationMessage) + copy(dAtA[i:], m.TerminationMessage) + i = encodeVarintEvents(dAtA, i, uint64(len(m.TerminationMessage))) + i-- + dAtA[i] = 0x12 + } + if m.ExitCode != 0 { + i = encodeVarintEvents(dAtA, i, uint64(m.ExitCode)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + func (m *JobRunPreempted) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -8218,6 +8373,10 @@ func (m *Error) Size() (n int) { if m.Reason != nil { n += m.Reason.Size() } + if m.FailureInfo != nil { + l = m.FailureInfo.Size() + n += 1 + l + sovEvents(uint64(l)) + } return n } @@ -8588,6 +8747,32 @@ func (m *ReconciliationError) Size() (n int) { return n } +func (m *FailureInfo) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.ExitCode != 0 { + n += 1 + sovEvents(uint64(m.ExitCode)) + } + l = len(m.TerminationMessage) + if l > 0 { + n += 1 + l + sovEvents(uint64(l)) + } + if len(m.Categories) > 0 { + for _, s := range m.Categories { + l = len(s) + n += 1 + l + sovEvents(uint64(l)) + } + } + l = len(m.ContainerName) + if l > 0 { + n += 1 + l + sovEvents(uint64(l)) + } + return n +} + func (m *JobRunPreempted) Size() (n int) { if m == nil { return 0 @@ -15002,6 +15187,42 @@ func (m *Error) Unmarshal(dAtA []byte) error { } m.Reason = &Error_ReconciliationError{v} iNdEx = postIndex + case 15: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field FailureInfo", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowEvents + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthEvents + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthEvents + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.FailureInfo == nil { + m.FailureInfo = &FailureInfo{} + } + if err := m.FailureInfo.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipEvents(dAtA[iNdEx:]) @@ -16584,6 +16805,171 @@ func (m *ReconciliationError) Unmarshal(dAtA []byte) error { } return nil } +func (m *FailureInfo) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowEvents + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: FailureInfo: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: FailureInfo: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ExitCode", wireType) + } + m.ExitCode = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowEvents + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.ExitCode |= int32(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field TerminationMessage", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowEvents + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthEvents + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthEvents + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.TerminationMessage = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Categories", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowEvents + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthEvents + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthEvents + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Categories = append(m.Categories, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ContainerName", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowEvents + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthEvents + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthEvents + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.ContainerName = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipEvents(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthEvents + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *JobRunPreempted) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 diff --git a/pkg/armadaevents/events.proto b/pkg/armadaevents/events.proto index abd6fc58466..b798b924bd0 100644 --- a/pkg/armadaevents/events.proto +++ b/pkg/armadaevents/events.proto @@ -436,6 +436,9 @@ message Error { JobRejected jobRejected = 13; ReconciliationError reconciliationError = 14; } + // Structured failure metadata (exit code, condition, categories). + // Set by the executor for pod failures; nil for non-pod errors. + FailureInfo failure_info = 15; } // Represents an error associated with a particular Kubernetes resource. @@ -530,6 +533,20 @@ message ReconciliationError { string message = 1; } +// Structured failure metadata extracted from pod status at the executor. +// Attached to Error messages to enable observability in Lookout and +// category-based filtering. +message FailureInfo { + // Exit code of the first failed container (0 if no container terminated with failure). + int32 exit_code = 1; + // Termination message from the first failed container. + string termination_message = 2; + // Executor-assigned category labels, matched from ErrorCategories config. + repeated string categories = 3; + // Name of the container that exit_code and termination_message were extracted from. + string container_name = 4; +} + // Message to indicate that a JobRun has been preempted. message JobRunPreempted{ reserved 1 to 4;