From 8da7107d31101c0f3f89a26fcd6a41abbb849a01 Mon Sep 17 00:00:00 2001 From: Mike Cohen Date: Fri, 1 Aug 2025 18:58:46 +1000 Subject: [PATCH] Refactored parsers to detect errors. Option parsing is now more consistent and stricted. This reveals more errors. --- Makefile | 3 + array.go | 75 +++------ bitfield.go | 65 +++++--- enumeration.go | 82 +++++----- errors.go | 7 + fixtures/TestErrors.golden | 26 ++- fixtures/TestFlagsParser.golden | 4 +- fixtures/TestStructParser.golden | 10 ++ flags.go | 75 +++------ models.go | 2 +- options.go | 267 +++++++++++++++++++++++++++++++ parser_test.go | 85 ++++++---- pointer.go | 27 ++-- profile.go | 10 +- protocol.go | 18 ++- string.go | 108 +++++-------- struct.go | 33 ++++ timestamp.go | 54 ++++--- union.go | 107 ++++++------- utils.go | 17 +- value.go | 7 +- varint.go | 31 +++- 22 files changed, 727 insertions(+), 386 deletions(-) create mode 100644 errors.go create mode 100644 options.go diff --git a/Makefile b/Makefile index 87a3ca8..c498f63 100644 --- a/Makefile +++ b/Makefile @@ -1,2 +1,5 @@ all: go test -v ./... + +test: + go test -v ./... diff --git a/array.go b/array.go index a211d44..799aa74 100644 --- a/array.go +++ b/array.go @@ -3,7 +3,6 @@ package vtypes import ( "context" "encoding/json" - "errors" "fmt" "io" @@ -11,17 +10,13 @@ import ( "www.velocidex.com/golang/vfilter" ) -var ( - NotFoundError = errors.New("NotFoundError") -) - type ArrayParserOptions struct { - Type string - TypeOptions *ordereddict.Dict - Count int64 - MaxCount int64 + Type string `vfilter:"required,field=type,doc=The underlying type of the choice"` + TypeOptions *ordereddict.Dict `vfilter:"optional,field=type_options,doc=Any additional options required to parse the type"` + Count int64 `vfilter:"optional,lambda=CountExpression,field=count,doc=Number of elements in the array (default 0)"` + MaxCount int64 `vfilter:"optional,field=max_count,doc=Maximum number of elements in the array (default 1000)"` CountExpression *vfilter.Lambda - SentinelExpression *vfilter.Lambda + SentinelExpression *vfilter.Lambda `vfilter:"optional,field=sentinel,doc=A lambda expression that will be used to determine the end of the array"` } type ArrayParser struct { @@ -33,61 +28,30 @@ type ArrayParser struct { } func (self *ArrayParser) New(profile *Profile, options *ordereddict.Dict) (Parser, error) { - var pres bool - if options == nil { return nil, fmt.Errorf("Array parser requires a type in the options") } result := &ArrayParser{profile: profile} - - result.options.Type, pres = options.GetString("type") - if !pres { - return nil, errors.New("Array must specify the type in options") + ctx := context.Background() + err := ParseOptions(ctx, options, &result.options) + if err != nil { + return nil, fmt.Errorf("ArrayParser: %v", err) } - - topts, pres := options.Get("type_options") - if pres { - topts_dict, ok := topts.(*ordereddict.Dict) - if ok { - result.options.TypeOptions = topts_dict - } - } - - // Default to 0 length - result.options.Count, _ = options.GetInt64("count") - max_count_any, pres := options.Get("max_count") - if pres { - result.options.MaxCount, pres = to_int64(max_count_any) - if !pres { - return nil, fmt.Errorf("Array max_count must be an int not %T", max_count_any) - } - } - if result.options.MaxCount == 0 { result.options.MaxCount = 1000 } - // Maybe add a count expression - expression, _ := options.GetString("count") - if expression != "" { - var err error - result.options.CountExpression, err = vfilter.ParseLambda(expression) - if err != nil { - return nil, fmt.Errorf("Array parser count expression '%v': %w", - expression, err) - } + // Get the parser now so we can catch errors in sub parser + // definitions + parser, err := maybeGetParser(profile, + result.options.Type, result.options.TypeOptions) + if err != nil { + return nil, err } - expression, _ = options.GetString("sentinel") - if expression != "" { - var err error - result.options.SentinelExpression, err = vfilter.ParseLambda(expression) - if err != nil { - return nil, fmt.Errorf("Array parser sentinel expression '%v': %w", - expression, err) - } - } + // Cache the parser for next time. + result.parser = parser return result, nil } @@ -142,8 +106,11 @@ func (self *ArrayParser) Parse( // Check for a sentinel value if self.options.SentinelExpression != nil { ctx := context.Background() + subscope := scope.Copy() sentinel := self.options.SentinelExpression.Reduce( - ctx, scope, []vfilter.Any{element}) + ctx, subscope, []vfilter.Any{element}) + subscope.Close() + if scope.Bool(sentinel) { break } diff --git a/bitfield.go b/bitfield.go index 1619af1..8a29d8b 100644 --- a/bitfield.go +++ b/bitfield.go @@ -1,6 +1,7 @@ package vtypes import ( + "context" "fmt" "io" @@ -8,53 +9,69 @@ import ( "www.velocidex.com/golang/vfilter" ) -type BitField struct { - StartBit int64 `json:"start_bit"` - EndBit int64 `json:"end_bit"` - Type string `json:"type"` +type BitFieldOptions struct { + StartBit int64 `json:"start_bit" vfilter:"optional,field=start_bit,doc=The start bit in the int to read"` + EndBit int64 `json:"end_bit" vfilter:"optional,field=end_bit,doc=The end bit in the int to read"` + Type string `json:"type" vfilter:"required,field=type,doc=The underlying type of the bit field"` +} + +type BitFieldParser struct { + options BitFieldOptions parser Parser } -func (self *BitField) New(profile *Profile, options *ordereddict.Dict) (Parser, error) { - parser_type, pres := options.GetString("type") - if !pres { - return nil, fmt.Errorf("BitField parser requires a type in the options") +func (self *BitFieldParser) New(profile *Profile, options *ordereddict.Dict) (Parser, error) { + if options == nil { + return nil, fmt.Errorf("BitField parser requires an options dict") } - parser, err := profile.GetParser(parser_type, ordereddict.NewDict()) + result := &BitFieldParser{} + ctx := context.Background() + err := ParseOptions(ctx, options, &result.options) if err != nil { - return nil, fmt.Errorf("BitField parser requires a type in the options: %w", err) + return nil, err + } + + if result.options.EndBit == 0 { + result.options.EndBit = 64 } - start_bit, pres := options.GetInt64("start_bit") - if !pres || start_bit < 0 { - start_bit = 0 + if result.options.StartBit < 0 || result.options.StartBit > 64 { + return nil, fmt.Errorf("BitField start_bit should be between 0-64") } - end_bit, pres := options.GetInt64("end_bit") - if !pres || end_bit > 64 { - end_bit = 64 + if result.options.EndBit < 0 || result.options.EndBit > 64 { + return nil, fmt.Errorf("BitField end_bit should be between 0-64") } - return &BitField{ - StartBit: start_bit, - EndBit: end_bit, - parser: parser, - }, nil + if result.options.EndBit <= result.options.StartBit { + return nil, fmt.Errorf( + "BitField end_bit (%v) should be larger than start_bit (%v)", + result.options.EndBit, result.options.StartBit) + } + + // Type must be available at definition time because bit fields + // can not operate on custome types. + result.parser, err = profile.GetParser(result.options.Type, nil) + return result, err } -func (self *BitField) Parse( +func (self *BitFieldParser) Parse( scope vfilter.Scope, reader io.ReaderAt, offset int64) interface{} { + if self.parser == nil { + return vfilter.Null{} + } + result := int64(0) value, ok := to_int64(self.parser.Parse(scope, reader, offset)) if !ok { return 0 } - for i := self.StartBit; i < self.EndBit; i++ { + for i := self.options.StartBit; i < self.options.EndBit; i++ { result |= value & (1 << uint8(i)) } - return result >> self.StartBit + return result >> self.options.StartBit } diff --git a/enumeration.go b/enumeration.go index 6489488..2186066 100644 --- a/enumeration.go +++ b/enumeration.go @@ -1,6 +1,7 @@ package vtypes import ( + "context" "fmt" "io" "strconv" @@ -10,52 +11,39 @@ import ( ) type EnumerationParserOptions struct { - Type string - TypeOptions *ordereddict.Dict - Choices map[int64]string + Type string `vfilter:"required,field=type,doc=The underlying type of the choice"` + TypeOptions *ordereddict.Dict `vfilter:"optional,field=type_options,doc=Any additional options required to parse the type"` + Choices *ordereddict.Dict `vfilter:"optional,field=choices,doc=A mapping between numbers and strings."` + Map *ordereddict.Dict `vfilter:"optional,field=map,doc=A mapping between strings and numbers."` + + choices map[int64]string } type EnumerationParser struct { options EnumerationParserOptions profile *Profile parser Parser + + invalid_parser bool } func (self *EnumerationParser) New(profile *Profile, options *ordereddict.Dict) (Parser, error) { - var pres bool - if options == nil { return nil, fmt.Errorf("Enumeration parser requires an options dict") } result := &EnumerationParser{profile: profile} - result.options.Type, pres = options.GetString("type") - if !pres { - return nil, fmt.Errorf("Enumeration parser requires a type in the options") + ctx := context.Background() + err := ParseOptions(ctx, options, &result.options) + if err != nil { + return nil, err } - topts, pres := options.Get("type_options") - if pres { - topts_dict, ok := topts.(*ordereddict.Dict) - if !ok { - return nil, fmt.Errorf("Enumeration parser options should be a dict") - } - result.options.TypeOptions = topts_dict - } + result.options.choices = make(map[int64]string) - mapping := make(map[int64]string) - - // Support 2 ways of providing the mapping - choices has ints - // as keys and map has strings as keys. - choices, pres := options.Get("choices") - if pres { - choices_dict, ok := choices.(*ordereddict.Dict) - if !ok { - return nil, fmt.Errorf("Enumeration parser requires choices to be a mapping between numbers and strings") - } - - for _, k := range choices_dict.Keys() { - v, _ := choices_dict.Get(k) + if result.options.Choices != nil { + for _, k := range result.options.Choices.Keys() { + v, _ := result.options.Choices.Get(k) i, err := strconv.ParseInt(k, 0, 64) if err != nil { return nil, fmt.Errorf("Enumeration parser requires choices to be a mapping between numbers and strings (not %v)", k) @@ -66,28 +54,32 @@ func (self *EnumerationParser) New(profile *Profile, options *ordereddict.Dict) return nil, fmt.Errorf("Enumeration parser requires choices to be a mapping between numbers and strings") } - mapping[i] = v_str + result.options.choices[i] = v_str } } - choices, pres = options.Get("map") - if pres { - choices_dict, ok := choices.(*ordereddict.Dict) - if !ok { - return nil, fmt.Errorf("Enumeration parser requires map to be a mapping between strings and numbers") - } - for _, k := range choices_dict.Keys() { - v, _ := choices_dict.Get(k) + if result.options.Map != nil { + for _, k := range result.options.Map.Keys() { + v, _ := result.options.Map.Get(k) v_int, ok := to_int64(v) if !ok { return nil, fmt.Errorf("Enumeration parser requires map to be a mapping between strings and numbers") } - mapping[v_int] = k + result.options.choices[v_int] = k } } - result.options.Choices = mapping + // Get the parser now so we can catch errors in sub parser + // definitions + parser, err := maybeGetParser(profile, + result.options.Type, result.options.TypeOptions) + if err != nil { + return nil, err + } + + // Cache the parser for next time. + result.parser = parser return result, nil } @@ -95,12 +87,16 @@ func (self *EnumerationParser) New(profile *Profile, options *ordereddict.Dict) func (self *EnumerationParser) Parse( scope vfilter.Scope, reader io.ReaderAt, offset int64) interface{} { + if self.invalid_parser { + return vfilter.Null{} + } + if self.parser == nil { parser, err := self.profile.GetParser( self.options.Type, self.options.TypeOptions) if err != nil { - scope.Log("ERROR:binary_parser: Enumeration: %v", err) - self.parser = NullParser{} + scope.Log("ERROR:binary_parser: EnumerationParser: %v", err) + self.invalid_parser = true return vfilter.Null{} } @@ -113,7 +109,7 @@ func (self *EnumerationParser) Parse( return vfilter.Null{} } - string_value, pres := self.options.Choices[value] + string_value, pres := self.options.choices[value] if !pres { string_value = fmt.Sprintf("%#x", value) } diff --git a/errors.go b/errors.go new file mode 100644 index 0000000..c55af06 --- /dev/null +++ b/errors.go @@ -0,0 +1,7 @@ +package vtypes + +import "errors" + +var ( + NotFoundError = errors.New("NotFoundError") +) diff --git a/fixtures/TestErrors.golden b/fixtures/TestErrors.golden index ff00a3b..1f49ab7 100644 --- a/fixtures/TestErrors.golden +++ b/fixtures/TestErrors.golden @@ -1,6 +1,21 @@ -{ - "Flags": null, - "Enumeration": null, + + +Test Case 0: +struct TestStruct field 'Flags': Field type is required in *vtypes.BitFieldOptions +Parsing: { + "Flags": null +} +Logging: + +Test Case 1: +struct TestStruct field 'Enumeration': Unexpected parameters provided: [bitmap] +Parsing: { + "Enumeration": null +} +Logging: + +Test Case 2: No Error +Parsing: { "Undefined": { "ArrayOfUnderfinedStruct": null }, @@ -8,7 +23,4 @@ "ArrayOfUnderfinedStruct": null } } - ERROR:binary_parser: Flags: BitField parser requires a type in the options - ERROR:binary_parser: Enumeration: BitField parser requires a type in the options - ERROR:binary_parser: ArrayParser: Parser Undefined not found - +Logging: ERROR:binary_parser: ArrayParser: NotFoundError: Parser Undefined not found diff --git a/fixtures/TestFlagsParser.golden b/fixtures/TestFlagsParser.golden index 8429aff..0536ad5 100644 --- a/fixtures/TestFlagsParser.golden +++ b/fixtures/TestFlagsParser.golden @@ -4,5 +4,7 @@ "FirstBit", "FourthBit" ], - "FlagsBitfieldSecondNibble": [] + "FlagsBitfieldSecondNibble": [ + "ZeroBit" + ] } \ No newline at end of file diff --git a/fixtures/TestStructParser.golden b/fixtures/TestStructParser.golden index 14bd166..0e4ccd7 100644 --- a/fixtures/TestStructParser.golden +++ b/fixtures/TestStructParser.golden @@ -3,6 +3,16 @@ "Field2": { "SecondField1": 7 }, + "X": { + "Field1": 3, + "Field2": { + "SecondField1": 7 + }, + "Field3": 578437695752307201, + "Field4": { + "SecondField1": 6 + } + }, "Field3": 578437695752307201, "Field4": { "SecondField1": 6 diff --git a/flags.go b/flags.go index 02aee04..aadd666 100644 --- a/flags.go +++ b/flags.go @@ -1,6 +1,7 @@ package vtypes import ( + "context" "fmt" "io" "sort" @@ -11,10 +12,12 @@ import ( // Accepts option bitmap: name (string) -> bit number type FlagsOptions struct { - Type string - TypeOptions *ordereddict.Dict - Bitmap map[int64]string - Bits []int64 + Type string `vfilter:"required,field=type,doc=The underlying type of the choice"` + TypeOptions *ordereddict.Dict `vfilter:"optional,field=type_options,doc=Any additional options required to parse the type"` + Bitmap *ordereddict.Dict `vfilter:"required,field=bitmap,doc=A mapping between names and the bit number"` + + bits []int64 + bitmap map[int64]string } type Flags struct { @@ -24,52 +27,35 @@ type Flags struct { } func (self *Flags) New(profile *Profile, options *ordereddict.Dict) (Parser, error) { - var pres bool - if options == nil { return nil, fmt.Errorf("Bitmap parser requires a type in the options") } result := &Flags{profile: profile} - - result.options.Type, pres = options.GetString("type") - if !pres { - return nil, fmt.Errorf("Bitmap parser requires a type in the options") + ctx := context.Background() + err := ParseOptions(ctx, options, &result.options) + if err != nil { + return nil, fmt.Errorf("FlagsParser: %v", err) } + result.options.bitmap = make(map[int64]string) - topts, pres := options.Get("type_options") - if pres { - topts_dict, ok := topts.(*ordereddict.Dict) - if !ok { - return nil, fmt.Errorf("Enumeration parser options should be a dict") - } - result.options.TypeOptions = topts_dict - } - - bitmap, pres := options.Get("bitmap") - if !pres { - bitmap = ordereddict.NewDict() - } - - bitmap_dict, ok := bitmap.(*ordereddict.Dict) - if !ok { - return nil, fmt.Errorf("Bitmap parser requires bitmap to be a mapping between names and the bit number") - } - - result.options.Bitmap = make(map[int64]string) - - for _, name := range bitmap_dict.Keys() { - idx_any, _ := bitmap_dict.Get(name) + for _, name := range result.options.Bitmap.Keys() { + idx_any, _ := result.options.Bitmap.Get(name) idx, ok := to_int64(idx_any) if !ok || idx < 0 || idx >= 64 { - return nil, fmt.Errorf("Bitmap parser requires bitmap bit number between 0 and 64") + return nil, fmt.Errorf( + "Bitmap parser requires bitmap bit number between 0 and 64") } - result.options.Bitmap[int64(1)<= 2 { + options[components[0]] = components[1] + } + } else { + options[directive] = "Y" + } + } + + return options +} + +func ParseOptions(ctx context.Context, args *ordereddict.Dict, target interface{}) error { + v := reflect.ValueOf(target) + t := v.Type() + + if t.Kind() == reflect.Ptr { + v = v.Elem() + t = v.Type() + } + + if t.Kind() != reflect.Struct { + return errors.New("Only structs can be set with ParseOptions()") + } + + args_spefied := make(map[string]bool) + for _, k := range args.Keys() { + args_spefied[k] = true + } + + for i := 0; i < v.NumField(); i++ { + // Get the field tag value + field_types_value := t.Field(i) + options := getTag(field_types_value) + if options == nil { + continue + } + + // Is the name specified in the tag? + field_name, pres := options["field"] + if !pres { + field_name = field_types_value.Name + } + + field_value := v.Field(field_types_value.Index[0]) + if !field_value.IsValid() || !field_value.CanSet() { + return fmt.Errorf("Field %s is unsettable.", field_name) + } + + _, required := options["required"] + if required { + _, pres := args.Get(field_name) + if !pres { + return fmt.Errorf("Field %v is required in %T", + field_name, target) + } + } + + field_data, pres := args.Get(field_name) + if !pres { + continue + } + delete(args_spefied, field_name) + + // Reduce if needed. + lazy_arg, ok := field_data.(types.LazyExpr) + if ok { + field_data = lazy_arg.Reduce(ctx) + } + + // Does it look like a lambda? + if isFieldLambda(field_data) { + // The field tag indicates to store the lambda in an + // alternative field. + target_field, pres := options["lambda"] + if pres { + lambda, err := vfilter.ParseLambda(field_data.(string)) + if err != nil { + return fmt.Errorf("Error parsing lambda for field %v: %v", + field_name, err) + } + + // Set the other field with the lambda + lambda_target := v.FieldByName(target_field) + if !lambda_target.IsValid() || !lambda_target.CanSet() { + return fmt.Errorf( + "field %v wants to store lambda in %v but this field does not exist", + field_name, target_field) + } + + lambda_target.Set(reflect.ValueOf(lambda)) + continue + } + } + + switch field_types_value.Type.String() { + + case "string": + str, ok := field_data.(string) + if ok { + + field_value.Set(reflect.ValueOf(str)) + } + + case "int64": + a, ok := to_int64(field_data) + if ok { + field_value.Set(reflect.ValueOf(int64(a))) + continue + } + return fmt.Errorf("field %v: Expecting an integer not %T", + field_name, field_data) + + case "uint64": + a, ok := to_int64(field_data) + if ok { + field_value.Set(reflect.ValueOf(uint64(a))) + continue + } + return fmt.Errorf("field %v: Expecting an integer not %T", + field_name, field_data) + + case "bool": + a, ok := to_int64(field_data) + if ok { + field_value.Set(reflect.ValueOf(a > 0)) + continue + } + return fmt.Errorf("field %v: Expecting a bool not %T", + field_name, field_data) + + case "*string": + str, ok := field_data.(string) + if ok { + x := reflect.New(field_value.Type().Elem()) + x.Elem().Set(reflect.ValueOf(str)) + field_value.Set(x) + continue + } + return fmt.Errorf("field %v: Expecting a string not %T", + field_name, field_data) + + case "*int64": + a, ok := to_int64(field_data) + if ok { + x := reflect.New(field_value.Type().Elem()) + x.Elem().Set(reflect.ValueOf(a)) + field_value.Set(x) + continue + } + return fmt.Errorf("field %v: Expecting a string not %T", + field_name, field_data) + + case "*ordereddict.Dict": + dict, ok := field_data.(*ordereddict.Dict) + if ok { + field_value.Set(reflect.ValueOf(dict)) + continue + } + + map_obj, ok := field_data.(map[string]interface{}) + if ok { + res := ordereddict.NewDict() + for k, v := range map_obj { + res.Set(k, v) + } + field_value.Set(reflect.ValueOf(res)) + continue + } + + return fmt.Errorf("field %v: Expecting a mapping not %T", + field_name, field_data) + + case "*vfilter.Lambda": + str, ok := field_data.(string) + if ok { + lambda, err := vfilter.ParseLambda(str) + if err != nil { + return fmt.Errorf("Error parsing lambda for field %v: %v", + field_name, err) + } + + field_value.Set(reflect.ValueOf(lambda)) + continue + } + return fmt.Errorf("field %v: Expecting a vql lambda not %T", + field_name, field_data) + + default: + fmt.Printf("Unable to handle field type %v\n", + field_types_value.Type.String()) + + } + } + + // Report any unexpected parameters + if len(args_spefied) > 0 { + var extras []string + for k := range args_spefied { + extras = append(extras, k) + } + return fmt.Errorf("Unexpected parameters provided: %v", extras) + } + + return nil +} + +var ( + lambdaRegex = regexp.MustCompile("^[a-zA-Z0-9]+ *=>") +) + +func isFieldLambda(value interface{}) bool { + str, ok := value.(string) + if !ok { + return false + } + + return lambdaRegex.MatchString(str) +} + +func maybeGetParser( + profile *Profile, + type_name string, + options *ordereddict.Dict) (Parser, error) { + // Get the parser now so we can catch errors in sub parser + // definitions + parser, err := profile.GetParser(type_name, options) + + // It is fine if the underlying type is not known yet. It may be + // defined later. + if err != nil && !errors.Is(err, NotFoundError) { + return nil, err + } + + return parser, nil +} diff --git a/parser_test.go b/parser_test.go index 5dfbe13..81db5bc 100644 --- a/parser_test.go +++ b/parser_test.go @@ -143,7 +143,8 @@ func TestArrayParserError(t *testing.T) { err := profile.ParseStructDefinitions(definition) assert.Error(t, err) - assert.Contains(t, err.Error(), "Array max_count must be an int not string") + assert.Contains(t, err.Error(), + "struct TestStruct field 'Field1': ArrayParser: field max_count: Expecting an integer not string") } func TestArrayParser(t *testing.T) { @@ -526,6 +527,7 @@ func TestFlagsParser(t *testing.T) { end_bit: 8, }, bitmap: { + "ZeroBit": 0, "FirstBit": 1, "SecondBit": 2, "ThirdBit": 3, @@ -610,51 +612,76 @@ func TestEpochTimestampParser(t *testing.T) { // return null. But if the profile definition is invalid then we need // to report it as an error to the user. func TestErrors(t *testing.T) { - profile := NewProfile() - AddModel(profile) - - scope := MakeScope() - log_buffer := &strings.Builder{} - scope.SetLogger(log.New(log_buffer, " ", 0)) - - definition := ` + definitions := []string{ + // No erorr - this is a valid definition + ` [ - ["TestSubStruct", 0, [ - ["ArrayOfUnderfinedStruct", 0, "Array", { - type: "Undefined", - count: 100, - }], - ]], ["TestStruct", 0, [ ["Flags", 0, "Flags", { type: "BitField", bitmap: { "FirstBit": 1, - }, - }], + } + }] + ]] +]`, + // bitmap is not a valid option for Enumeration + ` +[ + ["TestStruct", 0, [ ["Enumeration", 0, "Enumeration", { type: "BitField", bitmap: { - "FirstBit": 1, - }, + "FirstBit": 1, + } + }] + ]] +]`, + // No error at parse time but an error at runtime. + ` +[ + ["TestSubStruct", 0, [ + ["ArrayOfUnderfinedStruct", 0, "Array", { + type: "Undefined", + count: 100, }], + ]], + ["TestStruct", 0, [ ["Undefined", 0, "TestSubStruct"], ["Undefined2", 0, "TestSubStruct"], ]] ] -` - err := profile.ParseStructDefinitions(definition) - assert.NoError(t, err) +`} - // Parse TestStruct over the reader - reader := bytes.NewReader(sample) - obj, err := profile.Parse(scope, "TestStruct", reader, 0) - assert.NoError(t, err) + golden := "" - serialized, err := json.MarshalIndent(obj, "", " ") - assert.NoError(t, err) + for idx, definition := range definitions { + profile := NewProfile() + AddModel(profile) - golden := string(serialized) + fmt.Sprintf("\n%v\n", log_buffer.String()) + golden += fmt.Sprintf("\n\nTest Case %d: ", idx) + err := profile.ParseStructDefinitions(definition) + if err != nil { + golden += "\n" + err.Error() + } else { + golden += "No Error" + } + + // Parse TestStruct over the reader + reader := bytes.NewReader(sample) + + scope := MakeScope() + log_buffer := &strings.Builder{} + scope.SetLogger(log.New(log_buffer, " ", 0)) + + obj, err := profile.Parse(scope, "TestStruct", reader, 0) + assert.NoError(t, err) + serialized, err := json.MarshalIndent(obj, "", " ") + assert.NoError(t, err) + + golden += fmt.Sprintf("\nParsing: %v", string(serialized)) + golden += fmt.Sprintf("\nLogging: %v", log_buffer.String()) + } goldie.Assert(t, "TestErrors", []byte(golden)) } diff --git a/pointer.go b/pointer.go index 8f7aac2..a0c8f19 100644 --- a/pointer.go +++ b/pointer.go @@ -1,6 +1,7 @@ package vtypes import ( + "context" "encoding/binary" "errors" "fmt" @@ -11,8 +12,8 @@ import ( ) type PointerParserOptions struct { - Type string - TypeOptions *ordereddict.Dict + Type string `vfilter:"required,field=type,doc=The underlying type of the choice"` + TypeOptions *ordereddict.Dict `vfilter:"optional,field=type_options,doc=Any additional options required to parse the type"` } type PointerParser struct { @@ -22,26 +23,23 @@ type PointerParser struct { } func (self *PointerParser) New(profile *Profile, options *ordereddict.Dict) (Parser, error) { - var pres bool - if options == nil { return nil, fmt.Errorf("Pointer parser requires a type in the options") } result := &PointerParser{profile: profile} - - result.options.Type, pres = options.GetString("type") - if !pres { - return nil, errors.New("Pointer must specify the type in options") + ctx := context.Background() + err := ParseOptions(ctx, options, &result.options) + if err != nil { + return nil, fmt.Errorf("PointerParser: %v", err) } - topts, pres := options.Get("type_options") - if pres { - topts_dict, ok := topts.(*ordereddict.Dict) - if ok { - result.options.TypeOptions = topts_dict - } + parser, err := maybeGetParser(profile, + result.options.Type, result.options.TypeOptions) + if err != nil { + return nil, err } + result.parser = parser return result, nil } @@ -49,6 +47,7 @@ func (self *PointerParser) New(profile *Profile, options *ordereddict.Dict) (Par func (self *PointerParser) Parse( scope vfilter.Scope, reader io.ReaderAt, offset int64) interface{} { + if self.parser == nil { parser, err := self.profile.GetParser( self.options.Type, self.options.TypeOptions) diff --git a/profile.go b/profile.go index c7133d6..4568191 100644 --- a/profile.go +++ b/profile.go @@ -1,4 +1,3 @@ -// package vtypes import ( @@ -53,7 +52,8 @@ func (self *Profile) AddParser(type_name string, parser Parser) { func (self *Profile) GetParser(name string, options *ordereddict.Dict) (Parser, error) { parser, pres := self.types[name] if !pres { - return nil, fmt.Errorf("Parser %v not found", name) + return nil, fmt.Errorf("%w: Parser %v not found", + NotFoundError, name) } if options == nil { options = ordereddict.NewDict() @@ -84,6 +84,12 @@ func (self *Profile) ParseStructDefinitions(definitions string) (err error) { } for _, struct_def := range profile_definitions { + _, pres := self.types[struct_def.Name] + if pres { + return fmt.Errorf("Struct definition for %v masks an existing definition", + struct_def.Name) + } + struct_parser := NewStructParser(struct_def.Name, struct_def.Size) self.types[struct_def.Name] = struct_parser diff --git a/protocol.go b/protocol.go index 0f80903..89f0f59 100644 --- a/protocol.go +++ b/protocol.go @@ -32,21 +32,29 @@ func (self StructAssociative) Associative(scope vfilter.Scope, return vfilter.Null{}, false } + // A Struct definition overrides default fields - this way a + // struct may define a field called "Offset" and it will be + // honored but if not defined we retur the default offset. + if lhs.HasField(rhs) { + return lhs.Get(rhs) + } + switch rhs { - case "SizeOf": + case "SizeOf", "Size": return lhs.Size(), true - case "StartOf": + case "StartOf", "Start", "OffsetOf": return lhs.Start(), true - case "ParentOf": + case "ParentOf", "Parent": return lhs.Parent(), true - case "EndOf": + case "EndOf", "End": return lhs.End(), true default: - return lhs.Get(rhs) + // scope.Log("No field %v defined on struct %v", b, lhs.TypeName()) + return nil, false } } diff --git a/string.go b/string.go index d564b58..c89ef5b 100644 --- a/string.go +++ b/string.go @@ -2,6 +2,7 @@ package vtypes import ( "bytes" + "context" "encoding/binary" "encoding/hex" "fmt" @@ -12,13 +13,19 @@ import ( "www.velocidex.com/golang/vfilter" ) +var ( + defaultTerm = "\x00" +) + type StringParserOptions struct { - Length int64 + Length *int64 `vfilter:"optional,lambda=LengthExpression,field=length,doc=Length of the string to read in bytes (Can be a lambda)"` LengthExpression *vfilter.Lambda - MaxLength int64 - Term string - TermExpression *vfilter.Lambda - Encoding string + MaxLength int64 `vfilter:"optional,field=max_length,doc=Maximum length that is enforced on the string size"` + Term *string `vfilter:"optional,lambda=TermExpression,field=term,doc=Terminating string (can be an expression)"` + TermHex *string `vfilter:"optional,field=term_hex,doc=A Terminator in hex encoding"` + TermExpression *vfilter.Lambda `vfilter:"optional,field=term_exp,doc=A Terminator expression"` + Encoding string `vfilter:"optional,field=encoding,doc=The encoding to use, can be utf8 or utf16"` + Bytes bool `vfilter:"optional,field=byte_string,doc=Terminating string (can be an expression)"` } type StringParser struct { @@ -26,70 +33,35 @@ type StringParser struct { } func (self *StringParser) New(profile *Profile, options *ordereddict.Dict) (Parser, error) { - var pres bool result := &StringParser{} - - if options == nil { - options = ordereddict.NewDict() + ctx := context.Background() + err := ParseOptions(ctx, options, &result.options) + if err != nil { + return nil, fmt.Errorf("StringParser: %v", err) } - // Some defaults - result.options.Length = -1 // -1 means this is not set - result.options.MaxLength = 1024 - - result.options.Encoding, _ = options.GetString("encoding") - result.options.Term, pres = options.GetString("term") - if !pres { - result.options.Term = "\x00" + if result.options.MaxLength == 0 { + result.options.MaxLength = 1024 } - termhex, pres := options.GetString("term_hex") - if pres { - term, err := hex.DecodeString(termhex) + if result.options.TermHex != nil { + term, err := hex.DecodeString(*result.options.TermHex) if err != nil { return nil, err } - result.options.Term = string(term) - } - - // Add a termexpression if exist - termexpression, _ := options.GetString("term_exp") - if termexpression != "" { - var err error - result.options.TermExpression, err = vfilter.ParseLambda(termexpression) - if err != nil { - return nil, fmt.Errorf("String parser term expression '%v': %w", - termexpression, err) - } - } - - // Default to 0 length - length, pres := options.GetInt64("length") - if pres { - result.options.Length = length - } - - max_length, pres := options.GetInt64("max_length") - if pres { - result.options.MaxLength = max_length - } - - // Maybe add a length expression if length is a string. - expression, _ := options.GetString("length") - if expression != "" { - var err error - result.options.LengthExpression, err = vfilter.ParseLambda(expression) - if err != nil { - return nil, fmt.Errorf("String parser length expression '%v': %w", - expression, err) - } + term_str := string(term) + result.options.Term = &term_str } return result, nil } func (self *StringParser) getCount(scope vfilter.Scope) int64 { - result := self.options.Length + var result int64 = 1024 + + if self.options.Length != nil { + result = *self.options.Length + } if self.options.LengthExpression != nil { // Evaluate the offset expression with the current scope. @@ -108,10 +80,6 @@ func (self *StringParser) Parse( reader io.ReaderAt, offset int64) interface{} { result_len := self.getCount(scope) - if result_len < 0 { - // length is not specified - max read 1kb. - result_len = 1024 - } buf := make([]byte, result_len) @@ -133,18 +101,30 @@ func (self *StringParser) Parse( result = []byte(string(utf16.Decode(u16s))) } - // if lamda term_exp configured evaluate and add as a standard term + // If a terminator is specified read up to that. + term := defaultTerm + + // if lamda term_exp configured evaluate and add as a standard + // term if self.options.TermExpression != nil { - self.options.Term = EvalLambdaAsString(self.options.TermExpression, scope) + term = EvalLambdaAsString( + self.options.TermExpression, scope) } - // If a terminator is specified read up to that. - if self.options.Term != "" { - idx := bytes.Index(result, []byte(self.options.Term)) + if self.options.Term != nil { + term = *self.options.Term + } + + if term != "" { + idx := bytes.Index(result, []byte(term)) if idx >= 0 { result = result[:idx] } } + if self.options.Bytes { + return result + } + return string(result) } diff --git a/struct.go b/struct.go index 53f86e9..feeeb49 100644 --- a/struct.go +++ b/struct.go @@ -24,6 +24,11 @@ func (self *StructParser) New(profile *Profile, options *ordereddict.Dict) (Pars return self, nil } +func (self *StructParser) HasField(name string) bool { + _, pres := self.fields[name] + return pres +} + func (self *StructParser) Size() int { return self.size } @@ -124,6 +129,14 @@ func (self *StructObject) Start() int64 { return self.offset } +func (self *StructObject) HasField(name string) bool { + return self.parser.HasField(name) +} + +func (self *StructObject) TypeName() string { + return self.parser.type_name +} + func (self *StructObject) End() int64 { return self.offset + int64(self.Size()) } @@ -187,3 +200,23 @@ func (self *StructObject) MarshalJSON() ([]byte, error) { res, err := result.MarshalJSON() return res, err } + +func getThis(scope vfilter.Scope) (interface{}, bool) { + this_obj, pres := scope.Resolve("this") + if !pres { + return nil, false + } + + this_struct, ok := this_obj.(*StructObject) + if ok { + return &StructObject{ + parser: this_struct.parser, + reader: this_struct.reader, + offset: this_struct.offset, + scope: scope, + cache: this_struct.cache, + parent: this_struct.parent, + }, true + } + return this_obj, true +} diff --git a/timestamp.go b/timestamp.go index 52a7efb..5ca38ce 100644 --- a/timestamp.go +++ b/timestamp.go @@ -1,6 +1,7 @@ package vtypes import ( + "context" "fmt" "io" "time" @@ -10,9 +11,9 @@ import ( ) type EpochTimestampOptions struct { - Type string - TypeOptions *ordereddict.Dict - Factor int64 + Type string `vfilter:"optional,field=type,doc=The underlying type of the choice"` + TypeOptions *ordereddict.Dict `vfilter:"optional,field=type_options,doc=Any additional options required to parse the type"` + Factor int64 `vfilter:"optional,field=factor,doc=A factor to be applied prior to parsing"` } type EpochTimestamp struct { @@ -22,35 +23,37 @@ type EpochTimestamp struct { } func (self *EpochTimestamp) New(profile *Profile, options *ordereddict.Dict) (Parser, error) { - var pres bool result := &EpochTimestamp{profile: profile} + ctx := context.Background() + err := ParseOptions(ctx, options, &result.options) + if err != nil { + return nil, fmt.Errorf("EpochTimestamp: %w", err) + } - result.options.Type, pres = options.GetString("type") - if !pres { + if result.options.Type == "" { result.options.Type = "uint64" } - topts, pres := options.Get("type_options") - if !pres { - result.options.TypeOptions = ordereddict.NewDict() - - } else { - - topts_dict, ok := topts.(*ordereddict.Dict) - if !ok { - return nil, fmt.Errorf("Timestamp parser options should be a dict") - } - result.options.TypeOptions = topts_dict + if result.options.Factor == 0 { + result.options.Factor = 1 } - result.options.Factor, pres = options.GetInt64("factor") - if !pres { - result.options.Factor = 1 + parser, err := maybeGetParser(profile, + result.options.Type, result.options.TypeOptions) + if err != nil { + return nil, err } + // Cache the parser for next time. + result.parser = parser + return result, nil } +func (self *EpochTimestamp) Size() int { + return SizeOf(self.parser) +} + func (self *EpochTimestamp) Parse( scope vfilter.Scope, reader io.ReaderAt, offset int64) interface{} { @@ -71,7 +74,16 @@ func (self *EpochTimestamp) Parse( if !ok { return vfilter.Null{} } - return time.Unix(value/self.options.Factor, value%self.options.Factor).UTC() + + res := time.Unix(value/self.options.Factor, + value%self.options.Factor).UTC() + + // Catch invalid timestamps. + _, err := res.MarshalJSON() + if err != nil { + return vfilter.Null{} + } + return res } type WinFileTime struct { diff --git a/union.go b/union.go index 93d3771..5a83582 100644 --- a/union.go +++ b/union.go @@ -9,11 +9,15 @@ import ( "www.velocidex.com/golang/vfilter" ) +type UnionOptions struct { + Selector *vfilter.Lambda `vfilter:"required,field=selector,doc=A lambda selector"` + Choices *ordereddict.Dict `vfilter:"required,field=choices,doc=A between values and strings"` + choices map[string]Parser +} + type Union struct { - Selector *vfilter.Lambda - choice_names *ordereddict.Dict - Choices map[string]Parser - profile *Profile + options UnionOptions + profile *Profile } func (self *Union) New(profile *Profile, options *ordereddict.Dict) (Parser, error) { @@ -21,38 +25,13 @@ func (self *Union) New(profile *Profile, options *ordereddict.Dict) (Parser, err return nil, fmt.Errorf("Union parser requires options") } - expression, pres := options.GetString("selector") - if !pres { - return nil, fmt.Errorf("Union parser requires a lambda selector") - } - - selector, err := vfilter.ParseLambda(expression) + result := &Union{profile: profile} + ctx := context.Background() + err := ParseOptions(ctx, options, &result.options) if err != nil { - return nil, fmt.Errorf("Union parser selector expression '%v': %w", - expression, err) + return nil, fmt.Errorf("Union: %w", err) } - choices, pres := options.Get("choices") - if !pres { - choices = ordereddict.NewDict() - } - - choices_dict, ok := choices.(*ordereddict.Dict) - if !ok { - return nil, fmt.Errorf("Union parser requires choices to be a mapping between values and strings") - } - - result := &Union{ - Selector: selector, - - // Map the value to the name of the type - choice_names: choices_dict, - - // Map the value to the actual parser - Choices: make(map[string]Parser), - - profile: profile, - } return result, nil } @@ -60,44 +39,52 @@ func (self *Union) Parse( scope vfilter.Scope, reader io.ReaderAt, offset int64) interface{} { var value interface{} - if self.Selector != nil { - this_obj, pres := scope.Resolve("this") - if pres { - value = self.Selector.Reduce( - context.Background(), scope, []vfilter.Any{this_obj}) + + // Initialize the choices late to ensure they are all defined by + // now. + if self.options.choices == nil { + self.options.choices = make(map[string]Parser) + + for _, k := range self.options.Choices.Keys() { + parser_name, pres := self.options.Choices.GetString(k) + if !pres { + continue + } + + parser, err := self.profile.GetParser( + parser_name, ordereddict.NewDict()) + if err != nil { + scope.Log("ERROR:binary_parser: Union: %v", err) + } else { + self.options.choices[k] = parser + } } } + + subscope := scope.Copy() + defer subscope.Close() + + this_obj, pres := getThis(subscope) + if pres { + value = self.options.Selector.Reduce( + context.Background(), subscope, []vfilter.Any{this_obj}) + } + if IsNil(value) { return &vfilter.Null{} } value_str := fmt.Sprintf("%v", value) - parser, pres := self.Choices[value_str] + parser, pres := self.options.choices[value_str] if pres { return parser.Parse(scope, reader, offset) } - // Resolve the parser from the profile now. - parser_name, pres := self.choice_names.GetString(value_str) - if !pres { - // Try the default - parser_name, pres = self.choice_names.GetString("default") - if !pres { - // Can not find the type - return null - return vfilter.Null{} - } - parser_name = "default" - } - - // Resolve the parser from the profile - parser, err := self.profile.GetParser(parser_name, ordereddict.NewDict()) - if err != nil { - scope.Log("ERROR:binary_parser: Union: %v", err) - return vfilter.Null{} + // Try the default + parser, pres = self.options.choices["default"] + if pres { + return parser.Parse(scope, reader, offset) } - if value_str != "default" { - self.Choices[value_str] = parser - } - return parser.Parse(scope, reader, offset) + return &vfilter.Null{} } diff --git a/utils.go b/utils.go index f6cbfef..c6b0536 100644 --- a/utils.go +++ b/utils.go @@ -118,23 +118,32 @@ func IsNil(v interface{}) bool { } func EvalLambdaAsInt64(expression *vfilter.Lambda, scope vfilter.Scope) int64 { - this_obj, pres := scope.Resolve("this") + subscope := scope.Copy() + defer subscope.Close() + + this_obj, pres := getThis(subscope) if !pres { return 0 } - result := expression.Reduce(context.Background(), scope, []vfilter.Any{this_obj}) + result := expression.Reduce(context.Background(), + subscope, []vfilter.Any{this_obj}) + result_int, _ := to_int64(result) return result_int } func EvalLambdaAsString(expression *vfilter.Lambda, scope vfilter.Scope) string { - this_obj, pres := scope.Resolve("this") + subscope := scope.Copy() + defer subscope.Close() + + this_obj, pres := getThis(subscope) if !pres { return "" } - result := expression.Reduce(context.Background(), scope, []vfilter.Any{this_obj}) + result := expression.Reduce(context.Background(), + subscope, []vfilter.Any{this_obj}) result_int, _ := result.(string) return result_int } diff --git a/value.go b/value.go index 5dd3b9d..90b6689 100644 --- a/value.go +++ b/value.go @@ -47,10 +47,13 @@ func (self *ValueParser) New(profile *Profile, options *ordereddict.Dict) (Parse func (self *ValueParser) Parse(scope vfilter.Scope, reader io.ReaderAt, offset int64) interface{} { if self.expression != nil { - this_obj, pres := scope.Resolve("this") + subscope := scope.Copy() + defer subscope.Close() + + this_obj, pres := getThis(subscope) if pres { return self.expression.Reduce( - context.Background(), scope, []vfilter.Any{this_obj}) + context.Background(), subscope, []vfilter.Any{this_obj}) } } if IsNil(self.value) { diff --git a/varint.go b/varint.go index 9210f2d..7cfaea7 100644 --- a/varint.go +++ b/varint.go @@ -11,18 +11,35 @@ import ( ) type VarInt struct { - base uint64 - size int + base uint64 + size int + offset int64 } func (self VarInt) Size() int { return self.size } +func (self VarInt) SizeOf() int { + return self.size +} + +func (self VarInt) EndOf() uint64 { + return uint64(self.offset) + uint64(self.size) +} + +func (self VarInt) OffsetOf() uint64 { + return uint64(self.offset) +} + func (self VarInt) Value() interface{} { return self.base } +func (self VarInt) ValueOf() interface{} { + return self.Value() +} + func (self VarInt) MarshalJSON() ([]byte, error) { return json.Marshal(self.base) } @@ -65,15 +82,17 @@ func (self *Leb128Parser) Parse(scope vfilter.Scope, reader io.ReaderAt, offset res |= value << (i * 7) if next == 0 { return VarInt{ - base: res, - size: i + 1, + offset: offset, + base: res, + size: i + 1, } } } return VarInt{ - base: res, - size: len(buf), + base: res, + offset: offset, + size: len(buf), } }