Refactored parsers to detect errors.

Makefile

@@ -1,2 +1,5 @@
 all:
 	go test -v ./...
+
+test:
+	go test -v ./...

array.go

@@ -3,25 +3,20 @@ package vtypes
 import (
 	"context"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"io"
 
 	"github.com/Velocidex/ordereddict"
 	"www.velocidex.com/golang/vfilter"
 )
 
-var (
-	NotFoundError = errors.New("NotFoundError")
-)
-
 type ArrayParserOptions struct {
-	Type               string
-	TypeOptions        *ordereddict.Dict
-	Count              int64
-	MaxCount           int64
+	Type               string            `vfilter:"required,field=type,doc=The underlying type of the choice"`
+	TypeOptions        *ordereddict.Dict `vfilter:"optional,field=type_options,doc=Any additional options required to parse the type"`
+	Count              int64             `vfilter:"optional,lambda=CountExpression,field=count,doc=Number of elements in the array (default 0)"`
+	MaxCount           int64             `vfilter:"optional,field=max_count,doc=Maximum number of elements in the array (default 1000)"`
 	CountExpression    *vfilter.Lambda
-	SentinelExpression *vfilter.Lambda
+	SentinelExpression *vfilter.Lambda `vfilter:"optional,field=sentinel,doc=A lambda expression that will be used to determine the end of the array"`
 }
 
 type ArrayParser struct {
@@ -33,61 +28,30 @@ type ArrayParser struct {
 }
 
 func (self *ArrayParser) New(profile *Profile, options *ordereddict.Dict) (Parser, error) {
-	var pres bool
-
 	if options == nil {
 		return nil, fmt.Errorf("Array parser requires a type in the options")
 	}
 
 	result := &ArrayParser{profile: profile}
-
-	result.options.Type, pres = options.GetString("type")
-	if !pres {
-		return nil, errors.New("Array must specify the type in options")
+	ctx := context.Background()
+	err := ParseOptions(ctx, options, &result.options)
+	if err != nil {
+		return nil, fmt.Errorf("ArrayParser: %v", err)
 	}
-
-	topts, pres := options.Get("type_options")
-	if pres {
-		topts_dict, ok := topts.(*ordereddict.Dict)
-		if ok {
-			result.options.TypeOptions = topts_dict
-		}
-	}
-
-	// Default to 0 length
-	result.options.Count, _ = options.GetInt64("count")
-	max_count_any, pres := options.Get("max_count")
-	if pres {
-		result.options.MaxCount, pres = to_int64(max_count_any)
-		if !pres {
-			return nil, fmt.Errorf("Array max_count must be an int not %T", max_count_any)
-		}
-	}
-
 	if result.options.MaxCount == 0 {
 		result.options.MaxCount = 1000
 	}
 
-	// Maybe add a count expression
-	expression, _ := options.GetString("count")
-	if expression != "" {
-		var err error
-		result.options.CountExpression, err = vfilter.ParseLambda(expression)
-		if err != nil {
-			return nil, fmt.Errorf("Array parser count expression '%v': %w",
-				expression, err)
-		}
+	// Get the parser now so we can catch errors in sub parser
+	// definitions
+	parser, err := maybeGetParser(profile,
+		result.options.Type, result.options.TypeOptions)
+	if err != nil {
+		return nil, err
 	}
 
-	expression, _ = options.GetString("sentinel")
-	if expression != "" {
-		var err error
-		result.options.SentinelExpression, err = vfilter.ParseLambda(expression)
-		if err != nil {
-			return nil, fmt.Errorf("Array parser sentinel expression '%v': %w",
-				expression, err)
-		}
-	}
+	// Cache the parser for next time.
+	result.parser = parser
 
 	return result, nil
 }
@@ -142,8 +106,11 @@ func (self *ArrayParser) Parse(
 		// Check for a sentinel value
 		if self.options.SentinelExpression != nil {
 			ctx := context.Background()
+			subscope := scope.Copy()
 			sentinel := self.options.SentinelExpression.Reduce(
-				ctx, scope, []vfilter.Any{element})
+				ctx, subscope, []vfilter.Any{element})
+			subscope.Close()
+
 			if scope.Bool(sentinel) {
 				break
 			}

bitfield.go

@@ -1,60 +1,77 @@
 package vtypes
 
 import (
+	"context"
 	"fmt"
 	"io"
 
 	"github.com/Velocidex/ordereddict"
 	"www.velocidex.com/golang/vfilter"
 )
 
-type BitField struct {
-	StartBit int64  `json:"start_bit"`
-	EndBit   int64  `json:"end_bit"`
-	Type     string `json:"type"`
+type BitFieldOptions struct {
+	StartBit int64  `json:"start_bit" vfilter:"optional,field=start_bit,doc=The start bit in the int to read"`
+	EndBit   int64  `json:"end_bit" vfilter:"optional,field=end_bit,doc=The end bit in the int to read"`
+	Type     string `json:"type" vfilter:"required,field=type,doc=The underlying type of the bit field"`
+}
+
+type BitFieldParser struct {
+	options BitFieldOptions
 
 	parser Parser
 }
 
-func (self *BitField) New(profile *Profile, options *ordereddict.Dict) (Parser, error) {
-	parser_type, pres := options.GetString("type")
-	if !pres {
-		return nil, fmt.Errorf("BitField parser requires a type in the options")
+func (self *BitFieldParser) New(profile *Profile, options *ordereddict.Dict) (Parser, error) {
+	if options == nil {
+		return nil, fmt.Errorf("BitField parser requires an options dict")
 	}
 
-	parser, err := profile.GetParser(parser_type, ordereddict.NewDict())
+	result := &BitFieldParser{}
+	ctx := context.Background()
+	err := ParseOptions(ctx, options, &result.options)
 	if err != nil {
-		return nil, fmt.Errorf("BitField parser requires a type in the options: %w", err)
+		return nil, err
+	}
+
+	if result.options.EndBit == 0 {
+		result.options.EndBit = 64
 	}
 
-	start_bit, pres := options.GetInt64("start_bit")
-	if !pres || start_bit < 0 {
-		start_bit = 0
+	if result.options.StartBit < 0 || result.options.StartBit > 64 {
+		return nil, fmt.Errorf("BitField start_bit should be between 0-64")
 	}
 
-	end_bit, pres := options.GetInt64("end_bit")
-	if !pres || end_bit > 64 {
-		end_bit = 64
+	if result.options.EndBit < 0 || result.options.EndBit > 64 {
+		return nil, fmt.Errorf("BitField end_bit should be between 0-64")
 	}
 
-	return &BitField{
-		StartBit: start_bit,
-		EndBit:   end_bit,
-		parser:   parser,
-	}, nil
+	if result.options.EndBit <= result.options.StartBit {
+		return nil, fmt.Errorf(
+			"BitField end_bit (%v) should be larger than start_bit (%v)",
+			result.options.EndBit, result.options.StartBit)
+	}
+
+	// Type must be available at definition time because bit fields
+	// can not operate on custome types.
+	result.parser, err = profile.GetParser(result.options.Type, nil)
+	return result, err
 }
 
-func (self *BitField) Parse(
+func (self *BitFieldParser) Parse(
 	scope vfilter.Scope, reader io.ReaderAt, offset int64) interface{} {
 
+	if self.parser == nil {
+		return vfilter.Null{}
+	}
+
 	result := int64(0)
 	value, ok := to_int64(self.parser.Parse(scope, reader, offset))
 	if !ok {
 		return 0
 	}
-	for i := self.StartBit; i < self.EndBit; i++ {
+	for i := self.options.StartBit; i < self.options.EndBit; i++ {
 		result |= value & (1 << uint8(i))
 	}
 
-	return result >> self.StartBit
+	return result >> self.options.StartBit
 }

enumeration.go

@@ -1,6 +1,7 @@
 package vtypes
 
 import (
+	"context"
 	"fmt"
 	"io"
 	"strconv"
@@ -10,52 +11,39 @@ import (
 )
 
 type EnumerationParserOptions struct {
-	Type        string
-	TypeOptions *ordereddict.Dict
-	Choices     map[int64]string
+	Type        string            `vfilter:"required,field=type,doc=The underlying type of the choice"`
+	TypeOptions *ordereddict.Dict `vfilter:"optional,field=type_options,doc=Any additional options required to parse the type"`
+	Choices     *ordereddict.Dict `vfilter:"optional,field=choices,doc=A mapping between numbers and strings."`
+	Map         *ordereddict.Dict `vfilter:"optional,field=map,doc=A mapping between strings and numbers."`
+
+	choices map[int64]string
 }
 
 type EnumerationParser struct {
 	options EnumerationParserOptions
 	profile *Profile
 	parser  Parser
+
+	invalid_parser bool
 }
 
 func (self *EnumerationParser) New(profile *Profile, options *ordereddict.Dict) (Parser, error) {
-	var pres bool
-
 	if options == nil {
 		return nil, fmt.Errorf("Enumeration parser requires an options dict")
 	}
 
 	result := &EnumerationParser{profile: profile}
-	result.options.Type, pres = options.GetString("type")
-	if !pres {
-		return nil, fmt.Errorf("Enumeration parser requires a type in the options")
+	ctx := context.Background()
+	err := ParseOptions(ctx, options, &result.options)
+	if err != nil {
+		return nil, err
 	}
 
-	topts, pres := options.Get("type_options")
-	if pres {
-		topts_dict, ok := topts.(*ordereddict.Dict)
-		if !ok {
-			return nil, fmt.Errorf("Enumeration parser options should be a dict")
-		}
-		result.options.TypeOptions = topts_dict
-	}
+	result.options.choices = make(map[int64]string)
 
-	mapping := make(map[int64]string)
-
-	// Support 2 ways of providing the mapping - choices has ints
-	// as keys and map has strings as keys.
-	choices, pres := options.Get("choices")
-	if pres {
-		choices_dict, ok := choices.(*ordereddict.Dict)
-		if !ok {
-			return nil, fmt.Errorf("Enumeration parser requires choices to be a mapping between numbers and strings")
-		}
-
-		for _, k := range choices_dict.Keys() {
-			v, _ := choices_dict.Get(k)
+	if result.options.Choices != nil {
+		for _, k := range result.options.Choices.Keys() {
+			v, _ := result.options.Choices.Get(k)
 			i, err := strconv.ParseInt(k, 0, 64)
 			if err != nil {
 				return nil, fmt.Errorf("Enumeration parser requires choices to be a mapping between numbers and strings (not %v)", k)
@@ -66,41 +54,49 @@ func (self *EnumerationParser) New(profile *Profile, options *ordereddict.Dict)
 				return nil, fmt.Errorf("Enumeration parser requires choices to be a mapping between numbers and strings")
 			}
 
-			mapping[i] = v_str
+			result.options.choices[i] = v_str
 		}
 	}
 
-	choices, pres = options.Get("map")
-	if pres {
-		choices_dict, ok := choices.(*ordereddict.Dict)
-		if !ok {
-			return nil, fmt.Errorf("Enumeration parser requires map to be a mapping between strings and numbers")
-		}
-		for _, k := range choices_dict.Keys() {
-			v, _ := choices_dict.Get(k)
+	if result.options.Map != nil {
+		for _, k := range result.options.Map.Keys() {
+			v, _ := result.options.Map.Get(k)
 			v_int, ok := to_int64(v)
 			if !ok {
 				return nil, fmt.Errorf("Enumeration parser requires map to be a mapping between strings and numbers")
 			}
 
-			mapping[v_int] = k
+			result.options.choices[v_int] = k
 		}
 	}
 
-	result.options.Choices = mapping
+	// Get the parser now so we can catch errors in sub parser
+	// definitions
+	parser, err := maybeGetParser(profile,
+		result.options.Type, result.options.TypeOptions)
+	if err != nil {
+		return nil, err
+	}
+
+	// Cache the parser for next time.
+	result.parser = parser
 
 	return result, nil
 }
 
 func (self *EnumerationParser) Parse(
 	scope vfilter.Scope, reader io.ReaderAt, offset int64) interface{} {
 
+	if self.invalid_parser {
+		return vfilter.Null{}
+	}
+
 	if self.parser == nil {
 		parser, err := self.profile.GetParser(
 			self.options.Type, self.options.TypeOptions)
 		if err != nil {
-			scope.Log("ERROR:binary_parser: Enumeration: %v", err)
-			self.parser = NullParser{}
+			scope.Log("ERROR:binary_parser: EnumerationParser: %v", err)
+			self.invalid_parser = true
 			return vfilter.Null{}
 		}
 
@@ -113,7 +109,7 @@ func (self *EnumerationParser) Parse(
 		return vfilter.Null{}
 	}
 
-	string_value, pres := self.options.Choices[value]
+	string_value, pres := self.options.choices[value]
 	if !pres {
 		string_value = fmt.Sprintf("%#x", value)
 	}

errors.go

@@ -0,0 +1,7 @@
+package vtypes
+
+import "errors"
+
+var (
+	NotFoundError = errors.New("NotFoundError")
+)