Better globbing support

Author: noisesfromspace

README.md

@@ -50,55 +50,10 @@ cat source.xml | ./unaware -format xml -method deterministic > masked.xml
 
 ### Filtering
 
-You can control which fields are masked using the `-include` and `-exclude` flags, which both accept glob patterns (e.g., `user.*`, `session.ip_*`).
+You can control which fields are masked using the `-include` and `-exclude` flags, which both accept glob patterns (e.g., `user.*`, `session.ip_*`, `**.email`, `user.*.id`). These patterns allow for flexible matching of field names and nested paths.
 
 - **Default Behavior:** If no flags are used, all fields are masked.
 - **Using `-include`:** Specifies which fields *should* be masked. When `-include` patterns are used, only fields matching them will be considered for masking.
 - **Using `-exclude`:** Specifies fields that *should not* be masked, creating exceptions.
 - **Combining Flags:** When used together, `-exclude` always takes precedence. A field is only masked if it matches an `-include` pattern but does *not* match an `-exclude` pattern. If only `-exclude` is used, all fields are masked *except* for those that match an exclusion pattern.
 
-For example, given `data.json`:
-```json
-{
-  "user": {
-    "id": "aa1",
-    "personal_info": {
-      "subscriber": "uuid-123",
-      "name": "Jane Doe",
-      "email": "jane.doe@example.com"
-    }
-  },
-  "session": {
-    "ip_address": "198.51.100.22",
-    "timestamp": "2025-11-25T10:00:00Z"
-  }
-}
-```
-
-**Goal:** Mask all sensitive user details and the session IP address, but leave the subscriber field untouched for reference.
-
-**Command:**
-```shell
-./unaware -in data.json -include 'user.personal_info.*' -include 'session.ip_address' -exclude 'user.personal_info.subscriber'
-```
-
-**Explanation:**
-The command first designates all fields under `user.personal_info` and `session.ip_address` for masking with the `-include` flags. Then, the `-exclude` flag creates an exception for `user.personal_info.subscriber`, preventing it from being masked even though it was matched by the include pattern.
-
-**Result:**
-```json
-{
-  "user": {
-    "id": "aa1",
-    "personal_info": {
-      "subscriber": "uuid-123",
-      "name": "Burger Iron",
-      "email": "kees@friet.nl"
-    }
-  },
-  "session": {
-    "ip_address": "238.108.102.226",
-    "timestamp": "2025-11-25T10:00:00Z"
-  }
-}
-```

go.mod

@@ -19,6 +19,7 @@ require (
 	github.com/cespare/xxhash/v2 v2.1.1 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/gobwas/glob v0.2.3 // indirect
 	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect

go.sum

@@ -15,6 +15,8 @@ github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WA
 github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
 github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
+github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
+github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=

main.go

@@ -33,8 +33,9 @@ func main() {
 		fmt.Fprintf(out, "  unaware -format json -in input.json -out masked.json\n\n")
 		fmt.Fprintf(out, "  # Mask a CSV file, keeping the output consistent between runs\n")
 		fmt.Fprintf(out, "  STATIC_SALT=secret-key unaware -format csv -method deterministic -in data.csv > data_masked.csv\n\n")
-		fmt.Fprintf(out, "  # Mask only email fields in a large JSON file\n")
-		fmt.Fprintf(out, "  cat users.json | unaware -format json -include \"*.email\" > masked.json\n\n")
+		fmt.Fprintf(out, "  # Mask only email fields (using a glob pattern) in a large JSON file\n")
+		fmt.Fprintf(out, "  # Use \"**\" to match across multiple nested levels (e.g., \"**.email\")\n")
+		fmt.Fprintf(out, "  cat users.json | unaware -format json -include \"**.email\" > masked.json\n\n")
 		fmt.Fprintf(out, "FLAGS:\n")
 		flag.PrintDefaults()
 	}

pkg/engine.go

@@ -9,32 +9,34 @@ import (
 	"io"
 	"net"
 	"net/url"
-	"path/filepath"
 	"regexp"
 	"strconv"
 	"strings"
 	"time"
 
-	"github.com/araddon/dateparse"
 	"github.com/brianvoe/gofakeit/v6"
 	"github.com/dgraph-io/ristretto"
+	"github.com/gobwas/glob"
 	"github.com/google/uuid"
+	"github.com/jacoelho/banking/iban"
 	"github.com/nyaruka/phonenumbers"
 	"github.com/theplant/luhn"
 	"golang.org/x/text/cases"
 	"golang.org/x/text/language"
 
-	"github.com/jacoelho/banking/iban"
+	"github.com/araddon/dateparse"
 )
 
 // AppConfig holds the complete configuration for a masking operation.
 type AppConfig struct {
-	Format   string
-	CPUCount int
-	Include  []string
-	Exclude  []string
-	FirstN   int
-	Masker   MaskerConfig
+	Format       string   `json:"format"`
+	CPUCount     int      `json:"cpu_count"`
+	Include      []string `json:"include"`
+	Exclude      []string `json:"exclude"`
+	FirstN       int      `json:"first_n"`
+	Masker       MaskerConfig
+	IncludeGlobs []glob.Glob `json:"-"`
+	ExcludeGlobs []glob.Glob `json:"-"`
 }
 
 type processor interface {
@@ -57,6 +59,23 @@ type MaskerConfig struct {
 
 // Start initiates the masking process based on the provided configuration.
 func Start(r io.Reader, w io.Writer, config AppConfig) error {
+	// Pre-compile glob patterns once at startup for performance during masking.
+	// This avoids re-parsing the patterns for every key in the input data.
+	for _, pattern := range config.Include {
+		g, err := glob.Compile(pattern, '.')
+		if err != nil {
+			return fmt.Errorf("invalid include pattern %q: %w", pattern, err)
+		}
+		config.IncludeGlobs = append(config.IncludeGlobs, g)
+	}
+	for _, pattern := range config.Exclude {
+		g, err := glob.Compile(pattern, '.')
+		if err != nil {
+			return fmt.Errorf("invalid exclude pattern %q: %w", pattern, err)
+		}
+		config.ExcludeGlobs = append(config.ExcludeGlobs, g)
+	}
+
 	var p processor
 	switch config.Format {
 	case "json":
@@ -74,17 +93,17 @@ func Start(r io.Reader, w io.Writer, config AppConfig) error {
 	return p.Process(r, w)
 }
 
-func shouldMask(key string, include, exclude []string) bool {
+func shouldMask(key string, include, exclude []glob.Glob) bool {
 	if len(exclude) > 0 {
-		for _, pattern := range exclude {
-			if matched, _ := filepath.Match(pattern, key); matched {
+		for _, g := range exclude {
+			if g.Match(key) {
 				return false
 			}
 		}
 	}
 	if len(include) > 0 {
-		for _, pattern := range include {
-			if matched, _ := filepath.Match(pattern, key); matched {
+		for _, g := range include {
+			if g.Match(key) {
 				return true
 			}
 		}
@@ -155,11 +174,11 @@ func newConcurrentRunner(methodFactory func() *masker, config AppConfig) *concur
 }
 
 type masker struct {
-	faker        *gofakeit.Faker
-	seeder       seeder
-	cache        *ristretto.Cache
-	dateLayouts  []string
-	emailRegex   *regexp.Regexp
+	faker           *gofakeit.Faker
+	seeder          seeder
+	cache           *ristretto.Cache
+	dateLayouts     []string
+	emailRegex      *regexp.Regexp
 	numLikeRegex    *regexp.Regexp
 	ulidRegex       *regexp.Regexp
 	ksuidRegex      *regexp.Regexp
@@ -179,8 +198,8 @@ func newMasker(config MaskerConfig) *masker {
 			"01/02/2006",
 			time.RFC1123,
 		},
-		emailRegex:   regexp.MustCompile(`^[^@\s]+@[^@\s]+\.[^@\s]+$`),
-		numLikeRegex: regexp.MustCompile(`^[\d\s-]+$`),
+		emailRegex:      regexp.MustCompile(`^[^@\s]+@[^@\s]+\.[^@\s]+$`),
+		numLikeRegex:    regexp.MustCompile(`^[\d\s-]+$`),
 		ulidRegex:       regexp.MustCompile(`(?i)^[0-7][0-9a-hjkmnp-tv-z]{25}$`),
 		ksuidRegex:      regexp.MustCompile(`^[a-zA-Z0-9]{27}$`),
 		creditCardRegex: regexp.MustCompile(`^(?:\d[ -]*?){13,16}$`),

pkg/json.go

@@ -180,7 +180,7 @@ func isWhitespace(c byte) bool { return c == ' ' || c == '\n' || c == '\r' || c
 func (jp *jsonProcessor) recursiveMask(m *masker, key string, data any) any {
 	switch v := data.(type) {
 	case json.Number:
-		if shouldMask(key, jp.config.Include, jp.config.Exclude) {
+		if shouldMask(key, jp.config.IncludeGlobs, jp.config.ExcludeGlobs) {
 			s := v.String()
 			if strings.Contains(s, ".") {
 				parts := strings.Split(s, ".")
@@ -191,7 +191,7 @@ func (jp *jsonProcessor) recursiveMask(m *masker, key string, data any) any {
 		}
 		return v
 	case string, bool, nil:
-		if shouldMask(key, jp.config.Include, jp.config.Exclude) {
+		if shouldMask(key, jp.config.IncludeGlobs, jp.config.ExcludeGlobs) {
 			return m.mask(v)
 		}
 		return v

pkg/worker.go

@@ -97,7 +97,7 @@ func (cr *concurrentRunner) worker(wg *sync.WaitGroup, jobs <-chan job, results
 func (cr *concurrentRunner) recursiveMask(m *masker, key string, data any) any {
 	switch v := data.(type) {
 	case json.Number, string, bool, nil:
-		if shouldMask(key, cr.config.Include, cr.config.Exclude) {
+		if shouldMask(key, cr.config.IncludeGlobs, cr.config.ExcludeGlobs) {
 			return m.mask(v)
 		}
 		return v
@@ -107,7 +107,7 @@ func (cr *concurrentRunner) recursiveMask(m *masker, key string, data any) any {
 			if k == "#text" {
 				// This is the text content of the parent element (e.g., the "2002" in <year>2002</year>).
 				// The key for filtering is the parent's key, which is already in the 'key' variable.
-				if shouldMask(key, cr.config.Include, cr.config.Exclude) {
+				if shouldMask(key, cr.config.IncludeGlobs, cr.config.ExcludeGlobs) {
 					maskedMap[k] = m.mask(value)
 				} else {
 					maskedMap[k] = value
@@ -131,7 +131,7 @@ func (cr *concurrentRunner) recursiveMask(m *masker, key string, data any) any {
 		}
 		return maskedSlice
 	default:
-		if shouldMask(key, cr.config.Include, cr.config.Exclude) {
+		if shouldMask(key, cr.config.IncludeGlobs, cr.config.ExcludeGlobs) {
 			return m.mask(v)
 		}
 		return v

pkg/xml.go

@@ -262,7 +262,7 @@ func (xp *xmlProcessor) processSerially(decoder *xml.Decoder, w io.Writer) error
 			for i := range startElem.Attr {
 				attr := &startElem.Attr[i]
 				fullKey := strings.Join(path, ".") + "." + attr.Name.Local
-				if shouldMask(fullKey, xp.config.Include, xp.config.Exclude) {
+				if shouldMask(fullKey, xp.config.IncludeGlobs, xp.config.ExcludeGlobs) {
 					maskedValue := serialMasker.mask(attr.Value)
 					attr.Value = fmt.Sprintf("%v", maskedValue)
 				}
@@ -274,7 +274,7 @@ func (xp *xmlProcessor) processSerially(decoder *xml.Decoder, w io.Writer) error
 			trimmedData := strings.TrimSpace(string(se))
 			if len(trimmedData) > 0 {
 				fullKey := strings.Join(path, ".")
-				if shouldMask(fullKey, xp.config.Include, xp.config.Exclude) {
+				if shouldMask(fullKey, xp.config.IncludeGlobs, xp.config.ExcludeGlobs) {
 					maskedValue := serialMasker.mask(trimmedData)
 					maskedString := fmt.Sprintf("%v", maskedValue)
 					if err := encoder.EncodeToken(xml.CharData(maskedString)); err != nil {

test/filtering_test.go

@@ -75,7 +75,7 @@ func TestFilteringScenarios(t *testing.T) {
 			name:    "JSON - Exclude Only (Blacklist)",
 			format:  "json",
 			input:   jsonInput,
-			exclude: []string{"*.id", "*.ip_address"},
+			exclude: []string{"**.id", "**.ip_address"},
 			expected: []string{
 				`"id": "user-123"`,
 				`"ip_address": "203.0.113.195"`,
@@ -84,8 +84,7 @@ func TestFilteringScenarios(t *testing.T) {
 			},
 		},
 		{
-			name:    "JSON - Include Only (Whitelist)",
-			format:  "json",
+			name: "JSON - Include Only (Whitelist)", format: "json",
 			input:   jsonInput,
 			include: []string{"user.personal.*"},
 			expected: []string{