Add CSV

Author: noisesfromspace

.github/workflows/release.yml

@@ -16,7 +16,7 @@ jobs:
         goarch: [ amd64, arm64]
     steps:
     - uses: actions/checkout@v4
-    - uses: martijnboers/go-release-action@v1
+    - uses: wangyoucao577/go-release-action@v1
       with:
         github_token: ${{ secrets.GITHUB_TOKEN }}
         goos: ${{ matrix.goos }}

README.md

@@ -1,5 +1,5 @@
 ## Description
-`unaware` is a command-line tool for masking sensitive data within XML and JSON files. It processes data from files or `stdin` and anonymizes specified property values. Masked values mimick the length and appearance of the original data types.
+`unaware` is a command-line tool for masking sensitive data within JSON, XML, and CSV files. It processes data from files or `stdin` and anonymizes specified property values. Masked values mimick the length and appearance of the original data types.
 
 The program is a cross-platform, statically linked binary with no external dependencies. It leverages streaming and concurrency to efficiently process large files entirely offline.
 
@@ -13,7 +13,7 @@ Alternatively, check the releases page for pre-built binaries.
 
 ### Usage
 ```
-Anonymize data in JSON and XML files by replacing values with realistic-looking fakes.
+Anonymize data in JSON, XML, and CSV files by replacing values with realistic-looking fakes.
 
 Use the -method hashed option to preserve relationships by ensuring identical input values get the same masked output value. By default every run uses a random salt, use STATIC_SALT=test123 environment variable for consistent masking.
 
@@ -22,7 +22,7 @@ Use the -method hashed option to preserve relationships by ensuring identical in
   -exclude value
     	Glob pattern to exclude keys from masking (can be specified multiple times)
   -format string
-    	The format of the input data (json or xml) (default "json")
+    	The format of the input data (json, xml, or csv) (default "json")
   -in string
     	Input file path (default: stdin)
   -include value

pkg/csv.go

@@ -0,0 +1,106 @@
+package pkg
+
+import (
+	"encoding/csv"
+	"fmt"
+	"io"
+	"sync"
+)
+
+// csvProcessor handles the reading, concurrent masking, and writing of CSV data.
+type csvProcessor struct {
+	methodFactory func() *masker
+	include       []string
+	exclude       []string
+}
+
+// newCSVProcessor creates a new processor for CSV files.
+func newCSVProcessor(strategy MaskingStrategy, include, exclude []string) *csvProcessor {
+	return &csvProcessor{
+		methodFactory: func() *masker {
+			return newMasker(strategy)
+		},
+		include: include,
+		exclude: exclude,
+	}
+}
+
+// Process orchestrates the reading and assembling of the CSV data.
+func (p *csvProcessor) Process(r io.Reader, w io.Writer, cpuCount int) error {
+	csvReader := csv.NewReader(r)
+
+	// Read the header to get column names.
+	header, err := csvReader.Read()
+	if err == io.EOF {
+		return nil // Handle empty file
+	}
+	if err != nil {
+		return fmt.Errorf("error reading CSV header: %w", err)
+	}
+
+	// The chunkReader function reads one row at a time and converts it into a map,
+	// which is the "chunk" our concurrent runner will process.
+	chunkReader := func() (any, error) {
+		record, err := csvReader.Read()
+		if err != nil {
+			return nil, err // Let the runner handle io.EOF
+		}
+		// Convert the row (a slice of strings) into a map using the header.
+		// This provides the necessary key (column name) for masking.
+		rowMap := make(map[string]any, len(header))
+		for i, value := range record {
+			if i < len(header) {
+				rowMap[header[i]] = value
+			}
+		}
+		return rowMap, nil
+	}
+
+	assembler := &csvAssembler{
+		header: header,
+		writer: csv.NewWriter(w),
+	}
+	runner := newConcurrentRunner(p.methodFactory, cpuCount, p.include, p.exclude)
+
+	return runner.Run(w, chunkReader, assembler)
+}
+
+// csvAssembler is responsible for writing the processed data back into a CSV format.
+type csvAssembler struct {
+	header []string
+	writer *csv.Writer
+	// A mutex is needed because multiple workers will call WriteItem concurrently.
+	mu sync.Mutex
+}
+
+func (a *csvAssembler) WriteStart(w io.Writer) error {
+	return a.writer.Write(a.header)
+}
+
+func (a *csvAssembler) WriteItem(w io.Writer, item any, isFirst bool) error {
+	// The concurrent runner might call this from multiple goroutines,
+	// so we lock to ensure writes are not interleaved.
+	a.mu.Lock()
+	defer a.mu.Unlock()
+
+	rowMap, ok := item.(map[string]any)
+	if !ok {
+		return fmt.Errorf("csv assembler expected map[string]any, but got %T", item)
+	}
+
+	// Convert the map back into a slice of strings in the correct order.
+	record := make([]string, len(a.header))
+	for i, key := range a.header {
+		if val, ok := rowMap[key]; ok {
+			record[i] = fmt.Sprintf("%v", val)
+		}
+	}
+
+	return a.writer.Write(record)
+}
+
+func (a *csvAssembler) WriteEnd(w io.Writer) error {
+	// The csv.Writer needs to be flushed to ensure all buffered data is written.
+	a.writer.Flush()
+	return a.writer.Error()
+}

pkg/engine.go

@@ -32,6 +32,8 @@ func Start(format string, cpuCount int, r io.Reader, w io.Writer, strategy Maski
 		p = newJSONProcessor(strategy, include, exclude)
 	case "xml":
 		p = newXMLProcessor(strategy, include, exclude)
+	case "csv":
+		p = newCSVProcessor(strategy, include, exclude)
 	default:
 		return fmt.Errorf("unsupported format: %s", format)
 	}

test/csv_test.go

@@ -0,0 +1,132 @@
+package test
+
+import (
+	"bytes"
+	"encoding/csv"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"unaware/pkg"
+)
+
+func TestCSVProcessing_Hashed(t *testing.T) {
+	salt := []byte("csv-salt")
+	input := `id,name,email,ip_address,notes
+1,John Doe,john.doe@example.com,192.168.1.1,some notes
+2,Jane Smith,jane.smith@example.net,10.0.0.2,more data`
+
+	var buf bytes.Buffer
+	err := pkg.Start("csv", 2, strings.NewReader(input), &buf, pkg.Hashed(salt), nil, nil)
+	require.NoError(t, err)
+
+	output := buf.String()
+	assert.NotEmpty(t, output)
+
+	// Verify header is preserved
+	assert.True(t, strings.HasPrefix(output, "id,name,email,ip_address,notes\n"))
+
+	// Verify original sensitive data is gone
+	assert.NotContains(t, output, "John Doe")
+	assert.NotContains(t, output, "john.doe@example.com")
+	assert.NotContains(t, output, "192.168.1.1")
+	assert.NotContains(t, output, "Jane Smith")
+	assert.NotContains(t, output, "jane.smith@example.net")
+	assert.NotContains(t, output, "10.0.0.2")
+
+	// Verify the structure is a valid CSV with the correct number of records
+	r := csv.NewReader(strings.NewReader(output))
+	records, err := r.ReadAll()
+	require.NoError(t, err)
+	assert.Len(t, records, 3) // Header + 2 rows
+	assert.Len(t, records[1], 5)
+	assert.Len(t, records[2], 5)
+}
+
+func TestCSVProcessing_WithInclude(t *testing.T) {
+	salt := []byte("csv-include-salt")
+	input := `id,name,email,ip_address,notes
+1,John Doe,john.doe@example.com,192.168.1.1,some notes
+2,Jane Smith,jane.smith@example.net,10.0.0.2,more data`
+
+	// Only include 'email' and 'ip_address' for masking
+	include := []string{"email", "ip_address"}
+
+	var buf bytes.Buffer
+	err := pkg.Start("csv", 2, strings.NewReader(input), &buf, pkg.Hashed(salt), include, nil)
+	require.NoError(t, err)
+
+	output := buf.String()
+
+	// Verify sensitive data that should be masked is gone
+	assert.NotContains(t, output, "john.doe@example.com")
+	assert.NotContains(t, output, "192.168.1.1")
+
+	// Verify data that should be preserved is still there
+	assert.Contains(t, output, "John Doe")
+	assert.Contains(t, output, "Jane Smith")
+	assert.Contains(t, output, "some notes")
+
+	// Verify structure and read the output to check specific fields
+	r := csv.NewReader(strings.NewReader(output))
+	records, err := r.ReadAll()
+	require.NoError(t, err)
+	require.Len(t, records, 3)
+
+	// Check first data row
+	assert.Equal(t, "1", records[1][0])
+	assert.Equal(t, "John Doe", records[1][1])
+	assert.NotEqual(t, "john.doe@example.com", records[1][2]) // Masked
+	assert.NotEqual(t, "192.168.1.1", records[1][3])          // Masked
+	assert.Equal(t, "some notes", records[1][4])
+}
+
+func TestCSVProcessing_WithExclude(t *testing.T) {
+	salt := []byte("csv-exclude-salt")
+	input := `id,name,email,ip_address,notes
+1,John Doe,john.doe@example.com,192.168.1.1,some notes`
+
+	// Exclude 'id' and 'notes' from masking
+	exclude := []string{"id", "notes"}
+
+	var buf bytes.Buffer
+	err := pkg.Start("csv", 2, strings.NewReader(input), &buf, pkg.Hashed(salt), nil, exclude)
+	require.NoError(t, err)
+
+	output := buf.String()
+
+	// Verify sensitive data that should be masked is gone
+	assert.NotContains(t, output, "John Doe")
+	assert.NotContains(t, output, "john.doe@example.com")
+
+	// Verify data that should be preserved is still there
+	assert.Contains(t, output, "1,")         // id is preserved
+	assert.Contains(t, output, ",some notes") // notes is preserved
+
+	// Verify structure
+	r := csv.NewReader(strings.NewReader(output))
+	records, err := r.ReadAll()
+	require.NoError(t, err)
+	require.Len(t, records, 2)
+	assert.Equal(t, "1", records[1][0])                      // Preserved
+	assert.NotEqual(t, "John Doe", records[1][1])            // Masked
+	assert.NotEqual(t, "john.doe@example.com", records[1][2]) // Masked
+	assert.NotEqual(t, "192.168.1.1", records[1][3])          // Masked
+	assert.Equal(t, "some notes", records[1][4])             // Preserved
+}
+
+func TestEmptyReader_CSV(t *testing.T) {
+	var buf bytes.Buffer
+	err := pkg.Start("csv", 1, strings.NewReader(""), &buf, pkg.Random(), nil, nil)
+	require.NoError(t, err)
+	assert.Equal(t, "", buf.String())
+}
+
+func TestReaderError_CSV(t *testing.T) {
+	errorReader := &errorReader{}
+	var buf bytes.Buffer
+	err := pkg.Start("csv", 1, errorReader, &buf, pkg.Random(), nil, nil)
+	require.Error(t, err)
+}

test/json_test.go

@@ -3,7 +3,6 @@ package test
 import (
 	"bytes"
 	"encoding/json"
-	"io"
 	"strings"
 	"testing"
 
@@ -197,9 +196,3 @@ func TestJSONStreamingNestedArray(t *testing.T) {
 	err = json.Unmarshal([]byte(output), &result)
 	require.NoError(t, err, "Output should be valid JSON. Got: %s", output)
 }
-
-type errorReader struct{}
-
-func (r *errorReader) Read(p []byte) (n int, err error) {
-	return 0, io.ErrUnexpectedEOF
-}

test/test_utils.go

@@ -0,0 +1,10 @@
+package test
+
+import "io"
+
+// errorReader is a helper for testing that simulates an error during reading.
+type errorReader struct{}
+
+func (r *errorReader) Read(p []byte) (n int, err error) {
+	return 0, io.ErrUnexpectedEOF
+}