diff --git a/go/appencryption/.tool-versions b/go/appencryption/.tool-versions new file mode 100644 index 000000000..1a59aeb0b --- /dev/null +++ b/go/appencryption/.tool-versions @@ -0,0 +1 @@ +golang 1.23.0 diff --git a/go/appencryption/concurrent_benchmark_test.go b/go/appencryption/concurrent_benchmark_test.go new file mode 100644 index 000000000..320abf7ba --- /dev/null +++ b/go/appencryption/concurrent_benchmark_test.go @@ -0,0 +1,254 @@ +package appencryption + +import ( + "context" + "fmt" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/godaddy/asherah/go/appencryption/internal" +) + +// Concurrent benchmarks with allocation tracking to measure performance under load +// These simulate realistic high-concurrency production scenarios + +// BenchmarkSession_Encrypt_Concurrent benchmarks concurrent encryption operations +func BenchmarkSession_Encrypt_Concurrent(b *testing.B) { + b.ReportAllocs() + + session := newBenchmarkSession(b) + defer session.Close() + + ctx := context.Background() + + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + payload := internal.GetRandBytes(benchmarkPayloadSize) + _, err := session.Encrypt(ctx, payload) + if err != nil { + b.Error(err) + } + } + }) +} + +// BenchmarkSession_Decrypt_Concurrent benchmarks concurrent decryption operations +func BenchmarkSession_Decrypt_Concurrent(b *testing.B) { + b.ReportAllocs() + + session := newBenchmarkSession(b) + defer session.Close() + + // Pre-encrypt data for concurrent decryption + ctx := context.Background() + payload := internal.GetRandBytes(benchmarkPayloadSize) + + // Create multiple copies of the encrypted data to avoid race conditions + const numCopies = 100 + drrs := make([]*DataRowRecord, numCopies) + for i := 0; i < numCopies; i++ { + drr, err := session.Encrypt(ctx, payload) + require.NoError(b, err) + drrs[i] = drr + } + + var counter int64 + + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + // Use atomic counter for round-robin to avoid race + idx := atomic.AddInt64(&counter, 1) + _, err := session.Decrypt(ctx, *drrs[idx%numCopies]) + if err != nil { + b.Error(err) + } + } + }) +} + +// BenchmarkSessionFactory_GetSession_Concurrent benchmarks concurrent session creation +func BenchmarkSessionFactory_GetSession_Concurrent(b *testing.B) { + b.ReportAllocs() + + factory := newBenchmarkSessionFactory(b) + defer factory.Close() + + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + session, err := factory.GetSession(benchmarkPartitionID) + if err != nil { + b.Error(err) + } + session.Close() + } + }) +} + +// BenchmarkKeyCache_Concurrent_SameKey benchmarks concurrent access to the same key +func BenchmarkKeyCache_Concurrent_SameKey(b *testing.B) { + b.ReportAllocs() + + cache := newKeyCache(CacheTypeIntermediateKeys, NewCryptoPolicy()) + defer cache.Close() + + keyMeta := KeyMeta{ID: "concurrent_key", Created: time.Now().Unix()} + + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + key, err := cache.GetOrLoad(keyMeta, func(meta KeyMeta) (*internal.CryptoKey, error) { + return internal.NewCryptoKeyForTest(meta.Created, false), nil + }) + if err != nil { + b.Error(err) + } + key.Close() + } + }) +} + +// BenchmarkKeyCache_Concurrent_UniqueKeys benchmarks concurrent access to different keys +func BenchmarkKeyCache_Concurrent_UniqueKeys(b *testing.B) { + b.ReportAllocs() + + cache := newKeyCache(CacheTypeIntermediateKeys, NewCryptoPolicy()) + defer cache.Close() + + var counter int64 + + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + keyID := fmt.Sprintf("concurrent_key_%d", atomic.AddInt64(&counter, 1)) + keyMeta := KeyMeta{ID: keyID, Created: time.Now().Unix()} + + key, err := cache.GetOrLoad(keyMeta, func(meta KeyMeta) (*internal.CryptoKey, error) { + return internal.NewCryptoKeyForTest(meta.Created, false), nil + }) + if err != nil { + b.Error(err) + } + key.Close() + } + }) +} + +// BenchmarkSession_Mixed_Operations_Concurrent benchmarks mixed encrypt/decrypt operations +func BenchmarkSession_Mixed_Operations_Concurrent(b *testing.B) { + b.ReportAllocs() + + session := newBenchmarkSession(b) + defer session.Close() + + ctx := context.Background() + + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + payload := internal.GetRandBytes(benchmarkPayloadSize) + + // Encrypt + drr, err := session.Encrypt(ctx, payload) + if err != nil { + b.Error(err) + continue + } + + // Decrypt + _, err = session.Decrypt(ctx, *drr) + if err != nil { + b.Error(err) + } + } + }) +} + +// BenchmarkSessionCache_Concurrent benchmarks concurrent session cache operations +func BenchmarkSessionCache_Concurrent(b *testing.B) { + b.ReportAllocs() + + config := &Config{ + Policy: &CryptoPolicy{ + CacheSessions: true, + SessionCacheMaxSize: 1000, + }, + Product: "benchmark", + Service: "cache", + } + + factory := NewSessionFactory(config, &benchmarkMetastore{}, &benchmarkKMS{}, &benchmarkCrypto{}, WithSecretFactory(benchmarkSecretFactory)) + defer factory.Close() + + var counter int64 + + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + // Use different partition IDs to test cache behavior + partitionID := fmt.Sprintf("partition_%d", atomic.AddInt64(&counter, 1)%100) + + session, err := factory.GetSession(partitionID) + if err != nil { + b.Error(err) + continue + } + session.Close() + } + }) +} + +// BenchmarkCachedCryptoKey_Concurrent_RefCounting benchmarks concurrent reference counting +func BenchmarkCachedCryptoKey_Concurrent_RefCounting(b *testing.B) { + b.ReportAllocs() + + key := internal.NewCryptoKeyForTest(time.Now().Unix(), false) + cachedKey := newCachedCryptoKey(key) + + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + // Simulate typical concurrent usage + cachedKey.increment() // Add reference + cachedKey.Close() // Remove reference + } + }) + + // Final cleanup + cachedKey.Close() +} + +// BenchmarkMemoryPressure_Concurrent_LargePayload benchmarks concurrent performance with larger payloads +func BenchmarkMemoryPressure_Concurrent_LargePayload(b *testing.B) { + b.ReportAllocs() + + session := newBenchmarkSession(b) + defer session.Close() + + // Test with larger payloads to understand memory pressure + largePayloadSize := 64 * 1024 // 64KB + ctx := context.Background() + + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + payload := internal.GetRandBytes(largePayloadSize) + + drr, err := session.Encrypt(ctx, payload) + if err != nil { + b.Error(err) + continue + } + + _, err = session.Decrypt(ctx, *drr) + if err != nil { + b.Error(err) + } + } + }) +} diff --git a/go/appencryption/go.work.sum b/go/appencryption/go.work.sum index 569518e7b..03a93143a 100644 --- a/go/appencryption/go.work.sum +++ b/go/appencryption/go.work.sum @@ -227,7 +227,6 @@ github.com/containerd/typeurl v1.0.2 h1:Chlt8zIieDbzQFzXzAeBEF92KhExuE4p9p92/QmY github.com/containerd/typeurl v1.0.2/go.mod h1:9trJWW2sRlGub4wZJRTW83VtbOLS6hwcDZXTn6oPz9s= github.com/containerd/typeurl/v2 v2.1.1 h1:3Q4Pt7i8nYwy2KmQWIw2+1hTvwTE/6w9FqcttATPO/4= github.com/containerd/typeurl/v2 v2.1.1/go.mod h1:IDp2JFvbwZ31H8dQbEIY7sDl2L3o3HZj1hsSQlywkQ0= -github.com/containerd/typeurl/v2 v2.2.0/go.mod h1:8XOOxnyatxSWuG8OfsZXVnAF4iZfedjS/8UHSPJnX4g= github.com/containerd/zfs v1.0.0 h1:cXLJbx+4Jj7rNsTiqVfm6i+RNLx6FFA2fMmDlEf+Wm8= github.com/containerd/zfs v1.1.0 h1:n7OZ7jZumLIqNJqXrEc/paBM840mORnmGdJDmAmJZHM= github.com/containerd/zfs v1.1.0/go.mod h1:oZF9wBnrnQjpWLaPKEinrx3TQ9a+W/RJO7Zb41d8YLE= @@ -324,7 +323,6 @@ github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= @@ -410,11 +408,8 @@ github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg= github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= -github.com/moby/sys/mount v0.3.4/go.mod h1:KcQJMbQdJHPlq5lcYT+/CjatWM4PuxKe+XLSVS4J6Os= github.com/moby/sys/mountinfo v0.6.2 h1:BzJjoreD5BMFNmD9Rus6gdd1pLuecOFPt8wC+Vygl78= github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= -github.com/moby/sys/mountinfo v0.7.2/go.mod h1:1YOa8w8Ih7uW0wALDUgT1dTTSBrZ+HiBLGws92L2RU4= -github.com/moby/sys/reexec v0.1.0/go.mod h1:EqjBg8F3X7iZe5pU6nRZnYCMUTXoxsjiIfHup5wYIN8= github.com/moby/sys/signal v0.6.0 h1:aDpY94H8VlhTGa9sNYUFCFsMZIUh5wm0B6XkIoJj/iY= github.com/moby/sys/signal v0.7.0 h1:25RW3d5TnQEoKvRbEKUGay6DCQ46IxAVTT9CUMgmsSI= github.com/moby/sys/signal v0.7.0/go.mod h1:GQ6ObYZfqacOwTtlXvcmh9A26dVRul/hbOZn88Kg8Tg= @@ -472,7 +467,6 @@ github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8 h1:2c1EFnZHIPCW8qKWgHMH/fX2PkSabFc5mrVzfUNdg5U= -github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY= github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/sclevine/spec v1.2.0 h1:1Jwdf9jSfDl9NVmt8ndHqbTZ7XCCPbh1jI3hkDBHVYA= github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646 h1:RpforrEYXWkmGwJHIGnLZ3tTWStkjVVstwzNGqxX2Ds= @@ -575,6 +569,7 @@ golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= +golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/telemetry v0.0.0-20240521205824-bda55230c457/go.mod h1:pRgIJT+bRLFKnoM1ldnzKoxTIn14Yxz928LQRYYgIN0= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= diff --git a/go/appencryption/hotpath_benchmark_test.go b/go/appencryption/hotpath_benchmark_test.go new file mode 100644 index 000000000..a7cc9b2f3 --- /dev/null +++ b/go/appencryption/hotpath_benchmark_test.go @@ -0,0 +1,315 @@ +package appencryption + +import ( + "context" + "testing" + "time" + + "github.com/godaddy/asherah/go/securememory/memguard" + "github.com/stretchr/testify/require" + + "github.com/godaddy/asherah/go/appencryption/internal" +) + +// Hot path benchmarks with allocation tracking for performance monitoring +// These benchmarks focus on the most frequently used operations in production systems + +const ( + benchmarkPartitionID = "benchmark_partition" + benchmarkPayloadSize = 1024 // 1KB payload for realistic testing +) + +var benchmarkSecretFactory = new(memguard.SecretFactory) + +// Create minimal test implementations to avoid import cycles + +type benchmarkMetastore struct{} + +func (m *benchmarkMetastore) Load(ctx context.Context, keyID string, created int64) (*EnvelopeKeyRecord, error) { + return nil, nil // Simulate no existing key +} + +func (m *benchmarkMetastore) LoadLatest(ctx context.Context, keyID string) (*EnvelopeKeyRecord, error) { + return nil, nil // Simulate no existing key +} + +func (m *benchmarkMetastore) Store(ctx context.Context, keyID string, created int64, envelope *EnvelopeKeyRecord) (bool, error) { + return true, nil // Simulate successful store +} + +type benchmarkKMS struct{} + +// Remove GenerateDataKey as it's not part of the interface + +func (k *benchmarkKMS) EncryptKey(ctx context.Context, key []byte) ([]byte, error) { + return internal.GetRandBytes(48), nil // Simulated encrypted key +} + +func (k *benchmarkKMS) DecryptKey(ctx context.Context, encryptedKey []byte) ([]byte, error) { + return internal.GetRandBytes(32), nil // Simulated decrypted key +} + +func (k *benchmarkKMS) Close() error { + return nil +} + +type benchmarkCrypto struct{} + +func (c *benchmarkCrypto) Encrypt(plaintext, key []byte) ([]byte, error) { + // Simulate encryption overhead by doing some work + result := make([]byte, len(plaintext)+16) // Add tag + copy(result, plaintext) + return result, nil +} + +func (c *benchmarkCrypto) Decrypt(ciphertext, key []byte) ([]byte, error) { + // Simulate decryption by returning the original length + if len(ciphertext) < 16 { + return nil, nil + } + return ciphertext[:len(ciphertext)-16], nil +} + +func (c *benchmarkCrypto) GenerateKey() ([]byte, error) { + return internal.GetRandBytes(32), nil +} + +// Helper functions for creating test instances + +func newBenchmarkSessionFactory(_ *testing.B) *SessionFactory { + config := &Config{ + Policy: NewCryptoPolicy(), + Product: "benchmark", + Service: "test", + } + + return NewSessionFactory( + config, + &benchmarkMetastore{}, + &benchmarkKMS{}, + &benchmarkCrypto{}, + WithSecretFactory(benchmarkSecretFactory), + ) +} + +func newBenchmarkSession(b *testing.B) *Session { + factory := newBenchmarkSessionFactory(b) + session, err := factory.GetSession(benchmarkPartitionID) + require.NoError(b, err) + return session +} + +// BenchmarkSessionFactory_GetSession_HotPath benchmarks the hot path of getting a session +// This is one of the most critical operations as it's called for every encrypt/decrypt +func BenchmarkSessionFactory_GetSession_HotPath(b *testing.B) { + b.ReportAllocs() + + factory := newBenchmarkSessionFactory(b) + defer factory.Close() + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + session, err := factory.GetSession(benchmarkPartitionID) + if err != nil { + b.Fatal(err) + } + session.Close() + } +} + +// BenchmarkSessionFactory_GetSession_Cached benchmarks session retrieval when cached +func BenchmarkSessionFactory_GetSession_Cached(b *testing.B) { + b.ReportAllocs() + + config := &Config{ + Policy: &CryptoPolicy{ + CacheSessions: true, + SessionCacheMaxSize: 1000, + SharedIntermediateKeyCache: true, + }, + Product: "benchmark", + Service: "test", + } + + factory := NewSessionFactory(config, &benchmarkMetastore{}, &benchmarkKMS{}, &benchmarkCrypto{}) + defer factory.Close() + + // Pre-warm the cache + session, err := factory.GetSession(benchmarkPartitionID) + require.NoError(b, err) + session.Close() + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + session, err := factory.GetSession(benchmarkPartitionID) + if err != nil { + b.Fatal(err) + } + session.Close() + } +} + +// BenchmarkSession_Encrypt_HotPath benchmarks the critical encrypt operation +func BenchmarkSession_Encrypt_HotPath(b *testing.B) { + b.ReportAllocs() + + session := newBenchmarkSession(b) + defer session.Close() + + payload := internal.GetRandBytes(benchmarkPayloadSize) + ctx := context.Background() + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, err := session.Encrypt(ctx, payload) + if err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkSession_Decrypt_HotPath benchmarks the critical decrypt operation +func BenchmarkSession_Decrypt_HotPath(b *testing.B) { + b.ReportAllocs() + + session := newBenchmarkSession(b) + defer session.Close() + + payload := internal.GetRandBytes(benchmarkPayloadSize) + ctx := context.Background() + + // Pre-encrypt the data for decryption benchmark + drr, err := session.Encrypt(ctx, payload) + require.NoError(b, err) + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, err := session.Decrypt(ctx, *drr) + if err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkSession_EncryptDecrypt_RoundTrip benchmarks full round-trip operation +func BenchmarkSession_EncryptDecrypt_RoundTrip(b *testing.B) { + b.ReportAllocs() + + session := newBenchmarkSession(b) + defer session.Close() + + ctx := context.Background() + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + payload := internal.GetRandBytes(benchmarkPayloadSize) + + drr, err := session.Encrypt(ctx, payload) + if err != nil { + b.Fatal(err) + } + + decrypted, err := session.Decrypt(ctx, *drr) + if err != nil { + b.Fatal(err) + } + + if len(decrypted) != len(payload) { + b.Fatal("payload size mismatch") + } + } +} + +// BenchmarkKeyCache_GetOrLoad_WithAllocation benchmarks key cache with allocation tracking +func BenchmarkKeyCache_GetOrLoad_WithAllocation(b *testing.B) { + b.ReportAllocs() + + cache := newKeyCache(CacheTypeIntermediateKeys, NewCryptoPolicy()) + defer cache.Close() + + keyMeta := KeyMeta{ID: "benchmark_key", Created: time.Now().Unix()} + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + key, err := cache.GetOrLoad(keyMeta, func(meta KeyMeta) (*internal.CryptoKey, error) { + return internal.NewCryptoKeyForTest(meta.Created, false), nil + }) + if err != nil { + b.Fatal(err) + } + key.Close() + } +} + +// BenchmarkKeyCache_GetOrLoadLatest_WithAllocation benchmarks latest key retrieval +func BenchmarkKeyCache_GetOrLoadLatest_WithAllocation(b *testing.B) { + b.ReportAllocs() + + cache := newKeyCache(CacheTypeIntermediateKeys, NewCryptoPolicy()) + defer cache.Close() + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + key, err := cache.GetOrLoadLatest("benchmark_key", func(meta KeyMeta) (*internal.CryptoKey, error) { + return internal.NewCryptoKeyForTest(time.Now().Unix(), false), nil + }) + if err != nil { + b.Fatal(err) + } + key.Close() + } +} + +// BenchmarkCachedCryptoKey_Operations_WithAllocation benchmarks key reference operations +func BenchmarkCachedCryptoKey_Operations_WithAllocation(b *testing.B) { + b.ReportAllocs() + + key := internal.NewCryptoKeyForTest(time.Now().Unix(), false) + cachedKey := newCachedCryptoKey(key) + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + // Simulate typical usage pattern + cachedKey.increment() // Get reference + cachedKey.Close() // Release reference + } + + // Final cleanup + cachedKey.Close() +} + +// BenchmarkMemoryPressure_LargePayload benchmarks performance with larger payloads +func BenchmarkMemoryPressure_LargePayload(b *testing.B) { + b.ReportAllocs() + + session := newBenchmarkSession(b) + defer session.Close() + + // Test with larger payloads to understand memory pressure + largePayloadSize := 64 * 1024 // 64KB + ctx := context.Background() + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + payload := internal.GetRandBytes(largePayloadSize) + + drr, err := session.Encrypt(ctx, payload) + if err != nil { + b.Fatal(err) + } + + _, err = session.Decrypt(ctx, *drr) + if err != nil { + b.Fatal(err) + } + } +}