From 1fc702f890f9aed08dad0678a17a2f949ea93ee1 Mon Sep 17 00:00:00 2001 From: Florian Date: Tue, 31 Mar 2026 23:36:29 +0200 Subject: [PATCH 1/2] Adding more pester tests for encryption --- .../Tests/SqlPipeline_DuckDB.Tests.ps1 | 107 ++++++++++++++++++ 1 file changed, 107 insertions(+) diff --git a/SqlPipeline/Tests/SqlPipeline_DuckDB.Tests.ps1 b/SqlPipeline/Tests/SqlPipeline_DuckDB.Tests.ps1 index 9cf5dee..2fa60ab 100644 --- a/SqlPipeline/Tests/SqlPipeline_DuckDB.Tests.ps1 +++ b/SqlPipeline/Tests/SqlPipeline_DuckDB.Tests.ps1 @@ -389,6 +389,113 @@ Describe "Initialize-SQLPipeline and Close-SqlPipeline" -Skip:(-not $script:duck } +Describe "Encryption (Initialize-SQLPipeline -EncryptionKey)" -Skip:(-not $script:duckDBAvailable) { + + BeforeAll { + $script:encKey = 'pester-test-secret-key-32chars!!' + } + + It "Returns a DuckDB connection object when -EncryptionKey is supplied" { + $filePath = Join-Path ([System.IO.Path]::GetTempPath()) "pester_enc_$(Get-Random).db" + $conn = Initialize-SQLPipeline -DbPath $filePath -EncryptionKey $script:encKey + $conn | Should -Not -BeNullOrEmpty + $conn.GetType().Name | Should -Be "DuckDBConnection" + Close-SqlPipeline -Connection $conn + Remove-Item $filePath -Force -ErrorAction SilentlyContinue + Remove-Item "$filePath.wal" -Force -ErrorAction SilentlyContinue + } + + It "Creates the encrypted database file on disk" { + $filePath = Join-Path ([System.IO.Path]::GetTempPath()) "pester_enc_$(Get-Random).db" + $conn = Initialize-SQLPipeline -DbPath $filePath -EncryptionKey $script:encKey + Test-Path $filePath | Should -Be $true + Close-SqlPipeline -Connection $conn + Remove-Item $filePath -Force -ErrorAction SilentlyContinue + Remove-Item "$filePath.wal" -Force -ErrorAction SilentlyContinue + } + + It "Connection state is Open after Initialize-SQLPipeline with encryption" { + $filePath = Join-Path ([System.IO.Path]::GetTempPath()) "pester_enc_$(Get-Random).db" + $conn = Initialize-SQLPipeline -DbPath $filePath -EncryptionKey $script:encKey + $conn.State | Should -Be "Open" + Close-SqlPipeline -Connection $conn + Remove-Item $filePath -Force -ErrorAction SilentlyContinue + Remove-Item "$filePath.wal" -Force -ErrorAction SilentlyContinue + } + + It "Can write and read data from an encrypted database" { + $filePath = Join-Path ([System.IO.Path]::GetTempPath()) "pester_enc_$(Get-Random).db" + $conn = Initialize-SQLPipeline -DbPath $filePath -EncryptionKey $script:encKey + [PSCustomObject]@{ Id = 1; Secret = "classified" } | Add-RowsToDuckDB -Connection $conn -TableName "enc_rw" + $result = Get-DuckDBData -Connection $conn -Query "SELECT * FROM enc_rw" + Close-SqlPipeline -Connection $conn + Remove-Item $filePath -Force -ErrorAction SilentlyContinue + Remove-Item "$filePath.wal" -Force -ErrorAction SilentlyContinue + + $result.Rows.Count | Should -Be 1 + $result.Rows[0]["Secret"] | Should -Be "classified" + } + + It "Encrypted data persists across reconnect with the correct key" { + $filePath = Join-Path ([System.IO.Path]::GetTempPath()) "pester_enc_$(Get-Random).db" + + $conn1 = Initialize-SQLPipeline -DbPath $filePath -EncryptionKey $script:encKey + [PSCustomObject]@{ Id = 42; Val = "encrypted-persist" } | + Add-RowsToDuckDB -Connection $conn1 -TableName "enc_persist" + Close-SqlPipeline -Connection $conn1 + + $conn2 = Initialize-SQLPipeline -DbPath $filePath -EncryptionKey $script:encKey + $result = Get-DuckDBData -Connection $conn2 -Query "SELECT * FROM enc_persist" + Close-SqlPipeline -Connection $conn2 + Remove-Item $filePath -Force -ErrorAction SilentlyContinue + Remove-Item "$filePath.wal" -Force -ErrorAction SilentlyContinue + + $result.Rows.Count | Should -Be 1 + $result.Rows[0]["Val"] | Should -Be "encrypted-persist" + } + + It "Opening an encrypted database with the wrong key throws" { + $filePath = Join-Path ([System.IO.Path]::GetTempPath()) "pester_enc_$(Get-Random).db" + $conn = Initialize-SQLPipeline -DbPath $filePath -EncryptionKey $script:encKey + Close-SqlPipeline -Connection $conn + + { Initialize-SQLPipeline -DbPath $filePath -EncryptionKey "definitely-wrong-key" } | Should -Throw + + Remove-Item $filePath -Force -ErrorAction SilentlyContinue + Remove-Item "$filePath.wal" -Force -ErrorAction SilentlyContinue + } + + It "Uses CTR cipher without throwing when -EncryptionCipher CTR is specified" { + $filePath = Join-Path ([System.IO.Path]::GetTempPath()) "pester_enc_$(Get-Random).db" + { + $conn = Initialize-SQLPipeline -DbPath $filePath -EncryptionKey $script:encKey -EncryptionCipher CTR + Close-SqlPipeline -Connection $conn + } | Should -Not -Throw + Remove-Item $filePath -Force -ErrorAction SilentlyContinue + Remove-Item "$filePath.wal" -Force -ErrorAction SilentlyContinue + } + + It "CTR-encrypted data can be read back with the same key and cipher" { + $filePath = Join-Path ([System.IO.Path]::GetTempPath()) "pester_enc_$(Get-Random).db" + + $conn1 = Initialize-SQLPipeline -DbPath $filePath -EncryptionKey $script:encKey -EncryptionCipher CTR + [PSCustomObject]@{ Id = 1; Val = "ctr-value" } | + Add-RowsToDuckDB -Connection $conn1 -TableName "enc_ctr" + Close-SqlPipeline -Connection $conn1 + + $conn2 = Initialize-SQLPipeline -DbPath $filePath -EncryptionKey $script:encKey -EncryptionCipher CTR + $result = Get-DuckDBData -Connection $conn2 -Query "SELECT * FROM enc_ctr" + Close-SqlPipeline -Connection $conn2 + Remove-Item $filePath -Force -ErrorAction SilentlyContinue + Remove-Item "$filePath.wal" -Force -ErrorAction SilentlyContinue + + $result.Rows.Count | Should -Be 1 + $result.Rows[0]["Val"] | Should -Be "ctr-value" + } + +} + + Describe "Export-DuckDBToParquet" -Skip:(-not $script:duckDBAvailable) { BeforeAll { From bedfb5937ac300270bf7307a3de5eb57f8633f09 Mon Sep 17 00:00:00 2001 From: Florian Date: Tue, 31 Mar 2026 23:38:06 +0200 Subject: [PATCH 2/2] Fixed Sqlpipeline duckdb encryption and push to 0.3.8 --- .../Private/duckdb/New-DuckDBConnection.ps1 | 54 +++++++++++++++---- .../Public/duckdb/Initialize-SQLPipeline.ps1 | 14 ++++- SqlPipeline/SqlPipeline/SqlPipeline.psd1 | 3 +- 3 files changed, 59 insertions(+), 12 deletions(-) diff --git a/SqlPipeline/SqlPipeline/Private/duckdb/New-DuckDBConnection.ps1 b/SqlPipeline/SqlPipeline/Private/duckdb/New-DuckDBConnection.ps1 index e9bc29c..6f7ad34 100644 --- a/SqlPipeline/SqlPipeline/Private/duckdb/New-DuckDBConnection.ps1 +++ b/SqlPipeline/SqlPipeline/Private/duckdb/New-DuckDBConnection.ps1 @@ -6,8 +6,14 @@ function New-DuckDBConnection { Path to the .db file (created if it does not exist). Use ':memory:' for an in-memory database. .PARAMETER EncryptionKey Optional encryption key (AES-256, requires DuckDB 1.4.0 or later). + When provided the database is attached via ATTACH ... (ENCRYPTION_KEY '...') and + set as the default catalog with USE, so all subsequent queries are transparent. + .PARAMETER EncryptionCipher + Cipher to use when EncryptionKey is set. 'GCM' (default, authenticated) or 'CTR' (faster, no integrity check). .EXAMPLE $conn = New-DuckDBConnection -DbPath '.\pipeline.db' + .EXAMPLE + $conn = New-DuckDBConnection -DbPath '.\pipeline.db' -EncryptionKey 'mysecretkey' #> [CmdletBinding()] [OutputType([DuckDB.NET.Data.DuckDBConnection])] @@ -15,21 +21,51 @@ function New-DuckDBConnection { [Parameter(Mandatory)] [string]$DbPath, - [string]$EncryptionKey - #[string]$LibPath = '.\lib' + [string]$EncryptionKey, + + [ValidateSet('GCM', 'CTR')] + [string]$EncryptionCipher = 'GCM' ) - #Initialize-DuckDB -LibPath $LibPath If ( -not $Script:isDuckDBLoaded ) { throw "DuckDB.NET is not loaded. Please ensure it is installed and available in the lib folder." } - $connStr = "DataSource=$DbPath" - if ($EncryptionKey) { $connStr += ";EncryptionKey=$EncryptionKey" } + if ($EncryptionKey -and $DbPath -ne ':memory:') { + # DuckDB encryption is configured via ATTACH, not via connection string. + # Open a plain in-memory bootstrap connection, attach the encrypted file, + # then make it the default catalog so all subsequent queries are transparent. + $conn = [DuckDB.NET.Data.DuckDBConnection]::new('DataSource=:memory:') + $conn.Open() + + # Escape single quotes in path and key to prevent SQL injection + $escapedPath = $DbPath -replace "'", "''" + $escapedKey = $EncryptionKey -replace "'", "''" + + $cmd = $conn.CreateCommand() + try { + # DuckDB 1.4.1+ requires OpenSSL (via httpfs) for writes to encrypted databases. + $cmd.CommandText = 'INSTALL httpfs' + $null = $cmd.ExecuteNonQuery() + $cmd.CommandText = 'LOAD httpfs' + $null = $cmd.ExecuteNonQuery() + + $cmd.CommandText = "ATTACH '$escapedPath' AS encrypted_db (ENCRYPTION_KEY '$escapedKey', ENCRYPTION_CIPHER '$EncryptionCipher')" + $null = $cmd.ExecuteNonQuery() + + $cmd.CommandText = 'USE encrypted_db' + $null = $cmd.ExecuteNonQuery() + } finally { + $cmd.Dispose() + } + + Write-Verbose "Encrypted connection opened: $DbPath" + } else { + $conn = [DuckDB.NET.Data.DuckDBConnection]::new("DataSource=$DbPath") + $conn.Open() + Write-Verbose "Connection opened: $DbPath" + } - $conn = [DuckDB.NET.Data.DuckDBConnection]::new($connStr) - $conn.Open() - Write-Verbose "Connection opened: $DbPath" $conn - + } diff --git a/SqlPipeline/SqlPipeline/Public/duckdb/Initialize-SQLPipeline.ps1 b/SqlPipeline/SqlPipeline/Public/duckdb/Initialize-SQLPipeline.ps1 index 29ec42a..91ad5b6 100644 --- a/SqlPipeline/SqlPipeline/Public/duckdb/Initialize-SQLPipeline.ps1 +++ b/SqlPipeline/SqlPipeline/Public/duckdb/Initialize-SQLPipeline.ps1 @@ -17,6 +17,11 @@ function Initialize-SQLPipeline { .PARAMETER EncryptionKey Optional encryption key (AES-256, requires DuckDB 1.4.0 or later). + Encryption is applied via ATTACH ... (ENCRYPTION_KEY '...') so the key + never appears in the connection string. + + .PARAMETER EncryptionCipher + Cipher to use when EncryptionKey is set. 'GCM' (default, authenticated) or 'CTR' (faster, no integrity check). .EXAMPLE # File-based database @@ -34,12 +39,17 @@ function Initialize-SQLPipeline { [OutputType([DuckDB.NET.Data.DuckDBConnection])] param( [Parameter(Mandatory)] [string]$DbPath, - [string]$EncryptionKey + [string]$EncryptionKey, + [ValidateSet('GCM', 'CTR')] + [string]$EncryptionCipher = 'GCM' ) $absolutePath = $ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath($DbPath) $params = @{ DbPath = $absolutePath } - if ($EncryptionKey) { $params['EncryptionKey'] = $EncryptionKey } + if ($EncryptionKey) { + $params['EncryptionKey'] = $EncryptionKey + $params['EncryptionCipher'] = $EncryptionCipher + } $conn = New-DuckDBConnection @params Initialize-PipelineMetadata -Connection $conn diff --git a/SqlPipeline/SqlPipeline/SqlPipeline.psd1 b/SqlPipeline/SqlPipeline/SqlPipeline.psd1 index 08ad111..a1fbc3f 100644 --- a/SqlPipeline/SqlPipeline/SqlPipeline.psd1 +++ b/SqlPipeline/SqlPipeline/SqlPipeline.psd1 @@ -5,7 +5,7 @@ RootModule = 'SqlPipeline.psm1' # Die Versionsnummer dieses Moduls -ModuleVersion = '0.3.7' +ModuleVersion = '0.3.8' # Unterstützte PSEditions # CompatiblePSEditions = @() @@ -126,6 +126,7 @@ PrivateData = @{ # 'ReleaseNotes' des Moduls ReleaseNotes = ' +0.3.8 Fixed the encryption for DuckDB connections 0.3.7 DuckDB: multi-row type inference & appender fixes with numeric and boolean types 0.3.6 Adding functionality to count updates and inserts when executing the MERGE 0.3.5 Added function to show open DuckDB connections: Show-DuckDBConnection