From f665031a29a7533c0bb54d163ee2153ad08ad905 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 28 Mar 2026 08:39:43 +0000 Subject: [PATCH 1/6] Add Pester tests for ImportDependency and DuckDB, continue InstallDependency in ImportDependency style MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - ImportDependency/Tests/ImportDependency.Tests.ps1 (new): Pester tests covering Get-TemporaryPath, Get-LocalPackage (nuspec discovery, properties, multi-root), Get-PSEnvironment (all keys, types, skip flags, local-package integration), and Import-Dependency (no-op, missing module, KeepLogfile, empty lib folder) - SqlPipeline/Tests/SqlPipeline.Tests.ps1: Extend with DuckDB test suites that skip gracefully when DuckDB.NET is unavailable; covers Invoke-DuckDBQuery, Get-DuckDBData, Add-RowsToDuckDB (insert, UPSERT, schema evolution, batching), Set-LoadMetadata / Get-LastLoadTimestamp, Initialize-SQLPipeline / Close-SqlPipeline (file persistence, connection state), and Export-DuckDBToParquet - InstallDependency/InstallDependency/InstallDependency.psm1: Rewritten to match ImportDependency style — verbose param, notes region, PSEdition=='Core' check, inaccessible module-path cleanup, Windows-specific function loading, executionPolicy variable, 'win' runtime fallback, MacOS elevation support, ListAvailable for PackageManagement/PowerShellGet, background jobs using .psd1 scanning - InstallDependency/InstallDependency/Public/Install-Dependency.ps1: Refactored to use proper Begin/Process/End blocks (logfile setup and counter init in Begin, install logic in Process, status summary and logfile restore in End), matching the Import-Dependency.ps1 pattern; SuppressWarnings/writeToHost wired through https://claude.ai/code/session_01W4sAhVb9Y6Lkr53VYq5hdq --- .../Tests/ImportDependency.Tests.ps1 | 295 ++++++++++++++ .../InstallDependency/InstallDependency.psm1 | 385 ++++++++++++------ .../Public/Install-Dependency.ps1 | 323 ++++++--------- SqlPipeline/Tests/SqlPipeline.Tests.ps1 | 309 +++++++++++++- 4 files changed, 975 insertions(+), 337 deletions(-) create mode 100644 ImportDependency/Tests/ImportDependency.Tests.ps1 diff --git a/ImportDependency/Tests/ImportDependency.Tests.ps1 b/ImportDependency/Tests/ImportDependency.Tests.ps1 new file mode 100644 index 0000000..5a63914 --- /dev/null +++ b/ImportDependency/Tests/ImportDependency.Tests.ps1 @@ -0,0 +1,295 @@ +BeforeAll { + Import-Module "$PSScriptRoot/../ImportDependency" -Force +} + +AfterAll { + Remove-Module ImportDependency -Force -ErrorAction SilentlyContinue + Remove-Module WriteLog -Force -ErrorAction SilentlyContinue +} + + +# --------------------------------------------------------------------------- +Describe "Get-TemporaryPath" { +# --------------------------------------------------------------------------- + + It "Returns a non-null string" { + $result = Get-TemporaryPath + $result | Should -Not -BeNullOrEmpty + } + + It "Returns an existing path" { + $result = Get-TemporaryPath + Test-Path $result | Should -Be $true + } + + It "Returns a string type" { + $result = Get-TemporaryPath + $result | Should -BeOfType [string] + } + +} + + +# --------------------------------------------------------------------------- +Describe "Get-LocalPackage" { +# --------------------------------------------------------------------------- + + BeforeAll { + # Create a temp directory with a mock nuspec file + $script:tempPkgDir = Join-Path ([System.IO.Path]::GetTempPath()) "pester_importdep_$(Get-Random)" + New-Item -ItemType Directory -Path $script:tempPkgDir | Out-Null + + @" + + + + TestPackage + 1.2.3 + A test package for Pester + Test Author + + +"@ | Set-Content -Path (Join-Path $script:tempPkgDir "TestPackage.nuspec") -Encoding UTF8 + + # Second package in a sub-directory + $script:tempPkgSubDir = Join-Path $script:tempPkgDir "subpkg" + New-Item -ItemType Directory -Path $script:tempPkgSubDir | Out-Null + + @" + + + + SubPackage + 0.5.0 + Sub package + Sub Author + + +"@ | Set-Content -Path (Join-Path $script:tempPkgSubDir "SubPackage.nuspec") -Encoding UTF8 + } + + AfterAll { + Remove-Item -Path $script:tempPkgDir -Recurse -Force -ErrorAction SilentlyContinue + } + + It "Returns a package found via nuspec file" { + $result = Get-LocalPackage -NugetRoot $script:tempPkgDir + $result | Should -Not -BeNullOrEmpty + } + + It "Returns the correct package Id" { + $result = Get-LocalPackage -NugetRoot $script:tempPkgDir + $result.Id | Should -Contain "TestPackage" + } + + It "Returns the correct package version" { + $result = Get-LocalPackage -NugetRoot $script:tempPkgDir + ($result | Where-Object { $_.Id -eq "TestPackage" }).Version | Should -Be "1.2.3" + } + + It "Returns the correct package authors" { + $result = Get-LocalPackage -NugetRoot $script:tempPkgDir + ($result | Where-Object { $_.Id -eq "TestPackage" }).Authors | Should -Be "Test Author" + } + + It "Returns the correct package description" { + $result = Get-LocalPackage -NugetRoot $script:tempPkgDir + ($result | Where-Object { $_.Id -eq "TestPackage" }).Description | Should -Be "A test package for Pester" + } + + It "Returns packages from sub-directories" { + $result = Get-LocalPackage -NugetRoot $script:tempPkgDir + $result.Id | Should -Contain "SubPackage" + } + + It "Returns PSCustomObject with all expected properties" { + $result = Get-LocalPackage -NugetRoot $script:tempPkgDir + $pkg = $result | Select-Object -First 1 + $pkg.PSObject.Properties.Name | Should -Contain "Id" + $pkg.PSObject.Properties.Name | Should -Contain "Version" + $pkg.PSObject.Properties.Name | Should -Contain "Description" + $pkg.PSObject.Properties.Name | Should -Contain "Authors" + $pkg.PSObject.Properties.Name | Should -Contain "Path" + $pkg.PSObject.Properties.Name | Should -Contain "SizeMB" + $pkg.PSObject.Properties.Name | Should -Contain "Source" + } + + It "Source property is 'nuspec' for nuspec-based packages" { + $result = Get-LocalPackage -NugetRoot $script:tempPkgDir + ($result | Where-Object { $_.Id -eq "TestPackage" }).Source | Should -Be "nuspec" + } + + It "Returns empty result for non-existent path" { + $nonExistent = Join-Path ([System.IO.Path]::GetTempPath()) "nonexistent_pkg_$(Get-Random)" + $result = Get-LocalPackage -NugetRoot $nonExistent + $result | Should -BeNullOrEmpty + } + + It "Accepts multiple NugetRoot paths" { + $anotherDir = Join-Path ([System.IO.Path]::GetTempPath()) "pester_importdep2_$(Get-Random)" + New-Item -ItemType Directory -Path $anotherDir | Out-Null + @" + +AnotherPkg2.0.0XY +"@ | Set-Content -Path (Join-Path $anotherDir "AnotherPkg.nuspec") -Encoding UTF8 + + $result = Get-LocalPackage -NugetRoot @($script:tempPkgDir, $anotherDir) + $result.Id | Should -Contain "AnotherPkg" + + Remove-Item -Path $anotherDir -Recurse -Force -ErrorAction SilentlyContinue + } + +} + + +# --------------------------------------------------------------------------- +Describe "Get-PSEnvironment" { +# --------------------------------------------------------------------------- + + It "Returns an ordered dictionary" { + $result = Get-PSEnvironment -SkipBackgroundCheck -SkipLocalPackageCheck + $result | Should -BeOfType [System.Collections.Specialized.OrderedDictionary] + } + + It "Contains all required top-level keys" { + $result = Get-PSEnvironment -SkipBackgroundCheck -SkipLocalPackageCheck + foreach ($key in @("PSVersion","PSEdition","OS","IsCore","IsCoreInstalled","DefaultPSCore", + "Architecture","CurrentRuntime","Is64BitOS","Is64BitProcess", + "ExecutingUser","ExecutionPolicy","IsElevated", + "RuntimePreference","FrameworkPreference", + "PackageManagement","PowerShellGet","VcRedist", + "BackgroundCheckCompleted","InstalledModules","InstalledGlobalPackages", + "LocalPackageCheckCompleted","InstalledLocalPackages")) { + $result.Keys | Should -Contain $key -Because "key '$key' must be present" + } + } + + It "PSVersion is a non-empty string" { + $result = Get-PSEnvironment -SkipBackgroundCheck -SkipLocalPackageCheck + $result.PSVersion | Should -Not -BeNullOrEmpty + $result.PSVersion | Should -BeOfType [string] + } + + It "OS is one of the recognised values" { + $result = Get-PSEnvironment -SkipBackgroundCheck -SkipLocalPackageCheck + $result.OS | Should -BeIn @("Windows", "Linux", "MacOS") + } + + It "IsCore is a boolean" { + $result = Get-PSEnvironment -SkipBackgroundCheck -SkipLocalPackageCheck + $result.IsCore | Should -BeOfType [bool] + } + + It "Is64BitOS is a boolean" { + $result = Get-PSEnvironment -SkipBackgroundCheck -SkipLocalPackageCheck + $result.Is64BitOS | Should -BeOfType [bool] + } + + It "Is64BitProcess is a boolean" { + $result = Get-PSEnvironment -SkipBackgroundCheck -SkipLocalPackageCheck + $result.Is64BitProcess | Should -BeOfType [bool] + } + + It "Architecture is a recognised value" { + $result = Get-PSEnvironment -SkipBackgroundCheck -SkipLocalPackageCheck + $result.Architecture | Should -BeIn @("x64", "x32", "ARM64", "ARM", "Unknown") + } + + It "RuntimePreference is a non-empty string" { + $result = Get-PSEnvironment -SkipBackgroundCheck -SkipLocalPackageCheck + $result.RuntimePreference | Should -Not -BeNullOrEmpty + } + + It "FrameworkPreference is a non-empty string" { + $result = Get-PSEnvironment -SkipBackgroundCheck -SkipLocalPackageCheck + $result.FrameworkPreference | Should -Not -BeNullOrEmpty + } + + It "DefaultPSCore contains Version, Is64Bit, Path keys" { + $result = Get-PSEnvironment -SkipBackgroundCheck -SkipLocalPackageCheck + $result.DefaultPSCore.Keys | Should -Contain "Version" + $result.DefaultPSCore.Keys | Should -Contain "Is64Bit" + $result.DefaultPSCore.Keys | Should -Contain "Path" + } + + It "BackgroundCheckCompleted is false when -SkipBackgroundCheck is set" { + $result = Get-PSEnvironment -SkipBackgroundCheck -SkipLocalPackageCheck + $result.BackgroundCheckCompleted | Should -Be $false + } + + It "LocalPackageCheckCompleted is false when -SkipLocalPackageCheck is set" { + $result = Get-PSEnvironment -SkipBackgroundCheck -SkipLocalPackageCheck + $result.LocalPackageCheckCompleted | Should -Be $false + } + + It "BackgroundCheckCompleted is true when background check runs" { + $result = Get-PSEnvironment -SkipLocalPackageCheck + $result.BackgroundCheckCompleted | Should -Be $true + } + + It "LocalPackageCheckCompleted is true when local check runs against a missing folder" { + # Folder does not exist - check still runs (and finds nothing) + $result = Get-PSEnvironment -SkipBackgroundCheck -LocalPackageFolder (Join-Path ([System.IO.Path]::GetTempPath()) "no_pkg_$(Get-Random)") + $result.LocalPackageCheckCompleted | Should -Be $true + } + + It "InstalledLocalPackages is empty for a temp folder with no packages" { + $emptyDir = Join-Path ([System.IO.Path]::GetTempPath()) "pester_empty_$(Get-Random)" + New-Item -ItemType Directory -Path $emptyDir | Out-Null + $result = Get-PSEnvironment -SkipBackgroundCheck -LocalPackageFolder $emptyDir + $result.InstalledLocalPackages | Should -BeNullOrEmpty + Remove-Item $emptyDir -Force + } + + It "InstalledLocalPackages returns packages when a valid lib folder is supplied" { + $libDir = Join-Path ([System.IO.Path]::GetTempPath()) "pester_lib_$(Get-Random)" + New-Item -ItemType Directory -Path $libDir | Out-Null + @" + +LibPkg3.0.0XY +"@ | Set-Content -Path (Join-Path $libDir "LibPkg.nuspec") -Encoding UTF8 + + $result = Get-PSEnvironment -SkipBackgroundCheck -LocalPackageFolder $libDir + $result.InstalledLocalPackages.Id | Should -Contain "LibPkg" + + Remove-Item $libDir -Recurse -Force + } + +} + + +# --------------------------------------------------------------------------- +Describe "Import-Dependency" { +# --------------------------------------------------------------------------- + + It "Runs without errors when called with no arguments" { + { Import-Dependency } | Should -Not -Throw + } + + It "Runs without errors with SuppressWarnings when loading a non-existent module" { + { Import-Dependency -Module "NonExistentModule_$(Get-Random)" -SuppressWarnings } | Should -Not -Throw + } + + It "Runs without errors with LoadWholePackageFolder when the folder does not exist" { + $missingLib = Join-Path ([System.IO.Path]::GetTempPath()) "no_lib_$(Get-Random)" + { Import-Dependency -LoadWholePackageFolder -LocalPackageFolder $missingLib } | Should -Not -Throw + } + + It "Accepts a KeepLogfile switch without throwing" { + { Import-Dependency -KeepLogfile } | Should -Not -Throw + } + + It "Does not load excluded modules (WriteLog and ImportDependency are skipped)" { + # If WriteLog is already loaded, importing it via Import-Dependency is a no-op + # The function itself must not throw + { Import-Dependency -Module "WriteLog" } | Should -Not -Throw + } + + It "Loads a local package folder that exists but is empty without throwing" { + $emptyLib = Join-Path ([System.IO.Path]::GetTempPath()) "empty_lib_$(Get-Random)" + New-Item -ItemType Directory -Path $emptyLib | Out-Null + { Import-Dependency -LocalPackageFolder $emptyLib -LoadWholePackageFolder } | Should -Not -Throw + Remove-Item $emptyLib -Recurse -Force + } + +} diff --git a/InstallDependency/InstallDependency/InstallDependency.psm1 b/InstallDependency/InstallDependency/InstallDependency.psm1 index 05ca63a..3b92951 100644 --- a/InstallDependency/InstallDependency/InstallDependency.psm1 +++ b/InstallDependency/InstallDependency/InstallDependency.psm1 @@ -1,6 +1,24 @@ - + #----------------------------------------------- -# NOTES +#region: VERBOSE OUTPUT +#----------------------------------------------- + +param( + [bool]$Verbose = $false +) + +If ( $Verbose -eq $true ) { + $previousVerbosePreference = $VerbosePreference + $VerbosePreference = "Continue" +} else { + $VerbosePreference = "SilentlyContinue" +} + +#endregion: VERBOSE OUTPUT + + +#----------------------------------------------- +#region: NOTES #----------------------------------------------- <# @@ -12,12 +30,16 @@ https://github.com/RamblingCookieMonster/PSStackExchange/blob/db1277453374cb1668 #> +#endregion: NOTES + #----------------------------------------------- # OS CHECK #----------------------------------------------- -$preCheckisCore = ($PSVersionTable.Keys -contains "PSEdition") -and ($PSVersionTable.PSEdition -ne 'Desktop') +Write-Verbose "Checking the Core and OS" + +$preCheckisCore = $PSVersionTable.Keys -contains "PSEdition" -and $PSVersionTable.PSEdition -eq 'Core' # Check the operating system, if Core if ($preCheckisCore -eq $true) { @@ -31,8 +53,6 @@ if ($preCheckisCore -eq $true) { throw "Unknown operating system" } } else { - # [System.Environment]::OSVersion.VersionString() - # [System.Environment]::Is64BitOperatingSystem $preCheckOs = "Windows" } @@ -41,7 +61,9 @@ if ($preCheckisCore -eq $true) { # ADD MODULE PATH, IF NOT PRESENT #----------------------------------------------- -If ( $preCheckOs -eq "Windows" ) { +If ( $preCheckOs -eq "Windows" -and $preCheckisCore -eq $false ) { + + Write-Verbose "Adding Module path on Windows (when not using Core)" $modulePath = @( [System.Environment]::GetEnvironmentVariable("PSModulePath") -split ";" ) + @( "$( [System.Environment]::GetEnvironmentVariable("ProgramFiles") )\WindowsPowerShell\Modules" @@ -55,29 +77,32 @@ If ( $preCheckOs -eq "Windows" ) { $modulePath += "$( [System.Environment]::GetEnvironmentVariable("ProgramW6432") )\WindowsPowerShell\Modules" } - # Add pwsh core path - If ( $preCheckisCore -eq $true ) { - If ( [System.Environment]::GetEnvironmentVariables().keys -contains "ProgramW6432" ) { - $modulePath += "$( [System.Environment]::GetEnvironmentVariable("ProgramW6432") )\powershell\7\Modules" - } - $modulePath += "$( [System.Environment]::GetEnvironmentVariable("ProgramFiles") )\powershell\7\Modules" - $modulePath += "$( [System.Environment]::GetEnvironmentVariable("ProgramFiles(x86)") )\powershell\7\Modules" - } - # Add all paths # Using $env:PSModulePath for only temporary override $Env:PSModulePath = @( $modulePath | Sort-Object -unique ) -join ";" } +# Check if all module paths are accessible, if not remove them from the path to avoid errors when loading modules +$pathSeparator = if ($preCheckOs -eq 'Windows') { ';' } else { ':' } +$env:PSModulePath = ($env:PSModulePath -split $pathSeparator | Where-Object { + try { + [System.IO.Directory]::GetFiles($_) | Out-Null + $true + } catch { + $false + } +}) -join $pathSeparator + #----------------------------------------------- # ADD SCRIPT PATH, IF NOT PRESENT #----------------------------------------------- -If ( $preCheckOs -eq "Windows" ) { +If ( $preCheckOs -eq "Windows" -and $preCheckisCore -eq $false ) { + + Write-Verbose "Adding Script path on Windows (when not using Core)" - #$envVariables = [System.Environment]::GetEnvironmentVariables() $scriptPath = @( [System.Environment]::GetEnvironmentVariable("Path") -split ";" ) + @( "$( [System.Environment]::GetEnvironmentVariable("ProgramFiles") )\WindowsPowerShell\Scripts" "$( [System.Environment]::GetEnvironmentVariable("ProgramFiles(x86)") )\WindowsPowerShell\Scripts" @@ -89,31 +114,17 @@ If ( $preCheckOs -eq "Windows" ) { $scriptPath += "$( [System.Environment]::GetEnvironmentVariable("ProgramW6432") )\WindowsPowerShell\Scripts" } - # Add pwsh core path - If ( $preCheckisCore -eq $true ) { - If ( [System.Environment]::GetEnvironmentVariables().keys -contains "ProgramW6432" ) { - $scriptPath += "$( [System.Environment]::GetEnvironmentVariable("ProgramW6432") )\powershell\7\Scripts" - } - $scriptPath += "$( [System.Environment]::GetEnvironmentVariable("ProgramFiles") )\powershell\7\Scripts" - $scriptPath += "$( [System.Environment]::GetEnvironmentVariable("ProgramFiles(x86)") )\powershell\7\Scripts" - } - # Using $env:Path for only temporary override $Env:Path = @( $scriptPath | Sort-Object -unique ) -join ";" } -#----------------------------------------------- -# ENUMS -#----------------------------------------------- - - #----------------------------------------------- # LOAD PUBLIC AND PRIVATE FUNCTIONS #----------------------------------------------- -#$PSBoundParameters["Verbose"].IsPresent -eq $true +Write-Verbose "Loading public and private functions" $Public = @( Get-ChildItem -Path "$( $PSScriptRoot )/Public/*.ps1" -ErrorAction SilentlyContinue ) $Private = @( Get-ChildItem -Path "$( $PSScriptRoot )/Private/*.ps1" -ErrorAction SilentlyContinue ) @@ -121,7 +132,7 @@ $Private = @( Get-ChildItem -Path "$( $PSScriptRoot )/Private/*.ps1" -ErrorActio # dot source the files @( $Public + $Private ) | ForEach-Object { $import = $_ - Write-Verbose "Load function $( $import.fullname )" #-verbose + Write-Verbose "Load function $( $import.fullname )" Try { . $import.fullname } Catch { @@ -131,27 +142,38 @@ $Private = @( Get-ChildItem -Path "$( $PSScriptRoot )/Private/*.ps1" -ErrorActio #----------------------------------------------- -# READ IN CONFIG FILES AND VARIABLES +# LOAD WINDOWS SPECIFIC FUNCTIONS #----------------------------------------------- -# ... +Write-Verbose "Loading Windows specific functions" +$WindowsPrivate = @( Get-ChildItem -Path "$( $PSScriptRoot )/Private/Windows/*.ps1" -ErrorAction SilentlyContinue ) -#----------------------------------------------- -# READ IN CONFIG FILES AND VARIABLES -#----------------------------------------------- +If ( $preCheckOs -eq "Windows" ) { + @( $WindowsPrivate ) | ForEach-Object { + $import = $_ + Write-Verbose "Load function $( $import.fullname )" + Try { + . $import.fullname + } Catch { + Write-Error -Message "Failed to import function $( $import.fullname ): $( $_ )" + } + } +} #----------------------------------------------- # SET SOME VARIABLES ONLY VISIBLE TO MODULE AND FUNCTIONS #----------------------------------------------- +Write-Verbose "Define internal module variables" + # Define the variables -#New-Variable -Name execPath -Value $null -Scope Script -Force # Path of the calling script New-Variable -Name psVersion -Value $null -Scope Script -Force # PowerShell version being used New-Variable -Name psEdition -Value $null -Scope Script -Force # Edition of PowerShell (e.g., Desktop, Core) New-Variable -Name platform -Value $null -Scope Script -Force # Platform type (e.g., Windows, Linux, macOS) New-Variable -Name frameworkPreference -Value $null -Scope Script -Force # Preferred .NET framework version +New-Variable -Name runtimePreference -Value $null -Scope Script -Force # Preferred OS native framework version New-Variable -Name isCore -Value $null -Scope Script -Force # Indicates if PowerShell Core is being used (True/False) New-Variable -Name isCoreInstalled -Value $null -Scope Script -Force # Indicates if PowerShell Core is already installed (True/False) New-Variable -Name defaultPsCoreVersion -Value $null -Scope Script -Force # Default version of PowerShell Core that is used @@ -165,65 +187,59 @@ New-Variable -Name isElevated -Value $null -Scope Script -Force # In New-Variable -Name packageManagement -Value $null -Scope Script -Force # Package management system in use (e.g., NuGet, APT) New-Variable -Name powerShellGet -Value $null -Scope Script -Force # Version of PowerShellGet module New-Variable -Name vcredist -Value $null -Scope Script -Force # Indicates if Visual C++ Redistributable is installed (True/False) -New-Variable -Name installedModules -Value $null -Scope Script -Force # Caches all installed PowerShell modules -New-Variable -Name backgroundJobs -Value $null -Scope Script -Force # Hidden variable to store background jobs -New-Variable -Name installedGlobalPackages -Value $null -Scope Script -Force # Caches all installed NuGet Global Packages +New-Variable -Name installedModules -Value $null -Scope Script -Force # Caches all installed PowerShell modules +New-Variable -Name backgroundJobs -Value $null -Scope Script -Force # Hidden variable to store background jobs +New-Variable -Name installedGlobalPackages -Value $null -Scope Script -Force # Caches all installed NuGet global packages +New-Variable -Name executionPolicy -Value $null -Scope Script -Force # Current execution policy # Filling some default values $Script:isCore = $preCheckisCore $Script:os = $preCheckOs $Script:psVersion = $PSVersionTable.PSVersion.ToString() -$Script:powerShellEdition = $PSVersionTable.PSEdition +$Script:powerShellEdition = $PSVersionTable.PSEdition # Need to write that out because psedition is reserved $Script:platform = $PSVersionTable.Platform $Script:is64BitOS = [System.Environment]::Is64BitOperatingSystem $Script:is64BitProcess = [System.Environment]::Is64BitProcess +$Script:executionPolicy = [PSCustomObject]@{ + "LocalMachine" = Get-ExecutionPolicy -Scope LocalMachine + "MachinePolicy" = Get-ExecutionPolicy -Scope MachinePolicy + "Process" = Get-ExecutionPolicy -Scope Process + "CurrentUser" = Get-ExecutionPolicy -Scope CurrentUser + "UserPolicy" = Get-ExecutionPolicy -Scope UserPolicy +} -<# -$Script:frameworkPreference = @( - - # .NET 8+ (future‑proof) - 'net9.0','net8.0','net8.0-windows','net7.0','net7.0-windows', - - # .NET 6 - 'net6.0','net6.0-windows', - - # .NET 5 - 'net5.0','net5.0-windows','netcore50', - - # .NET Standard 2.1 → 2.0 → 1.5 → 1.3 → 1.1 → 1.0 - 'netstandard2.1','netstandard2.0','netstandard1.5', - 'netstandard1.3','netstandard1.1','netstandard1.0', - # Classic .NET Framework descending - 'net48','net47','net462' +#----------------------------------------------- +# CHECKING POWERSHELL CORE DETAILS +#----------------------------------------------- -) -#> +Write-Verbose "Checking more details about PS Core" # Check if pscore is installed $pwshCommand = Get-Command -commandType Application -Name "pwsh*" +$Script:defaultPsCoreVersion = $pwshCommand[0].Version If ( $pwshCommand.Count -gt 0 ) { - If ( ( pwsh { 1+1 } ) -eq 2 ) { - $Script:isCoreInstalled = $true - $Script:defaultPsCoreVersion = pwsh { $PSVersionTable.PSVersion.ToString() } - $Script:defaultPsCoreIs64Bit = pwsh { [System.Environment]::Is64BitProcess } - if ($Script:os -eq "Windows") { - # For Windows - $Script:defaultPsCorePath = ( get-command -name "pwsh*" -CommandType Application | where-object { $_.Source.replace("\pwsh.exe","") -eq ( pwsh { $pshome } ) } ).Source - } elseif ( $Script:os -eq "Linux" ) { - # For Linux - If ( $null -ne (which pwse) ) { - $Script:defaultPsCorePath = (which pwse) - } + $Script:isCoreInstalled = $true + if ($Script:os -eq "Windows") { + # For Windows + $Script:defaultPsCorePath = ( get-command -name "pwsh*" -CommandType Application | where-object { $_.Source.replace("\pwsh.exe","") -eq ( pwsh { $pshome } ) } ).Source + } elseif ( $Script:os -eq "Linux" ) { + # For Linux + If ( $null -ne (which pwsh) ) { + $Script:defaultPsCorePath = (which pwsh) } - } else { - Write-Warning "pwsh command found, but pwsh execution test failed." } - } else { $Script:isCoreInstalled = $false } + +#----------------------------------------------- +# CHECKING PROCESSOR ARCHITECTURE +#----------------------------------------------- + +Write-Verbose "Checking the processor architecture" + # Checking the processor architecture and operating system architecture If ( $null -ne [System.Runtime.InteropServices.RuntimeInformation]::ProcessArchitecture ) { @@ -259,11 +275,18 @@ if ($arch -match "(?i)32") { $Script:architecture = "Unknown" } + +#----------------------------------------------- +# CHECKING .NET PACKAGE RUNTIME PREFERENCE ORDER +#----------------------------------------------- + +Write-Verbose "Checking the .NET package runtime preference order" + # Check which runtimes to prefer $Script:runtimePreference = @() switch ($Script:os) { - 'Windows'{ + 'Windows' { If ($Script:architecture -eq "ARM64") { $Script:runtimePreference = @( "win-arm64", "win-arm", "win-x64" ) @@ -278,10 +301,11 @@ switch ($Script:os) { } $Script:runtimePreference += @( "win-x86" ) + $Script:runtimePreference += @( "win" ) } - 'Linux' { + 'Linux' { If ($Script:architecture -eq "ARM64") { $Script:runtimePreference = @( "linux-arm64", "linux-arm", "linux-x64" ) @@ -296,9 +320,10 @@ switch ($Script:os) { } $Script:runtimePreference += @( "linux-x86" ) + } - 'MacOS' { + 'MacOS' { If ($Script:architecture -eq "ARM64") { $Script:runtimePreference = @( "osx-arm64" ) @@ -309,31 +334,26 @@ switch ($Script:os) { } } - default { - throw "Unsupported OS: $os" + + default { + throw "Unsupported OS: $Script:os" } -} -# Check lib preference -$Script:frameworkPreference = @() -$ver = [System.Environment]::Version +} -if ( $PSVersionTable.PSEdition -eq 'Desktop' ) { - # Desktop PowerShell can load any net4x up to the installed version - $maxFramework = switch ($ver.Major) { - 4 { "net48" } # most common Windows PowerShell 5.1 runs on .NET 4.8 - default { "net48" } - } +#----------------------------------------------- +# CHECKING .NET PACKAGE REF/LIB PREFERENCE ORDER +#----------------------------------------------- - # Add net4x folders descending from the max version - $net4x = @('net48','net47','net462','net461','net45','net40') - $Script:frameworkPreference += $net4x[($net4x.IndexOf($maxFramework))..($net4x.Count-1)] +Write-Verbose "Checking the .NET package ref/lib preference order" - # Then add netstandard (2.0 is the highest fully supported on .NET 4.8) - $Script:frameworkPreference += 'netstandard2.0','netstandard1.5','netstandard1.3','netstandard1.1','netstandard1.0' +# Check lib preference +$Script:frameworkPreference = @() +$ver = [System.Environment]::Version -} else { +# If this is core, add the important framework folders first +If ( $Script:isCore -eq $True ) { # PowerShell 7+ runs on .NET 6, 7, or 8 – pick the highest available $major = $ver.Major # 6,7,8 … @@ -341,7 +361,7 @@ if ( $PSVersionTable.PSEdition -eq 'Desktop' ) { # Add the exact netX.Y folder first $Script:frameworkPreference += "net$( $major ).$( $minor )" - # Add newer “windows” variants if they exist + # Add newer "windows" variants if they exist $Script:frameworkPreference += "net$( $major ).$( $minor )-windows" # Add previous major versions @@ -350,51 +370,159 @@ if ( $PSVersionTable.PSEdition -eq 'Desktop' ) { $Script:frameworkPreference += "net$( $m ).0-windows" } - # Finally netstandard fall‑back - $Script:frameworkPreference += 'netstandard2.1','netstandard2.0','netstandard1.5','netstandard1.3','netstandard1.1','netstandard1.0' + # Finally netcore/netstandard fall-back + $Script:frameworkPreference += 'netcoreapp2.1','netcoreapp2.0','netstandard2.1','netstandard2.0','netstandard1.5','netstandard1.3','netstandard1.1','netstandard1.0' + +} + +# Then add .NET Framework folders for Desktop PowerShell, it could be a try to load them +# Desktop PowerShell can load any net4x up to the installed version +$maxFramework = switch ($ver.Major) { + 4 { "net48" } # most common Windows PowerShell 5.1 runs on .NET 4.8 + default { "net48" } } +# Add net4x folders descending from the max version +$net4x = @('net48','net471','net47','net462','net461','net45','net40') +$Script:frameworkPreference += $net4x[($net4x.IndexOf($maxFramework))..($net4x.Count-1)] + +# Just the fallback for up to .NET 4.8 +if ( $Script:powerShellEdition -eq 'Desktop' ) { + + # Then add netstandard (2.0 is the highest fully supported on .NET 4.8) + $Script:frameworkPreference += 'netstandard2.0','netstandard1.5','netstandard1.3','netstandard1.1','netstandard1.0' + +} + + +#----------------------------------------------- +# CHECKING ELEVATION +#----------------------------------------------- + +Write-Verbose "Checking Elevation" + # Check elevation -# TODO check for MacOS if ($Script:os -eq "Windows") { $identity = [Security.Principal.WindowsIdentity]::GetCurrent() $Script:executingUser = $identity.Name $principal = [Security.Principal.WindowsPrincipal]::new($identity) $Script:isElevated = $principal.IsInRole([Security.Principal.WindowsBuiltinRole]::Administrator) -} elseif ( $Script:os -eq "Linux" ) { +} elseif ( $Script:os -eq "Linux" -or $Script:os -eq "MacOS" ) { $Script:executingUser = whoami $Script:isElevated = -not [String]::IsNullOrEmpty($env:SUDO_USER) } -# Check PowerShellGet and Packagemanagement -Import-Module PowerShellGet -ErrorAction SilentlyContinue -$modules = Get-Module + +#----------------------------------------------- +# CHECKING PACKAGEMANAGEMENT AND POWERSHELLGET VERSIONS +#----------------------------------------------- + +Write-Verbose "Checking PackageManagement and PowerShellGet versions" # Check if PackageManagement and PowerShellGet are available -$modules | where-object { $_.Name -eq "PackageManagement" } | ForEach-Object { - $Script:packageManagement = $_.Version.ToString() -} -$modules | where-object { $_.Name -eq "PowerShellGet" } | ForEach-Object { - $Script:powerShellGet = $_.Version.ToString() -} +$Script:packageManagement = ( Get-Module -Name "PackageManagement" -ListAvailable -ErrorAction SilentlyContinue | Sort-Object Version -Descending | Select-Object -First 1 ).Version.toString() +$Script:powerShellGet = ( Get-Module -Name "PowerShellGet" -ListAvailable -ErrorAction SilentlyContinue | Sort-Object Version -Descending | Select-Object -First 1 ).Version.toString() + + +#----------------------------------------------- +# ADD BACKGROUND JOBS +#----------------------------------------------- + +Write-Verbose "Add background jobs to work out the installed modules and packages" -# Add jobs to find out more about installed modules and packages in the background $Script:backgroundJobs = [System.Collections.ArrayList]@() + +If ( $Script:isCoreInstalled -eq $True ) { + + [void]$Script:backgroundJobs.Add(( + Start-Job -ScriptBlock { + pwsh { [System.Environment]::Is64BitProcess } + } -Name "PwshIs64Bit" + )) + +} + [void]$Script:backgroundJobs.Add(( Start-Job -ScriptBlock { - # Use Get-InstalledModule to retrieve installed modules - Get-InstalledModule -ErrorAction SilentlyContinue - } -Name "InstalledModule" + param($ModuleRoot, $OS) + + $pathSeparator = if ($IsWindows -or $OS -match 'Windows') { ';' } else { ':' } + + $env:PSModulePath -split $pathSeparator | ForEach-Object { + $modulePath = $_ + if (Test-Path $modulePath) { + Get-ChildItem $modulePath -Filter *.psd1 -Recurse -ErrorAction SilentlyContinue | ForEach-Object { + $content = Get-Content $_.FullName -Raw + + if ($content -match "ModuleVersion\s*=\s*(['\`"])(.+?)\1") { + $version = $matches[2] + } else { + $version = 'Unknown' + } + + if ($content -match "Author\s*=\s*(['\`"])(.+?)\1") { + $author = $matches[2] + } else { + $author = 'Unknown' + } + + if ($content -match "CompanyName\s*=\s*(['\`"])(.+?)\1") { + $companyName = $matches[2] + } else { + $companyName = 'Unknown' + } + + [PSCustomObject][Ordered]@{ + Name = $_.BaseName + Version = $version + Author = $author + CompanyName = $companyName + Path = $_.DirectoryName + } + } + } + } + + } -Name "InstalledModule" -ArgumentList $PSScriptRoot.ToString(), $preCheckOs )) + [void]$Script:backgroundJobs.Add(( Start-Job -ScriptBlock { - # Use Get-InstalledModule to retrieve installed modules - PackageManagement\Get-Package -ProviderName NuGet -ErrorAction SilentlyContinue - } -Name "InstalledGlobalPackages" + param($ModuleRoot, $OS) + + Add-Type -AssemblyName System.IO.Compression.FileSystem -ErrorAction Stop + + if ($OS -eq "Windows") { + $pathsToCheck = @( + ( Join-Path $env:USERPROFILE ".nuget\packages" ) + "$( [System.Environment]::GetEnvironmentVariable("ProgramFiles") )\PackageManagement\NuGet\Packages" + "$( [System.Environment]::GetEnvironmentVariable("ProgramFiles(x86)") )\PackageManagement\NuGet\Packages" + ) + } else { + $pathsToCheck = @( + ( Join-Path $HOME ".nuget/packages" ) + ) + } + + # Dot source the needed function from ImportDependency + $importDepPath = Join-Path (Split-Path (Split-Path $ModuleRoot -Parent) -Parent) "ImportDependency/ImportDependency/Public/Get-LocalPackage.ps1" + if ( Test-Path $importDepPath ) { + . $importDepPath + $packages = Get-LocalPackage -NugetRoot $pathsToCheck + $packages + } + + } -Name "InstalledGlobalPackages" -ArgumentList $PSScriptRoot.ToString(), $preCheckOs )) +#----------------------------------------------- +# CHECKING VCREDIST +#----------------------------------------------- + +Write-Verbose "Checking VCRedist" + # Check the vcredist installation $vcredistInstalled = $False $vcredist64 = $False @@ -426,7 +554,6 @@ If ( $Script:os -eq "Windows" ) { } ) ) - } } @@ -434,13 +561,12 @@ If ( $Script:os -eq "Windows" ) { Write-Verbose "VCRedist is not installed" } - } $Script:vcredist = [PSCustomObject]@{ "installed" = $vcredistInstalled "is64bit" = $vcredist64 - "versions" = $vcRedistCollection + "versions" = $vcRedistCollection } @@ -448,6 +574,15 @@ $Script:vcredist = [PSCustomObject]@{ # MAKE PUBLIC FUNCTIONS PUBLIC #----------------------------------------------- -#Write-Verbose "Export public functions: $(($Public.Basename -join ", "))" -verbose -Export-ModuleMember -Function $Public.Basename #-verbose #+ "Set-Logfile" -#Export-ModuleMember -Function $Private.Basename #-verbose #+ "Set-Logfile" +Write-Verbose "Exporting public functions" + +Export-ModuleMember -Function $Public.Basename + + +#----------------------------------------------- +# SET THE VERBOSE PREFERENCE BACK TO THE ORIGINAL VALUE +#----------------------------------------------- + +If ( $Verbose -eq $true ) { + $VerbosePreference = $previousVerbosePreference +} diff --git a/InstallDependency/InstallDependency/Public/Install-Dependency.ps1 b/InstallDependency/InstallDependency/Public/Install-Dependency.ps1 index 4dc23c8..83b3f73 100644 --- a/InstallDependency/InstallDependency/Public/Install-Dependency.ps1 +++ b/InstallDependency/InstallDependency/Public/Install-Dependency.ps1 @@ -1,4 +1,4 @@ - + # TODO make sure to use PowerShellGet v2.2.4 or higher and PackageManagement v1.4 or higher # TODO make heavy use of ImportDependency # TODO always use -allowclobber where possible @@ -56,10 +56,12 @@ Function Install-Dependency { Array of NuGet packages to install in a subfolder of the current folder. Can be changed with parameter LocalPackageFolder. .PARAMETER LocalPackageFolder Folder name of the local package folder. Default is "lib". -.PARAMETER InstallScriptAndModuleForCurrentUser - By default, the modules and scripts will be installed for all users. If you want to install them only for the current user, then set this parameter to $true. .PARAMETER ExcludeDependencies - By default, this script is installing dependencies for every nuget package. This can be deactivated with this switch + By default, this script is installing dependencies for every nuget package. This can be deactivated with this switch. +.PARAMETER SuppressWarnings + Flag to log warnings, but not redirect to the host. +.PARAMETER KeepLogfile + Flag to keep an existing logfile rather than creating a new one. .NOTES Created by : gitfvb .LINK @@ -67,95 +69,46 @@ Function Install-Dependency { #> -[CmdletBinding()] -Param( - - [Parameter(Mandatory=$false)] - [String[]]$Script = [Array]@() - - ,[Parameter(Mandatory=$false)] - [String[]]$Module = [Array]@() - - ,[Parameter(Mandatory=$false)] - [String[]]$GlobalPackage = [Array]@() - - ,[Parameter(Mandatory=$false)] - [String[]]$LocalPackage = [Array]@() - - ,[Parameter(Mandatory=$false)] - [String]$LocalPackageFolder = "lib" - - #,[Parameter(Mandatory=$false)] - # [Switch]$InstallScriptAndModuleForCurrentUser = $false - - ,[Parameter(Mandatory=$false)] - [Switch]$ExcludeDependencies = $false - - # TODO implement - ,[Parameter(Mandatory=$false)][ValidateNotNullOrEmpty()] - [Switch]$SuppressWarnings = $false # Flag to log warnings, but not put redirect to the host - - ,[Parameter(Mandatory=$false)][ValidateNotNullOrEmpty()] - [Switch]$KeepLogfile = $false # Flag to log warnings, but not put redirect to the host - -) - - -#----------------------------------------------- -# DEBUG -#----------------------------------------------- + [CmdletBinding()] + Param( -<# -Set-Location -Path "C:\Users\Florian\Downloads\20230918" - -$Script = [Array]@() -$Module = [Array]@() -$GlobalPackage = [Array]@() -$LocalPackage = [Array]@("npgsql") -$LocalPackageFolder = "lib" -$InstallScriptAndModuleForCurrentUser = $false -$VerbosePreference = "Continue" -#> + [Parameter(Mandatory=$false)] + [String[]]$Script = [Array]@() + ,[Parameter(Mandatory=$false)] + [String[]]$Module = [Array]@() -# TODO check if we can check if this is an admin user rather than enforce it -# TODO use write log instead of write verbose? + ,[Parameter(Mandatory=$false)] + [String[]]$GlobalPackage = [Array]@() -#----------------------------------------------- -# INPUT DEFINITION -#----------------------------------------------- + ,[Parameter(Mandatory=$false)] + [String[]]$LocalPackage = [Array]@() + ,[Parameter(Mandatory=$false)] + [String]$LocalPackageFolder = "lib" -<# + ,[Parameter(Mandatory=$false)] + [Switch]$ExcludeDependencies = $false -$psScripts = @( - #"WriteLogfile" -) - -$psModules = @( - "WriteLog" - "MeasureRows" - "EncryptCredential" - "ExtendFunction" - "ConvertUnixTimestamp" - #"Microsoft.PowerShell.Utility" -) - -# Define either a simple string or provide a pscustomobject with a specific version number -$psPackages = @( - [PSCustomObject]@{ - name="Npgsql" - version = "4.1.12" - includeDependencies = $true - type = "local" # local|global - } -#> + ,[Parameter(Mandatory=$false)][ValidateNotNullOrEmpty()] + [Switch]$SuppressWarnings = $false # Flag to log warnings, but not put redirect to the host + + ,[Parameter(Mandatory=$false)][ValidateNotNullOrEmpty()] + [Switch]$KeepLogfile = $false # Flag to keep existing logfile + + ) + Begin { + # Set explicitly verbose output and remember it + If ( $SuppressWarnings -ne $true -and $PSBoundParameters["Verbose"].IsPresent -eq $true -and $PSBoundParameters["Verbose"] -eq $True) { + $originalVerbosity = $VerbosePreference + $VerbosePreference = 'Continue' + } + Write-Verbose "Proceeding with start settings" - Process { #----------------------------------------------- # START @@ -191,14 +144,14 @@ $psPackages = @( #----------------------------------------------- # Check if this is Pwsh Core - $psEnv = Get-PSEnvironment - $isCore = $psEnv.isCore - $psVersion = $psEnv.PSVersion + $psEnv = Get-PSEnvironment -SkipLocalPackageCheck + $isCore = $psEnv.IsCore Write-Log -Message "Using PowerShell version $( $psEnv.PSVersion ) and $( $psEnv.PSEdition ) edition" - + # Operating system $os = $psEnv.OS Write-Log -Message "Using OS: $( $os )" + Write-Log -Message "Using architecture: $( $psEnv.Architecture )" # Check elevation if ($os -eq "Windows") { @@ -209,12 +162,10 @@ $psPackages = @( } # Check execution policy - $executionPolicy = $psEnv.ExecutionPolicy - Write-Log -Message "Your execution policy is currently: $( $executionPolicy )" -Severity VERBOSE + Write-Log -Message "Your execution policy is currently: $( $psEnv.ExecutionPolicy.Process )" -Severity VERBOSE # Check if elevated rights are needed - #If (( $GlobalPackage.Count -gt 0 -or $Module.Count -gt 0 -or $Script.count -gt 0 ) -and $isElevated -eq $false) { - If ( $GlobalPackage.Count -gt 0 -and $isElevated -eq $false) { + If ( $GlobalPackage.Count -gt 0 -and $psEnv.IsElevated -eq $false) { throw "To install global packages, you need elevated rights, so please restart PowerShell with Administrator privileges!" } @@ -223,7 +174,7 @@ $psPackages = @( # NUGET SETTINGS #----------------------------------------------- - $packageSourceName = "NuGet v2" # otherwise you could create a local repository and put all dependencies in there. You can find more infos here: https://github.com/Apteco/HelperScripts/tree/master/functions/Log#installation-via-local-repository + $packageSourceName = "NuGet v2" $packageSourceLocation = "https://www.nuget.org/api/v2" $packageSourceProviderName = "NuGet" @@ -232,25 +183,28 @@ $psPackages = @( # POWERSHELL GALLERY SETTINGS #----------------------------------------------- - $powerShellSourceName = "PSGallery" # otherwise you could create a local repository and put all dependencies in there. You can find more infos here: https://github.com/Apteco/HelperScripts/tree/master/functions/Log#installation-via-local-repository + $powerShellSourceName = "PSGallery" $powerShellSourceLocation = "https://www.powershellgallery.com/api/v2" $powerShellSourceProviderName = "PowerShellGet" + If ( $psEnv.IsElevated -eq $true ) { - $psScope = "AllUsers" # CurrentUser|AllUsers + $psScope = "AllUsers" } else { - $psScope = "CurrentUser" # CurrentUser|AllUsers + $psScope = "CurrentUser" } Write-Log -Message "Using installation scope: $( $psScope )" -Severity VERBOSE - # TODO GOT UNTIL HERE 2025-11-18 - - - - + # Initialise counters (used across Process and reported in End) + $Script:installCount_s = 0 + $Script:installCount_m = 0 + $Script:installCount_l = 0 + $Script:installCount_g = 0 + } + Process { #----------------------------------------------- # CHECK POWERSHELL GALLERY REPOSITORY @@ -258,25 +212,23 @@ $psPackages = @( # TODO Implement version checks with [System.Version]::Parse("x.y.z") If ( $Script.Count -gt 0 -or $Module.Count -gt 0 ) { - $powershellRepo = @( Get-PackageSource -ProviderName $powerShellSourceProviderName ) #@( Get-PSRepository -ProviderName $powerShellSourceProviderName ) #@( Get-PSRepository | where { $_.SourceLocation -like "https://www.powershellgallery.com*" } ) + $powershellRepo = @( Get-PackageSource -ProviderName $powerShellSourceProviderName ) If ( $powershellRepo.Count -eq 0 ) { Write-Log "No module/script repository found! Please make sure to add a repository to your machine!" -Severity WARNING } } - # Install newer PackageManagement + # Install newer PackageManagement if needed $currentPM = get-installedmodule | where-object { $_.Name -eq "PackageManagement" } If ( $currentPM.Version -eq "1.0.0.1" -or $currentPM.Count -eq 0 ) { Write-Log "PackageManagement is outdated with v$( $currentPM.Version ). This is updating it now." -Severity WARNING - #Install-Module PackageManagement -Force -Verbose -AllowClobber Install-Package -Name PackageManagement -Force } - # Install newer PowerShellGet version when it is the default at 1.0.0.1 + # Install newer PowerShellGet if needed $currentPSGet = get-installedmodule | where-object { $_.Name -eq "PowerShellGet" } If ( $currentPSGet.Version -eq "1.0.0.1" -or $currentPSGet.Count -eq 0 ) { Write-Log "PowerShellGet is outdated with v$( $currentPSGet.Version ). This is updating it now." -Severity WARNING - #Install-Module PowerShellGet -Force -Verbose -AllowClobber Install-Package -Name PowerShellGet -Force } @@ -285,7 +237,7 @@ $psPackages = @( try { # Get PowerShellGet sources - $powershellRepo = @( Get-PackageSource -ProviderName $powerShellSourceProviderName ) #@( Get-PSRepository -ProviderName $powerShellSourceProviderName ) + $powershellRepo = @( Get-PackageSource -ProviderName $powerShellSourceProviderName ) # See if PSRepo needs to get registered If ( $powershellRepo.count -ge 1 ) { @@ -295,15 +247,12 @@ $psPackages = @( $registerPsRepoDecision = $Host.UI.PromptForChoice("", "Register $( $powerShellSourceProviderName ) as repository?", @('&Yes'; '&No'), 1) If ( $registerPsRepoDecision -eq "0" ) { - # Means yes and proceed Register-PSRepository -Name $powerShellSourceName -SourceLocation $powerShellSourceLocation - #Register-PackageSource -Name $packageSourceName -Location $packageSourceLocation -ProviderName $packageSourceProviderName # Load sources again $powershellRepo = @( Get-PSRepository -ProviderName $powerShellSourceProviderName ) } else { - # Means no and leave Write-Log "No package repository found! Please make sure to add a PowerShellGet repository to your machine!" -Severity ERROR exit 0 } @@ -327,17 +276,11 @@ $psPackages = @( } - # TODO [x] ask if you want to trust the new repository - # Do you want to trust that source? If ( $psGetSource.IsTrusted -eq $false ) { Write-Log -Message "Your source is not trusted. Do you want to trust it now?" -Severity WARNING $trustChoice = Request-Choice -title "Trust script/module Source" -message "Do you want to trust $( $psGetSource.Name )?" -choices @("Yes", "No") If ( $trustChoice -eq 1 ) { - # Use - # Set-PSRepository -Name $psGetSource.Name -InstallationPolicy Untrusted - # To get it to the untrusted status again - Set-PSRepository -Name $psGetSource.Name -InstallationPolicy Trusted } } @@ -350,26 +293,17 @@ $psPackages = @( } - # TODO [x] allow local repositories - #----------------------------------------------- # CHECK SCRIPT DEPENDENCIES FOR INSTALLATION AND UPDATE #----------------------------------------------- - $s = 0 If ( $Script.Count -gt 0 ) { - # TODO [ ] Add psgallery possibly, too - try { - #If ( $ScriptsOnly -eq $true -or ( $PackagesOnly -eq $false -and $ScriptsOnly -eq $false -and $ModulesOnly -eq $false) ) { - Write-Log "Checking Script dependencies" -Severity VERBOSE - # SCRIPTS - #$installedScripts = Get-InstalledScript $Script | ForEach-Object { $psScript = $_ @@ -378,16 +312,12 @@ $psPackages = @( $installedScripts = Get-InstalledScript - # TODO [ ] possibly add dependencies on version number - # This is using -force to allow updates - If ( $ExcludeDependencies -eq $true ) { $psScriptDependencies = Find-Script -Name $psScript } else { $psScriptDependencies = Find-Script -Name $psScript -IncludeDependencies } - #$psScriptDependencies | Where-Object { $_.Name -notin $installedScripts.Name } | Install-Script -Scope AllUsers -Verbose -Force $psScriptDependencies | ForEach-Object { $scr = $_ @@ -395,31 +325,28 @@ $psPackages = @( If ( $installedScripts.Name -contains $scr.Name ) { Write-Log -Message "Script $( $scr.Name ) is already installed" -Severity VERBOSE - $alreadyInstalledScript = $installedScripts | Where-Object { $_.Name -eq $scr.Name } #| Select -first 1 + $alreadyInstalledScript = $installedScripts | Where-Object { $_.Name -eq $scr.Name } If ( $scr.Version -gt $alreadyInstalledScript.Version ) { Write-Log -Message "Script $( $scr.Name ) is installed with an older version $( $alreadyInstalledScript.Version ) than the available version $( $scr.Version )" -Severity VERBOSE Update-Script -Name $scr.Name - $s += 1 + $Script:installCount_s += 1 } else { Write-Log -Message "No need to update $( $scr.Name )" -Severity VERBOSE } } else { Write-Log -Message "Installing Script $( $scr.Name )" -Severity VERBOSE - Install-Script -Name $scr.Name -Scope $psScope #-Force - $s += 1 + Install-Script -Name $scr.Name -Scope $psScope + $Script:installCount_s += 1 } } } - #} - } catch { Write-Log -Message "Cannot install scripts!" -Severity WARNING - #$success = $false } @@ -434,16 +361,12 @@ $psPackages = @( # CHECK MODULES DEPENDENCIES FOR INSTALLATION AND UPDATE #----------------------------------------------- - $m = 0 If ( $Module.count -gt 0 ) { try { - # PSGallery should have been added automatically yet - Write-Log "Checking Module dependencies" -Severity VERBOSE - #$installedModules = Get-InstalledModule $Module | Where-Object { $_ -notin @("PowerShellGet","PackageManagement") } | ForEach-Object { $psModule = $_ @@ -452,13 +375,12 @@ $psPackages = @( $installedModules = Get-InstalledModule - # TODO [ ] possibly add dependencies on version number - # This is using -force to allow updates If ( $ExcludeDependencies -eq $true ) { - $psModuleDependencies = Find-Module -Name $psModule #-IncludeDependencies + $psModuleDependencies = Find-Module -Name $psModule } else { $psModuleDependencies = Find-Module -Name $psModule -IncludeDependencies } + $psModuleDependencies | ForEach-Object { $mod = $_ @@ -466,23 +388,22 @@ $psPackages = @( If ( $installedModules.Name -contains $mod.Name ) { Write-Log -Message "Module $( $mod.Name ) is already installed" -Severity VERBOSE - $alreadyInstalledModule = $installedModules | Where-Object { $_.Name -eq $mod.Name } #| Select -first 1 + $alreadyInstalledModule = $installedModules | Where-Object { $_.Name -eq $mod.Name } If ( $mod.Version -gt $alreadyInstalledModule.Version ) { Write-Log -Message "Module $( $mod.Name ) is installed with an older version $( $alreadyInstalledModule.Version ) than the available version $( $mod.Version )" -Severity VERBOSE Update-Module -Name $mod.Name - $m += 1 + $Script:installCount_m += 1 } else { Write-Log -Message "No need to update $( $mod.Name )" -Severity VERBOSE } } else { Write-Log -Message "Installing Module $( $mod.Name )" -Severity VERBOSE - Install-Module -Name $mod.Name -Scope $psScope -AllowClobber #-Force - $m += 1 + Install-Module -Name $mod.Name -Scope $psScope -AllowClobber + $Script:installCount_m += 1 } } - #$psModuleDependencies | where { $_.Name -notin $installedModules.Name } | Install-Module -Scope AllUsers -Verbose -Force } @@ -490,8 +411,6 @@ $psPackages = @( Write-Log -Message "Cannot install modules!" -Severity WARNING - #Write-Error -Message $_.Exception.Message #-Severity ERROR - } } else { @@ -505,29 +424,12 @@ $psPackages = @( # CHECK PACKAGES NUGET REPOSITORY #----------------------------------------------- - <# - - If this module is not installed via nuget, then this makes sense to check again - - # Add nuget first or make sure it is set - - Register-PackageSource -Name "Nuget v2" -Location "https://www.nuget.org/api/v2" –ProviderName Nuget - - # Make nuget trusted - Set-PackageSource -Name NuGet -Trusted - - #> - - # Get-PSRepository - - #Install-Package Microsoft.Data.Sqlite.Core -RequiredVersion 7.0.0-rc.2.22472.11 - If ( $GlobalPackage.Count -gt 0 -or $LocalPackage.Count -gt 0 ) { try { # Get NuGet sources - $sources = @( Get-PackageSource -ProviderName $packageSourceProviderName ) #| where { $_.Location -like "https://www.nuget.org*" } + $sources = @( Get-PackageSource -ProviderName $packageSourceProviderName ) # See if Nuget needs to get registered If ( $sources.count -ge 1 ) { @@ -537,14 +439,12 @@ $psPackages = @( $registerNugetDecision = $Host.UI.PromptForChoice("", "Register $( $packageSourceProviderName ) as repository?", @('&Yes'; '&No'), 1) If ( $registerNugetDecision -eq "0" ) { - # Means yes and proceed Register-PackageSource -Name $packageSourceName -Location $packageSourceLocation -ProviderName $packageSourceProviderName # Load sources again - $sources = @( Get-PackageSource -ProviderName $packageSourceProviderName ) #| where { $_.Location -like "https://www.nuget.org*" } + $sources = @( Get-PackageSource -ProviderName $packageSourceProviderName ) } else { - # Means no and leave Write-Log "No package repository found! Please make sure to add a NuGet repository to your machine!" -Severity ERROR exit 0 } @@ -568,16 +468,11 @@ $psPackages = @( } - # TODO [x] ask if you want to trust the new repository - # Do you want to trust that source? If ( $packageSource.IsTrusted -eq $false ) { Write-Log -Message "Your source is not trusted. Do you want to trust it now?" -Severity WARNING $trustChoice = Request-Choice -title "Trust Package Source" -message "Do you want to trust $( $packageSource.Name )?" -choices @("Yes", "No") If ( $trustChoice -eq 1 ) { - # Use - # Set-PackageSource -Name NuGet - # To get it to the untrusted status again Set-PackageSource -Name $packageSource.Name -Trusted } } @@ -592,11 +487,9 @@ $psPackages = @( #----------------------------------------------- - # CHECK LOCAL PACKAGES DEPENDENCIES FOR INSTALLATION AND UPDATE + # CHECK LOCAL AND GLOBAL PACKAGES FOR INSTALLATION AND UPDATE #----------------------------------------------- - $l = 0 - $g = 0 If ( $LocalPackage.count -gt 0 -or $GlobalPackage.Count -gt 0) { try { @@ -620,79 +513,70 @@ $psPackages = @( $pkg = [System.Collections.ArrayList]@() If ( $GlobalPackage -contains $psPackage ) { $globalFlag = $true - } # TODO [ ] Especially test global and local installation + } Write-Log "Checking package: $( $psPackage )" -severity VERBOSE - # This is using -force to allow updates - <# - Use of continue in case of error because sometimes this happens - AUSFÜHRLICH: Total package yield:'2' for the specified package 'System.ObjectModel'. - Find-Package : Unable to find dependent package(s) (nuget:Microsoft.NETCore.Platforms/3.1.0) - #> - If ( ($psPackage.gettype()).Name -eq "PsCustomObject" ) { If ( $null -eq $psPackage.version ) { Write-Verbose "Looking for $( $psPackage.name ) without specific version." If ( $ExcludeDependencies -eq $true ) { - [void]@( Find-Package $psPackage.name -Source $packageSource.Name -ErrorAction Continue ).foreach({$pkg.add($_)}) # add elements directly instead of saving everything into a variable + [void]@( Find-Package $psPackage.name -Source $packageSource.Name -ErrorAction Continue ).foreach({$pkg.add($_)}) } else { - [void]@( Find-Package $psPackage.name -IncludeDependencies -Source $packageSource.Name -ErrorAction Continue ).foreach({$pkg.add($_)}) # add elements directly instead of saving everything into a variable + [void]@( Find-Package $psPackage.name -IncludeDependencies -Source $packageSource.Name -ErrorAction Continue ).foreach({$pkg.add($_)}) } } else { Write-Verbose "Looking for $( $psPackage.name ) with version $( $psPackage.version )" If ( $ExcludeDependencies -eq $true ) { - [void]@( Find-Package $psPackage.name -Source $packageSource.Name -ErrorAction Continue -RequiredVersion $psPackage.version ).foreach({$pkg.add($_)}) # add elements directly instead of saving everything into a variable + [void]@( Find-Package $psPackage.name -Source $packageSource.Name -ErrorAction Continue -RequiredVersion $psPackage.version ).foreach({$pkg.add($_)}) } else { - [void]@( Find-Package $psPackage.name -IncludeDependencies -Source $packageSource.Name -ErrorAction Continue -RequiredVersion $psPackage.version ).foreach({$pkg.add($_)}) # add elements directly instead of saving everything into a variable + [void]@( Find-Package $psPackage.name -IncludeDependencies -Source $packageSource.Name -ErrorAction Continue -RequiredVersion $psPackage.version ).foreach({$pkg.add($_)}) } } } else { Write-Verbose "Looking for $( $psPackage ) without specific version" If ( $ExcludeDependencies -eq $true ) { - [void]@( Find-Package $psPackage -Source $packageSource.Name -ErrorAction Continue ).foreach({$pkg.add($_)}) # add elements directly instead of saving everything into a variable + [void]@( Find-Package $psPackage -Source $packageSource.Name -ErrorAction Continue ).foreach({$pkg.add($_)}) } else { - [void]@( Find-Package $psPackage -IncludeDependencies -Source $packageSource.Name -ErrorAction Continue ).foreach({$pkg.add($_)}) # add elements directly instead of saving everything into a variable } + [void]@( Find-Package $psPackage -IncludeDependencies -Source $packageSource.Name -ErrorAction Continue ).foreach({$pkg.add($_)}) } } - $pkg | ForEach-Object { # | Where-Object { $_.Name -notin $installedPackages.Name } # | Sort-Object Name, Version -Unique -Descending + $pkg | ForEach-Object { $p = $_ $pd = [PSCustomObject]@{ "GlobalFlag" = $globalFlag - "Package" = $p - "Name" = $p.Name - "Version" = $p.Version + "Package" = $p + "Name" = $p.Name + "Version" = $p.Version } [void]$packagesToInstall.Add($pd) - } } Write-Log -Message "Done with searching for $( $packagesToInstall.Count ) packages" - # Install the packages now, we only use packages of the current repository, so in there if other repositories are used for cross-reference, this won't work at the moment $pack = $packagesToInstall | Where-Object { $_.Package.Summary -notlike "*not reference directly*" -and $_.Package.Name -notlike "Xamarin.*"} | Where-Object { $_.Package.Source -eq $packageSource.Name } | Sort-Object Name, Version -Unique -Descending Write-Log -Message "This is likely to install $( $pack.Count ) packages" - #$packagesToInstall | Where-Object { $_.Source -eq $packageSource.Name -and $_.Name -notin $installedPackages.Name } | Sort-Object Name -Unique | ForEach-Object { #where-object { $_.Source -eq $packageSource.Name } | Select-Object * -Unique | ForEach-Object { - $pack | ForEach-Object { #where-object { $_.Source -eq $packageSource.Name } | Select-Object * -Unique | ForEach-Object { + $i = 0 + $pack | ForEach-Object { $p = $_ If ( $p.GlobalFlag -eq $true ) { Write-Log -message "Installing $( $p.Package.Name ) with version $( $p.Package.version ) from $( $p.Package.Source ) globally" Install-Package -Name $p.Name -Scope $psScope -Source $packageSource.Name -RequiredVersion $p.Version -SkipDependencies -Force - $g += 1 + $Script:installCount_g += 1 } else { Write-Log -message "Installing $( $p.Name ) with version $( $p.version ) from $( $p.Package.Source ) locally" Install-Package -Name $p.Name -Scope $psScope -Source $packageSource.Name -RequiredVersion $p.Version -SkipDependencies -Force -Destination $LocalPackageFolder - $l += 1 + $Script:installCount_l += 1 } - # Write progress Write-Progress -Activity "Package installation in progress" -Status "$( [math]::Round($i/$pack.Count*100) )% Complete:" -PercentComplete ([math]::Round($i/$pack.Count*100)) + $i += 1 } @@ -708,24 +592,45 @@ $psPackages = @( } + # Reset the process ID if another module overrode it + Set-ProcessId -Id $processId + } + + + End { #----------------------------------------------- - # FINISHING + # STATUS #----------------------------------------------- - # Installation Status Write-Log -Message "STATUS:" -Severity INFO - Write-Log -Message " $( $l ) local packages installed into '$( $LocalPackageFolder )'" -Severity INFO - Write-Log -Message " $( $g ) global packages installed" -Severity INFO - Write-Log -Message " $( $m ) modules installed with scope '$( $psScope )'" -Severity INFO - Write-Log -Message " $( $s ) scripts installed with scope '$( $psScope )'" -Severity INFO + Write-Log -Message " $( $Script:installCount_l ) local packages installed into '$( $LocalPackageFolder )'" -Severity INFO + Write-Log -Message " $( $Script:installCount_g ) global packages installed" -Severity INFO + Write-Log -Message " $( $Script:installCount_m ) modules installed with scope '$( $psScope )'" -Severity INFO + Write-Log -Message " $( $Script:installCount_s ) scripts installed with scope '$( $psScope )'" -Severity INFO + + + #----------------------------------------------- + # FINISHING + #----------------------------------------------- - # Performance information $processEnd = [datetime]::now $processDuration = New-TimeSpan -Start $processStart -End $processEnd Write-Log -Message "Done! Needed $( [int]$processDuration.TotalSeconds ) seconds in total" -Severity INFO + Write-Log -Message "Logfile override: $( Get-LogfileOverride )" + + If ( $KeepLogfile -eq $false -and $null -ne $getLogfile -and '' -ne $getLogfile ) { + Write-Log -Message "Changing logfile back to '$( $currentLogfile )'" + Set-Logfile -Path $currentLogfile + } + + # Set explicitly verbose output back + If ( $SuppressWarnings -ne $true -and $PSBoundParameters["Verbose"].IsPresent -eq $true -and $PSBoundParameters["Verbose"] -eq $True) { + $VerbosePreference = $originalVerbosity + } + } -} \ No newline at end of file +} diff --git a/SqlPipeline/Tests/SqlPipeline.Tests.ps1 b/SqlPipeline/Tests/SqlPipeline.Tests.ps1 index c25835d..74f1cb3 100644 --- a/SqlPipeline/Tests/SqlPipeline.Tests.ps1 +++ b/SqlPipeline/Tests/SqlPipeline.Tests.ps1 @@ -1,5 +1,5 @@ BeforeAll { - + Write-Host "Hello World 1" Import-Module "$PSScriptRoot/../SqlPipeline" -Force -Verbose # Create a test SQLite connection @@ -46,7 +46,7 @@ Describe "Add-RowsToSql" { } It "Throws if connection is not valid" { - { + { [PSCustomObject]@{ Name = "Test" } | Add-RowsToSql -TableName "FailTable" -SQLConnectionName "notvalid" } | Should -Throw } @@ -88,4 +88,307 @@ Describe "Add-RowsToSql" { $query.Data | Should -Match '"Key":"Value"' $query.Data | Should -Match '"Num":123' } -} \ No newline at end of file +} + + +# --------------------------------------------------------------------------- +# DuckDB tests +# All tests use the in-memory DuckDB connection that is auto-initialized +# when the SqlPipeline module is imported. The entire Describe block is +# skipped when DuckDB.NET is not available in the current environment. +# --------------------------------------------------------------------------- + +BeforeDiscovery { + # Probe whether DuckDB is usable so we can skip gracefully + $script:duckDBAvailable = $false + try { + Import-Module "$PSScriptRoot/../SqlPipeline" -Force -ErrorAction Stop 2>$null + Invoke-DuckDBQuery -Query "SELECT 1" -ErrorAction Stop + $script:duckDBAvailable = $true + } catch { + $script:duckDBAvailable = $false + } +} + +Describe "Invoke-DuckDBQuery" -Skip:(-not $script:duckDBAvailable) { + + AfterEach { + Invoke-DuckDBQuery -Query "DROP TABLE IF EXISTS dq_test" -ErrorAction SilentlyContinue + } + + It "Executes a CREATE TABLE without throwing" { + { Invoke-DuckDBQuery -Query "CREATE TABLE IF NOT EXISTS dq_test (id INTEGER, val VARCHAR)" } | Should -Not -Throw + } + + It "Executes an INSERT without throwing" { + Invoke-DuckDBQuery -Query "CREATE TABLE IF NOT EXISTS dq_test (id INTEGER, val VARCHAR)" + { Invoke-DuckDBQuery -Query "INSERT INTO dq_test VALUES (1, 'hello')" } | Should -Not -Throw + } + + It "Executes a DROP TABLE without throwing" { + Invoke-DuckDBQuery -Query "CREATE TABLE IF NOT EXISTS dq_test (id INTEGER)" + { Invoke-DuckDBQuery -Query "DROP TABLE dq_test" } | Should -Not -Throw + } + +} + + +Describe "Get-DuckDBData" -Skip:(-not $script:duckDBAvailable) { + + BeforeAll { + Invoke-DuckDBQuery -Query "CREATE TABLE IF NOT EXISTS gd_test (id INTEGER, name VARCHAR)" + Invoke-DuckDBQuery -Query "INSERT INTO gd_test VALUES (1, 'Alice'), (2, 'Bob')" + } + + AfterAll { + Invoke-DuckDBQuery -Query "DROP TABLE IF EXISTS gd_test" + } + + It "Returns a DataTable" { + $result = Get-DuckDBData -Query "SELECT * FROM gd_test" + $result | Should -BeOfType [System.Data.DataTable] + } + + It "Returns the correct number of rows" { + $result = Get-DuckDBData -Query "SELECT * FROM gd_test" + $result.Rows.Count | Should -Be 2 + } + + It "Returns expected column names" { + $result = Get-DuckDBData -Query "SELECT * FROM gd_test" + $result.Columns.ColumnName | Should -Contain "id" + $result.Columns.ColumnName | Should -Contain "name" + } + + It "Returns correct values" { + $result = Get-DuckDBData -Query "SELECT name FROM gd_test ORDER BY id" + $result.Rows[0]["name"] | Should -Be "Alice" + $result.Rows[1]["name"] | Should -Be "Bob" + } + + It "Returns empty DataTable for a query with no results" { + $result = Get-DuckDBData -Query "SELECT * FROM gd_test WHERE id = 9999" + $result | Should -BeOfType [System.Data.DataTable] + $result.Rows.Count | Should -Be 0 + } + +} + + +Describe "Add-RowsToDuckDB" -Skip:(-not $script:duckDBAvailable) { + + AfterEach { + Invoke-DuckDBQuery -Query "DROP TABLE IF EXISTS ard_people" -ErrorAction SilentlyContinue + Invoke-DuckDBQuery -Query "DROP TABLE IF EXISTS ard_upsert" -ErrorAction SilentlyContinue + Invoke-DuckDBQuery -Query "DROP TABLE IF EXISTS ard_schema" -ErrorAction SilentlyContinue + Invoke-DuckDBQuery -Query "DROP TABLE IF EXISTS ard_tx" -ErrorAction SilentlyContinue + } + + It "Inserts PSCustomObject rows" { + $rows = @( + [PSCustomObject]@{ Name = "Alice"; Age = 30 } + [PSCustomObject]@{ Name = "Bob"; Age = 25 } + ) + $rows | Add-RowsToDuckDB -TableName "ard_people" + + $result = Get-DuckDBData -Query "SELECT * FROM ard_people" + $result.Rows.Count | Should -Be 2 + $result.Rows.Name | Should -Contain "Alice" + $result.Rows.Name | Should -Contain "Bob" + } + + It "Creates the table automatically on first insert" { + [PSCustomObject]@{ Id = 1; Label = "auto" } | Add-RowsToDuckDB -TableName "ard_people" + + $result = Get-DuckDBData -Query "SELECT * FROM ard_people" + $result.Rows.Count | Should -BeGreaterOrEqual 1 + } + + It "Inserts rows with -UseTransaction without throwing" { + $rows = @( + [PSCustomObject]@{ X = 1 } + [PSCustomObject]@{ X = 2 } + ) + { $rows | Add-RowsToDuckDB -TableName "ard_tx" -UseTransaction } | Should -Not -Throw + } + + It "Performs UPSERT when PKColumns are specified" { + # Insert initial row + [PSCustomObject]@{ Id = 1; Val = "original" } | Add-RowsToDuckDB -TableName "ard_upsert" -PKColumns "Id" + # Upsert same PK with updated value + [PSCustomObject]@{ Id = 1; Val = "updated" } | Add-RowsToDuckDB -TableName "ard_upsert" -PKColumns "Id" + + $result = Get-DuckDBData -Query "SELECT * FROM ard_upsert WHERE Id = 1" + $result.Rows.Count | Should -Be 1 + $result.Rows[0]["Val"] | Should -Be "updated" + } + + It "Evolves schema when new columns appear in later rows" { + [PSCustomObject]@{ Col1 = "A" } | Add-RowsToDuckDB -TableName "ard_schema" + [PSCustomObject]@{ Col1 = "B"; Col2 = "extra" } | Add-RowsToDuckDB -TableName "ard_schema" + + $result = Get-DuckDBData -Query "SELECT Col2 FROM ard_schema WHERE Col2 IS NOT NULL" + $result.Rows.Count | Should -BeGreaterOrEqual 1 + $result.Rows[0]["Col2"] | Should -Be "extra" + } + + It "Inserts multiple batches without data loss" { + $rows = 1..25 | ForEach-Object { [PSCustomObject]@{ Num = $_ } } + $rows | Add-RowsToDuckDB -TableName "ard_people" -BatchSize 10 + + $result = Get-DuckDBData -Query "SELECT COUNT(*) AS cnt FROM ard_people" + [int]$result.Rows[0]["cnt"] | Should -Be 25 + } + +} + + +Describe "Set-LoadMetadata and Get-LastLoadTimestamp" -Skip:(-not $script:duckDBAvailable) { + + AfterEach { + # Clean up metadata rows written by these tests + Invoke-DuckDBQuery -Query "DELETE FROM _load_metadata WHERE table_name LIKE 'meta_%'" -ErrorAction SilentlyContinue + } + + It "Set-LoadMetadata does not throw" { + { Set-LoadMetadata -TableName "meta_orders" -RowsLoaded 100 } | Should -Not -Throw + } + + It "Get-LastLoadTimestamp returns 2000-01-01 before any load is recorded" { + $ts = Get-LastLoadTimestamp -TableName "meta_fresh_$(Get-Random)" + $ts | Should -Be ([datetime]"2000-01-01") + } + + It "Get-LastLoadTimestamp returns the timestamp written by Set-LoadMetadata" { + Set-LoadMetadata -TableName "meta_orders" -RowsLoaded 42 -Status "success" + $ts = Get-LastLoadTimestamp -TableName "meta_orders" + $ts | Should -BeGreaterThan ([datetime]"2000-01-01") + } + + It "Set-LoadMetadata stores the correct row count" { + Set-LoadMetadata -TableName "meta_counts" -RowsLoaded 999 + $result = Get-DuckDBData -Query "SELECT rows_loaded FROM _load_metadata WHERE table_name = 'meta_counts'" + [int]$result.Rows[0]["rows_loaded"] | Should -Be 999 + } + + It "Set-LoadMetadata stores the status correctly" { + Set-LoadMetadata -TableName "meta_status" -RowsLoaded 0 -Status "error" -ErrorMessage "Test error" + $result = Get-DuckDBData -Query "SELECT status, error_msg FROM _load_metadata WHERE table_name = 'meta_status'" + $result.Rows[0]["status"] | Should -Be "error" + $result.Rows[0]["error_msg"] | Should -Be "Test error" + } + + It "Set-LoadMetadata upserts on second call for same table" { + Set-LoadMetadata -TableName "meta_upsert" -RowsLoaded 10 + Set-LoadMetadata -TableName "meta_upsert" -RowsLoaded 20 + + $result = Get-DuckDBData -Query "SELECT rows_loaded FROM _load_metadata WHERE table_name = 'meta_upsert'" + $result.Rows.Count | Should -Be 1 + [int]$result.Rows[0]["rows_loaded"] | Should -Be 20 + } + +} + + +Describe "Initialize-SQLPipeline and Close-SqlPipeline" -Skip:(-not $script:duckDBAvailable) { + + BeforeAll { + $script:dbPath = Join-Path ([System.IO.Path]::GetTempPath()) "pester_duck_$(Get-Random).db" + } + + AfterAll { + Remove-Item $script:dbPath -Force -ErrorAction SilentlyContinue + } + + It "Initialize-SQLPipeline returns a DuckDB connection object" { + $conn = Initialize-SQLPipeline -DbPath $script:dbPath + $conn | Should -Not -BeNullOrEmpty + $conn.GetType().Name | Should -Be "DuckDBConnection" + Close-SqlPipeline -Connection $conn + } + + It "Creates the database file on disk" { + $filePath = Join-Path ([System.IO.Path]::GetTempPath()) "pester_duck_file_$(Get-Random).db" + $conn = Initialize-SQLPipeline -DbPath $filePath + Test-Path $filePath | Should -Be $true + Close-SqlPipeline -Connection $conn + Remove-Item $filePath -Force -ErrorAction SilentlyContinue + } + + It "Connection state is Open after Initialize-SQLPipeline" { + $filePath = Join-Path ([System.IO.Path]::GetTempPath()) "pester_duck_open_$(Get-Random).db" + $conn = Initialize-SQLPipeline -DbPath $filePath + $conn.State | Should -Be "Open" + Close-SqlPipeline -Connection $conn + Remove-Item $filePath -Force -ErrorAction SilentlyContinue + } + + It "Close-SqlPipeline closes the connection without throwing" { + $filePath = Join-Path ([System.IO.Path]::GetTempPath()) "pester_duck_close_$(Get-Random).db" + $conn = Initialize-SQLPipeline -DbPath $filePath + { Close-SqlPipeline -Connection $conn } | Should -Not -Throw + Remove-Item $filePath -Force -ErrorAction SilentlyContinue + } + + It "File-based connection persists data across reconnect" { + $filePath = Join-Path ([System.IO.Path]::GetTempPath()) "pester_duck_persist_$(Get-Random).db" + + $conn1 = Initialize-SQLPipeline -DbPath $filePath + [PSCustomObject]@{ Id = 42; Label = "persist" } | Add-RowsToDuckDB -Connection $conn1 -TableName "persist_test" + Close-SqlPipeline -Connection $conn1 + + $conn2 = Initialize-SQLPipeline -DbPath $filePath + $result = Get-DuckDBData -Connection $conn2 -Query "SELECT * FROM persist_test" + Close-SqlPipeline -Connection $conn2 + + $result.Rows.Count | Should -Be 1 + $result.Rows[0]["Id"] | Should -Be 42 + + Remove-Item $filePath -Force -ErrorAction SilentlyContinue + } + +} + + +Describe "Export-DuckDBToParquet" -Skip:(-not $script:duckDBAvailable) { + + BeforeAll { + Invoke-DuckDBQuery -Query "CREATE TABLE IF NOT EXISTS parquet_src (id INTEGER, val VARCHAR)" + Invoke-DuckDBQuery -Query "INSERT INTO parquet_src VALUES (1,'a'), (2,'b'), (3,'c')" + $script:parquetDir = Join-Path ([System.IO.Path]::GetTempPath()) "pester_parquet_$(Get-Random)" + $script:parquetFile = Join-Path $script:parquetDir "output.parquet" + } + + AfterAll { + Invoke-DuckDBQuery -Query "DROP TABLE IF EXISTS parquet_src" + Remove-Item $script:parquetDir -Recurse -Force -ErrorAction SilentlyContinue + } + + It "Creates the output file without throwing" { + { Export-DuckDBToParquet -TableName "parquet_src" -OutputPath $script:parquetFile } | Should -Not -Throw + Test-Path $script:parquetFile | Should -Be $true + } + + It "Creates output directory automatically when it does not exist" { + $newFile = Join-Path ([System.IO.Path]::GetTempPath()) "pester_parquet_newdir_$(Get-Random)/out.parquet" + Export-DuckDBToParquet -TableName "parquet_src" -OutputPath $newFile + Test-Path $newFile | Should -Be $true + Remove-Item (Split-Path $newFile -Parent) -Recurse -Force -ErrorAction SilentlyContinue + } + + It "Accepts SNAPPY compression without throwing" { + $f = Join-Path $script:parquetDir "snappy.parquet" + { Export-DuckDBToParquet -TableName "parquet_src" -OutputPath $f -Compression SNAPPY } | Should -Not -Throw + } + + It "Accepts GZIP compression without throwing" { + $f = Join-Path $script:parquetDir "gzip.parquet" + { Export-DuckDBToParquet -TableName "parquet_src" -OutputPath $f -Compression GZIP } | Should -Not -Throw + } + + It "Re-imports the exported Parquet file via DuckDB" { + $data = Get-DuckDBData -Query "SELECT COUNT(*) AS cnt FROM read_parquet('$($script:parquetFile)')" + [int]$data.Rows[0]["cnt"] | Should -Be 3 + } + +} From 78c27ebe8f2d57b9a7bac52a1e7f5a2f4a769051 Mon Sep 17 00:00:00 2001 From: Florian Date: Mon, 30 Mar 2026 13:54:06 +0200 Subject: [PATCH 2/6] Update .gitignore Ignore packages in a lib folder --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 9291c28..56828be 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,6 @@ tests.ps1 *.db *.csv *.txt + +# Nuget packages +**/lib/** \ No newline at end of file From 44e32e3a6b75e26af61d00a62b2351fe9e0eaf12 Mon Sep 17 00:00:00 2001 From: Florian Date: Mon, 30 Mar 2026 14:28:05 +0200 Subject: [PATCH 3/6] Fixing the Pester Tests for SqlPipeline --- ...Tests.ps1 => SqlPipeline_DuckDB.Tests.ps1} | 114 +++--------------- .../Tests/SqlPipeline_SimplySQL.Tests.ps1 | 91 ++++++++++++++ 2 files changed, 109 insertions(+), 96 deletions(-) rename SqlPipeline/Tests/{SqlPipeline.Tests.ps1 => SqlPipeline_DuckDB.Tests.ps1} (76%) create mode 100644 SqlPipeline/Tests/SqlPipeline_SimplySQL.Tests.ps1 diff --git a/SqlPipeline/Tests/SqlPipeline.Tests.ps1 b/SqlPipeline/Tests/SqlPipeline_DuckDB.Tests.ps1 similarity index 76% rename from SqlPipeline/Tests/SqlPipeline.Tests.ps1 rename to SqlPipeline/Tests/SqlPipeline_DuckDB.Tests.ps1 index 74f1cb3..8e8b728 100644 --- a/SqlPipeline/Tests/SqlPipeline.Tests.ps1 +++ b/SqlPipeline/Tests/SqlPipeline_DuckDB.Tests.ps1 @@ -1,96 +1,3 @@ -BeforeAll { - - Write-Host "Hello World 1" - Import-Module "$PSScriptRoot/../SqlPipeline" -Force -Verbose - # Create a test SQLite connection - Write-Host "Hello World 2" - Open-SQLiteConnection -DataSource "$PSScriptRoot/test.db" -ConnectionName "default" -Verbose - Write-Host "Hello Worl3" - -} - -AfterAll { - # Clean up test database and close connection - Close-SqlConnection - Remove-Item "$PSScriptRoot/test.db" -ErrorAction SilentlyContinue - Remove-Module SqlPipeline -Force - Remove-Module SimplySql -Force -} - -Describe "Add-RowsToSql" { - - It "Inserts PSCustomObject rows into a new table" { - $rows = @( - [PSCustomObject]@{ Name = "Alice"; Age = 30 } - [PSCustomObject]@{ Name = "Bob"; Age = 25 } - ) - $result = $rows | Add-RowsToSql -TableName "People" -UseTransaction -Verbose - $query = Invoke-SqlQuery -Query "SELECT * FROM People" - $query.Name | Should -Contain "Alice" - $query.Name | Should -Contain "Bob" - $query.Age | Should -Contain 30 - $query.Age | Should -Contain 25 - } - - It "Inserts hashtable rows into a new table" { - $rows = @( - @{ City = "Berlin"; Country = "DE" } - @{ City = "Paris"; Country = "FR" } - ) - $result = $rows | Add-RowsToSql -TableName "Cities" -UseTransaction -Verbose - $query = Invoke-SqlQuery -Query "SELECT * FROM Cities" - $query.City | Should -Contain "Berlin" - $query.City | Should -Contain "Paris" - $query.Country | Should -Contain "DE" - $query.Country | Should -Contain "FR" - } - - It "Throws if connection is not valid" { - { - [PSCustomObject]@{ Name = "Test" } | Add-RowsToSql -TableName "FailTable" -SQLConnectionName "notvalid" - } | Should -Throw - } - - It "Throws if input is not PSCustomObject or Hashtable and validation is not ignored" { - { - "string" | Add-RowsToSql -TableName "FailTable" - } | Should -Throw - } - - It "Allows non-object input when IgnoreInputValidation is set" { - $result = "string" | Add-RowsToSql -TableName "StringTable" -IgnoreInputValidation - $query = Invoke-SqlQuery -Query "SELECT * FROM StringTable" - $query | Should -Not -BeNullOrEmpty - } - - It "Passes input object to next pipeline step when PassThru is set" { - $input = [PSCustomObject]@{ Name = "Charlie"; Age = 40 } - $output = $input | Add-RowsToSql -TableName "PassThruTable" -PassThru - $output | Should -Be $input - } - - It "Creates new columns in existing table when CreateColumnsInExistingTable is set" { - $row1 = [PSCustomObject]@{ Col1 = "A" } - $row2 = [PSCustomObject]@{ Col1 = "B"; Col2 = "Extra" } - $row1 | Add-RowsToSql -TableName "ColTest" -UseTransaction - $row2 | Add-RowsToSql -TableName "ColTest" -UseTransaction -CreateColumnsInExistingTable - $query = Invoke-SqlQuery -Query "SELECT * FROM ColTest" - $query.Col2 | Should -Contain "Extra" - } - - It "Formats nested objects as JSON when FormatObjectAsJson is set" { - $row = [PSCustomObject]@{ - Name = "JsonTest" - Data = [PSCustomObject]@{ Key = "Value"; Num = 123 } - } - $row | Add-RowsToSql -TableName "JsonTable" -FormatObjectAsJson - $query = Invoke-SqlQuery -Query "SELECT Data FROM JsonTable WHERE Name = 'JsonTest'" - $query.Data | Should -Match '"Key":"Value"' - $query.Data | Should -Match '"Num":123' - } -} - - # --------------------------------------------------------------------------- # DuckDB tests # All tests use the in-memory DuckDB connection that is auto-initialized @@ -102,12 +9,27 @@ BeforeDiscovery { # Probe whether DuckDB is usable so we can skip gracefully $script:duckDBAvailable = $false try { - Import-Module "$PSScriptRoot/../SqlPipeline" -Force -ErrorAction Stop 2>$null + Import-Module "$PSScriptRoot/../SqlPipeline" -Force #-ErrorAction Stop 2>$null + Invoke-DuckDBQuery -Query "SELECT 1" -ErrorAction Stop + $script:duckDBAvailable = $true + } catch { + $script:duckDBAvailable = $false + # Windows PowerShell 5.1 requires older pinned versions of DuckDB.NET + System.Memory + if ($PSVersionTable.PSEdition -eq 'Desktop' -or $PSVersionTable.PSVersion.Major -le 5) { + Install-SqlPipeline -WindowsPowerShell + } else { + Install-SqlPipeline + } + } + + # Try again, if still false + try { Invoke-DuckDBQuery -Query "SELECT 1" -ErrorAction Stop $script:duckDBAvailable = $true } catch { $script:duckDBAvailable = $false } + } Describe "Invoke-DuckDBQuery" -Skip:(-not $script:duckDBAvailable) { @@ -146,7 +68,7 @@ Describe "Get-DuckDBData" -Skip:(-not $script:duckDBAvailable) { It "Returns a DataTable" { $result = Get-DuckDBData -Query "SELECT * FROM gd_test" - $result | Should -BeOfType [System.Data.DataTable] + Should -ActualValue $result -BeOfType [System.Data.DataTable] } It "Returns the correct number of rows" { @@ -168,7 +90,7 @@ Describe "Get-DuckDBData" -Skip:(-not $script:duckDBAvailable) { It "Returns empty DataTable for a query with no results" { $result = Get-DuckDBData -Query "SELECT * FROM gd_test WHERE id = 9999" - $result | Should -BeOfType [System.Data.DataTable] + Should -ActualValue $result -BeOfType [System.Data.DataTable] $result.Rows.Count | Should -Be 0 } diff --git a/SqlPipeline/Tests/SqlPipeline_SimplySQL.Tests.ps1 b/SqlPipeline/Tests/SqlPipeline_SimplySQL.Tests.ps1 new file mode 100644 index 0000000..7c82899 --- /dev/null +++ b/SqlPipeline/Tests/SqlPipeline_SimplySQL.Tests.ps1 @@ -0,0 +1,91 @@ +BeforeAll { + + Write-Host "Hello World 1" + Import-Module "$PSScriptRoot/../SqlPipeline" -Force -Verbose + # Create a test SQLite connection + Write-Host "Hello World 2" + Open-SQLiteConnection -DataSource "$PSScriptRoot/test.db" -ConnectionName "default" -Verbose + Write-Host "Hello Worl3" + +} + +AfterAll { + # Clean up test database and close connection + Close-SqlConnection + Remove-Item "$PSScriptRoot/test.db" -ErrorAction SilentlyContinue + Remove-Module SqlPipeline -Force + Remove-Module SimplySql -Force +} + +Describe "Add-RowsToSql" { + + It "Inserts PSCustomObject rows into a new table" { + $rows = @( + [PSCustomObject]@{ Name = "Alice"; Age = 30 } + [PSCustomObject]@{ Name = "Bob"; Age = 25 } + ) + $result = $rows | Add-RowsToSql -TableName "People" -UseTransaction -Verbose + $query = Invoke-SqlQuery -Query "SELECT * FROM People" + $query.Name | Should -Contain "Alice" + $query.Name | Should -Contain "Bob" + $query.Age | Should -Contain 30 + $query.Age | Should -Contain 25 + } + + It "Inserts hashtable rows into a new table" { + $rows = @( + @{ City = "Berlin"; Country = "DE" } + @{ City = "Paris"; Country = "FR" } + ) + $result = $rows | Add-RowsToSql -TableName "Cities" -UseTransaction -Verbose + $query = Invoke-SqlQuery -Query "SELECT * FROM Cities" + $query.City | Should -Contain "Berlin" + $query.City | Should -Contain "Paris" + $query.Country | Should -Contain "DE" + $query.Country | Should -Contain "FR" + } + + It "Throws if connection is not valid" { + { + [PSCustomObject]@{ Name = "Test" } | Add-RowsToSql -TableName "FailTable" -SQLConnectionName "notvalid" + } | Should -Throw + } + + It "Throws if input is not PSCustomObject or Hashtable and validation is not ignored" { + { + "string" | Add-RowsToSql -TableName "FailTable" + } | Should -Throw + } + + It "Allows non-object input when IgnoreInputValidation is set" { + $result = "string" | Add-RowsToSql -TableName "StringTable" -IgnoreInputValidation + $query = Invoke-SqlQuery -Query "SELECT * FROM StringTable" + $query | Should -Not -BeNullOrEmpty + } + + It "Passes input object to next pipeline step when PassThru is set" { + $input = [PSCustomObject]@{ Name = "Charlie"; Age = 40 } + $output = $input | Add-RowsToSql -TableName "PassThruTable" -PassThru + $output | Should -Be $input + } + + It "Creates new columns in existing table when CreateColumnsInExistingTable is set" { + $row1 = [PSCustomObject]@{ Col1 = "A" } + $row2 = [PSCustomObject]@{ Col1 = "B"; Col2 = "Extra" } + $row1 | Add-RowsToSql -TableName "ColTest" -UseTransaction + $row2 | Add-RowsToSql -TableName "ColTest" -UseTransaction -CreateColumnsInExistingTable + $query = Invoke-SqlQuery -Query "SELECT * FROM ColTest" + $query.Col2 | Should -Contain "Extra" + } + + It "Formats nested objects as JSON when FormatObjectAsJson is set" { + $row = [PSCustomObject]@{ + Name = "JsonTest" + Data = [PSCustomObject]@{ Key = "Value"; Num = 123 } + } + $row | Add-RowsToSql -TableName "JsonTable" -FormatObjectAsJson + $query = Invoke-SqlQuery -Query "SELECT Data FROM JsonTable WHERE Name = 'JsonTest'" + $query.Data | Should -Match '"Key":"Value"' + $query.Data | Should -Match '"Num":123' + } +} From 4d8f178fe8987951a80ad11b4aada3fe61430101 Mon Sep 17 00:00:00 2001 From: Florian Date: Mon, 30 Mar 2026 14:29:28 +0200 Subject: [PATCH 4/6] Pushing SqlPipeline to 0.3.3 - Extending Install-SqlPipeline to install DuckDB.net 1.4.4 when using PowerShell 5.1 (latest supported version), pwsh is supporting all latest versions - Fixing to not cancel the module import if SimplySQL does not match the current processor architecture - Fixing returned values for last loaded timestamp, duckdb columns, existing table, Get-DuckDB data in general - Fixing datatype matching for DuckDB, as there is not Int32 - Fixing the fallback of a closed connection to existing in-memory connection --- .../Private/duckdb/Get-DuckDBColumns.ps1 | 5 +-- .../Private/duckdb/Test-DuckDBTableExists.ps1 | 3 +- .../Private/duckdb/Write-DuckDBAppender.ps1 | 10 +++++- .../SqlPipeline/Install-SqlPipeline.ps1 | 35 ++++++++++++++----- .../Public/duckdb/Add-RowsToDuckDB.ps1 | 2 +- .../Public/duckdb/Close-SqlPipeline.ps1 | 9 ++++- .../Public/duckdb/Get-DuckDBData.ps1 | 2 +- .../Public/duckdb/Get-LastLoadTimestamp.ps1 | 3 +- .../Public/simplysql/Add-RowsToSql.ps1 | 9 +++++ SqlPipeline/SqlPipeline/SqlPipeline.psd1 | 9 +++-- SqlPipeline/SqlPipeline/SqlPipeline.psm1 | 15 ++++++-- 11 files changed, 76 insertions(+), 26 deletions(-) diff --git a/SqlPipeline/SqlPipeline/Private/duckdb/Get-DuckDBColumns.ps1 b/SqlPipeline/SqlPipeline/Private/duckdb/Get-DuckDBColumns.ps1 index 1b647d0..4559cee 100644 --- a/SqlPipeline/SqlPipeline/Private/duckdb/Get-DuckDBColumns.ps1 +++ b/SqlPipeline/SqlPipeline/Private/duckdb/Get-DuckDBColumns.ps1 @@ -11,9 +11,6 @@ function Get-DuckDBColumns { ) $result = Get-DuckDBData -Connection $Connection -Query "DESCRIBE '$TableName'" - #return @($result.Rows | ForEach-Object { $_['column_name'] }) - - # return - $result.column_name + return @($result.Rows | ForEach-Object { $_['column_name'] }) } diff --git a/SqlPipeline/SqlPipeline/Private/duckdb/Test-DuckDBTableExists.ps1 b/SqlPipeline/SqlPipeline/Private/duckdb/Test-DuckDBTableExists.ps1 index 4978472..ac9e263 100644 --- a/SqlPipeline/SqlPipeline/Private/duckdb/Test-DuckDBTableExists.ps1 +++ b/SqlPipeline/SqlPipeline/Private/duckdb/Test-DuckDBTableExists.ps1 @@ -16,7 +16,6 @@ function Test-DuckDBTableExists { WHERE table_name = '$TableName' AND table_schema = 'main' "@ - #return ([int]$result.Rows[0]['cnt'] -gt 0) - return ([int]$result['cnt'] -gt 0) + return ([int]$result.Rows[0]['cnt'] -gt 0) } \ No newline at end of file diff --git a/SqlPipeline/SqlPipeline/Private/duckdb/Write-DuckDBAppender.ps1 b/SqlPipeline/SqlPipeline/Private/duckdb/Write-DuckDBAppender.ps1 index c04a194..1d9e949 100644 --- a/SqlPipeline/SqlPipeline/Private/duckdb/Write-DuckDBAppender.ps1 +++ b/SqlPipeline/SqlPipeline/Private/duckdb/Write-DuckDBAppender.ps1 @@ -23,6 +23,14 @@ function Write-DuckDBAppender { $appenderRow = $appender.CreateRow() foreach ($name in $propNames) { $val = $row.$name + # Normalize integer subtypes to Int64 before any other check, + # because DuckDB.NET appender has no Int32 overload and PowerShell + # would otherwise fall back to AppendValue(string). + if ($val -is [int] -or $val -is [System.Int16] -or $val -is [byte] -or $val -is [uint16] -or $val -is [uint32]) { + $val = [long]$val + } elseif ($val -is [float]) { + $val = [double]$val + } # Inlined ConvertTo-DuckDBValue if ($null -eq $val) { [void]$appenderRow.AppendValue([DBNull]::Value) @@ -37,7 +45,7 @@ function Write-DuckDBAppender { } $appenderRow.EndRow() - If ( $i % 100 -eq 0 ) { + If ( $i % 10000 -eq 0 ) { Write-Verbose "[$TableName] Appender: Row $i written." } } diff --git a/SqlPipeline/SqlPipeline/Public/SqlPipeline/Install-SqlPipeline.ps1 b/SqlPipeline/SqlPipeline/Public/SqlPipeline/Install-SqlPipeline.ps1 index 8b91bf7..e50a3d1 100644 --- a/SqlPipeline/SqlPipeline/Public/SqlPipeline/Install-SqlPipeline.ps1 +++ b/SqlPipeline/SqlPipeline/Public/SqlPipeline/Install-SqlPipeline.ps1 @@ -3,16 +3,32 @@ function Install-SqlPipeline { [CmdletBinding()] - param() + param( + [Parameter(Mandatory=$false)] + [switch]$WindowsPowerShell + ) process { Write-Verbose "Starting installation of SQLPipeline dependencies..." - If ( $Script:psPackages.Count -gt 0 ) { + # Windows PowerShell 5.1 requires specific older package versions: + # DuckDB.NET 1.4.4 (last version compatible with .NET Framework / WinPS 5.1) + # System.Memory 4.6.0 (required polyfill not included in .NET Framework) + $packagesToInstall = if ($WindowsPowerShell) { + [Array]@( + [PSCustomObject]@{ Name = "DuckDB.NET.Bindings.Full"; Version = "1.4.4" } + [PSCustomObject]@{ Name = "DuckDB.NET.Data.Full"; Version = "1.4.4" } + [PSCustomObject]@{ Name = "System.Memory"; Version = "4.6.0" } + ) + } else { + $Script:psPackages + } + + If ( $packagesToInstall.Count -gt 0 ) { $pse = Get-PSEnvironment - Write-Verbose "There are currently $($Script:psPackages.Count) packages defined as dependencies: $($Script:psPackages -join ", ")" + Write-Verbose "There are currently $($packagesToInstall.Count) packages to install." Write-Verbose "Checking for already installed packages..." Write-Verbose "Installed local packages: $( $pse.InstalledLocalPackages.Id -join ", ")" Write-Verbose "To update already installed packages, please remove them first and then run Install-SqlPipeline again." @@ -20,15 +36,16 @@ function Install-SqlPipeline { $outputDir = Join-Path -Path $PWD.Path -ChildPath "/lib" New-Item -Path $outputDir -ItemType Directory -Force | Out-Null - $psPackages | ForEach-Object { + $packagesToInstall | ForEach-Object { $pkg = $_ - $pkgName = if ( $pkg -is [string] ) { $pkg } elseif ( $pkg -is [pscustomobject] -and $pkg.Name ) { $pkg.Name } else { throw "Invalid package definition: $pkg" } - Write-Verbose "Checking if package $pkg is already installed..." + $pkgName = if ( $pkg -is [string] ) { $pkg } elseif ( $pkg -is [pscustomobject] -and $pkg.Name ) { $pkg.Name } else { throw "Invalid package definition: $pkg" } + $pkgVersion = if ( $pkg -is [pscustomobject] -and $pkg.Version ) { $pkg.Version } else { "" } + Write-Verbose "Checking if package $pkgName is already installed..." If ( -not ( $pse.InstalledLocalPackages.Id -contains $pkgName ) ) { - Write-Verbose "Package $pkg is not installed. Downloading and installing..." - Install-NuGetPackage -PackageId $pkg -OutputDir $outputDir + Write-Verbose "Package $pkgName is not installed. Downloading and installing..." + Install-NuGetPackage -PackageId $pkgName -Version $pkgVersion -OutputDir $outputDir } else { - Write-Verbose "Package $pkg is already installed. Skipping download." + Write-Verbose "Package $pkgName is already installed. Skipping download." } } diff --git a/SqlPipeline/SqlPipeline/Public/duckdb/Add-RowsToDuckDB.ps1 b/SqlPipeline/SqlPipeline/Public/duckdb/Add-RowsToDuckDB.ps1 index f0dd000..5074e15 100644 --- a/SqlPipeline/SqlPipeline/Public/duckdb/Add-RowsToDuckDB.ps1 +++ b/SqlPipeline/SqlPipeline/Public/duckdb/Add-RowsToDuckDB.ps1 @@ -118,7 +118,7 @@ function Add-RowsToDuckDB { Write-Information "[$TableName] $rowCount rows inserted via pipeline." # Force DuckDB to flush changes to disk (important for in-memory connections or when using transactions) - Invoke-DuckDBQuery -Query "FORCE CHECKPOINT" + Invoke-DuckDBQuery -Connection $Connection -Query "FORCE CHECKPOINT" } } diff --git a/SqlPipeline/SqlPipeline/Public/duckdb/Close-SqlPipeline.ps1 b/SqlPipeline/SqlPipeline/Public/duckdb/Close-SqlPipeline.ps1 index 0e0bbfe..e96ac29 100644 --- a/SqlPipeline/SqlPipeline/Public/duckdb/Close-SqlPipeline.ps1 +++ b/SqlPipeline/SqlPipeline/Public/duckdb/Close-SqlPipeline.ps1 @@ -18,5 +18,12 @@ function Close-SqlPipeline { $Connection.Close() Write-Verbose 'DuckDB connection closed.' } - + + # If the closed connection was the active default, restore the in-memory connection + # so that subsequent calls without -Connection still work. + if ([object]::ReferenceEquals($Connection, $Script:DefaultConnection)) { + $Script:DefaultConnection = $Script:InMemoryConnection + Write-Verbose 'Default connection restored to in-memory database.' + } + } \ No newline at end of file diff --git a/SqlPipeline/SqlPipeline/Public/duckdb/Get-DuckDBData.ps1 b/SqlPipeline/SqlPipeline/Public/duckdb/Get-DuckDBData.ps1 index 407638a..6759241 100644 --- a/SqlPipeline/SqlPipeline/Public/duckdb/Get-DuckDBData.ps1 +++ b/SqlPipeline/SqlPipeline/Public/duckdb/Get-DuckDBData.ps1 @@ -20,6 +20,6 @@ function Get-DuckDBData { $reader = $cmd.ExecuteReader() $table = [System.Data.DataTable]::new() $table.Load($reader) - $table + ,$table } diff --git a/SqlPipeline/SqlPipeline/Public/duckdb/Get-LastLoadTimestamp.ps1 b/SqlPipeline/SqlPipeline/Public/duckdb/Get-LastLoadTimestamp.ps1 index fa4cd5f..b289879 100644 --- a/SqlPipeline/SqlPipeline/Public/duckdb/Get-LastLoadTimestamp.ps1 +++ b/SqlPipeline/SqlPipeline/Public/duckdb/Get-LastLoadTimestamp.ps1 @@ -27,6 +27,5 @@ function Get-LastLoadTimestamp { Write-Verbose "[$TableName] No previous load found - performing full load." return [datetime]'2000-01-01' } - #return [datetime]$result.Rows[0]['last_loaded'] - return [datetime]$result['last_loaded'] + return [datetime]$result.Rows[0]['last_loaded'] } diff --git a/SqlPipeline/SqlPipeline/Public/simplysql/Add-RowsToSql.ps1 b/SqlPipeline/SqlPipeline/Public/simplysql/Add-RowsToSql.ps1 index 30e50c4..7bb8027 100644 --- a/SqlPipeline/SqlPipeline/Public/simplysql/Add-RowsToSql.ps1 +++ b/SqlPipeline/SqlPipeline/Public/simplysql/Add-RowsToSql.ps1 @@ -160,6 +160,15 @@ function Add-RowsToSql { begin { + #----------------------------------------------- + # CHECK SIMPLYSQL AVAILABILITY + #----------------------------------------------- + + if (-not $Script:isSimplySqlLoaded) { + throw "SimplySql is not loaded. This may be due to platform incompatibility (e.g. ARM architecture). Use Add-RowsToDuckDB instead." + } + + #----------------------------------------------- # INITIALISE #----------------------------------------------- diff --git a/SqlPipeline/SqlPipeline/SqlPipeline.psd1 b/SqlPipeline/SqlPipeline/SqlPipeline.psd1 index b27bb03..b1b4ad5 100644 --- a/SqlPipeline/SqlPipeline/SqlPipeline.psd1 +++ b/SqlPipeline/SqlPipeline/SqlPipeline.psd1 @@ -5,7 +5,7 @@ RootModule = 'SqlPipeline.psm1' # Die Versionsnummer dieses Moduls -ModuleVersion = '0.3.2' +ModuleVersion = '0.3.3' # Unterstützte PSEditions # CompatiblePSEditions = @() @@ -20,7 +20,7 @@ Author = 'florian.von.bracht@apteco.de' CompanyName = 'Apteco GmbH' # Urheberrechtserklärung für dieses Modul -Copyright = '(c) 2025 Apteco GmbH. All rights reserved.' +Copyright = '(c) 2026 Apteco GmbH. All rights reserved.' # Beschreibung der von diesem Modul bereitgestellten Funktionen Description = 'Apteco PS Modules - Wrapper for SimplySQL @@ -124,6 +124,11 @@ PrivateData = @{ # 'ReleaseNotes' des Moduls ReleaseNotes = ' +0.3.3 Extending Install-SqlPipeline to install DuckDB.net 1.4.4 when using PowerShell 5.1 (latest supported version), pwsh is supporting all latest versions + Fixing to not cancel the module import if SimplySQL does not match the current processor architecture + Fixing returned values for last loaded timestamp, duckdb columns, existing table, Get-DuckDB data in general + Fixing datatype matching for DuckDB, as there is not Int32 + Fixing the fallback of a closed connection to existing in-memory connection 0.3.2 Renaming Close-DuckDBConnection to Close-SqlPipeline to not clash with Close-DuckDBConnection of AptecoPSFramework (which is used for direct DuckDB connections outside of SqlPipeline) 0.3.1 Skipping comparison with complex datatypes with -SimpleTypesOnly for better import performance Adding a csv importer to export big files first into a temporary file and directly import them via DuckDB into a staging table diff --git a/SqlPipeline/SqlPipeline/SqlPipeline.psm1 b/SqlPipeline/SqlPipeline/SqlPipeline.psm1 index 6f8f26b..3ccbb8d 100644 --- a/SqlPipeline/SqlPipeline/SqlPipeline.psm1 +++ b/SqlPipeline/SqlPipeline/SqlPipeline.psm1 @@ -79,7 +79,9 @@ New-Variable -Name moduleRoot -Value $null -Scope Script -Force # Current lo New-Variable -Name PipelineBuffer -Value $null -Scope Script -Force # Buffer for the incremental load pipeline New-Variable -Name PipelineOptions -Value $null -Scope Script -Force # Options for New-Variable -Name isDuckDBLoaded -Value $null -Scope Script -Force # Flag indicating whether DuckDB.NET is available -New-Variable -Name DefaultConnection -Value $null -Scope Script -Force # Default DuckDB connection (in-memory, auto-initialized on module load) +New-Variable -Name isSimplySqlLoaded -Value $null -Scope Script -Force # Flag indicating whether SimplySql is available +New-Variable -Name DefaultConnection -Value $null -Scope Script -Force # Default DuckDB connection (in-memory, auto-initialized on module load) +New-Variable -Name InMemoryConnection -Value $null -Scope Script -Force # The auto-initialized in-memory connection; used as fallback when a file connection is closed New-Variable -Name psModules -Value $null -Scope Script -Force # Module dependencies New-Variable -Name psPackages -Value $null -Scope Script -Force # NuGet package dependencies New-Variable -Name psAssemblies -Value $null -Scope Script -Force # .NET assembly dependencies @@ -91,6 +93,7 @@ $Script:moduleRoot = $PSScriptRoot.ToString() # Internal pipeline buffer per table $Script:isDuckDBLoaded = $false +$Script:isSimplySqlLoaded = $false $Script:PipelineBuffer = [System.Collections.Generic.Dictionary[string, System.Collections.Generic.List[PSObject]]]::new() $Script:PipelineOptions = [System.Collections.Generic.Dictionary[string, hashtable]]::new() @@ -105,7 +108,12 @@ $Script:PipelineOptions = [System.Collections.Generic.Dictionary[string, hashtab # Load modules Write-Verbose "There are currently $($Script:psModules.Count) modules defined as dependencies: $($Script:psModules -join ", ")" If ( $Script:psModules.Count -gt 0 ) { - Import-Dependency -Module $psModules + try { + Import-Dependency -Module $psModules + $Script:isSimplySqlLoaded = $true + } catch { + Write-Warning "Failed to load one or more module dependencies ($($Script:psModules -join ', ')): $_. SimplySql-based functions will not be available. DuckDB functions will still work if DuckDB.NET is installed." + } } # TODO For future you need in linux maybe this module for outgrid-view, which is also supported on console only: microsoft.powershell.consoleguitools @@ -122,7 +130,8 @@ $Script:psAssemblies | ForEach-Object { # is only required when a persistent file-based database is needed. if ($Script:isDuckDBLoaded) { try { - $Script:DefaultConnection = New-DuckDBConnection -DbPath ':memory:' + $Script:DefaultConnection = New-DuckDBConnection -DbPath ':memory:' + $Script:InMemoryConnection = $Script:DefaultConnection Initialize-PipelineMetadata -Connection $Script:DefaultConnection Write-Verbose "DuckDB in-memory connection initialized automatically. Call Initialize-SQLPipeline -DbPath to switch to a file-based database." } catch { From 3e18cd2e0069a3abc3c1a3011f703dd6858e4e1d Mon Sep 17 00:00:00 2001 From: Florian Date: Mon, 30 Mar 2026 15:53:59 +0200 Subject: [PATCH 5/6] Fixing pester test for SqlPipeline to reload module after installation of dependent packages --- SqlPipeline/Tests/SqlPipeline_DuckDB.Tests.ps1 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/SqlPipeline/Tests/SqlPipeline_DuckDB.Tests.ps1 b/SqlPipeline/Tests/SqlPipeline_DuckDB.Tests.ps1 index 8e8b728..2efe527 100644 --- a/SqlPipeline/Tests/SqlPipeline_DuckDB.Tests.ps1 +++ b/SqlPipeline/Tests/SqlPipeline_DuckDB.Tests.ps1 @@ -20,6 +20,9 @@ BeforeDiscovery { } else { Install-SqlPipeline } + # Re-import so the psm1 re-runs and creates $Script:DefaultConnection + # with the newly installed packages. + Import-Module "$PSScriptRoot/../SqlPipeline" -Force } # Try again, if still false From 72581446370ef690a52ba871bc44f31b5673dfa9 Mon Sep 17 00:00:00 2001 From: Florian Date: Mon, 30 Mar 2026 15:55:23 +0200 Subject: [PATCH 6/6] Pushing SqlPipeline to 0.3.4 Fixing package installation with PowerShell 5.1 because Expand-Archive only supports *.zip files --- SqlPipeline/Readme.md | 22 +++++++++++++++++++ .../SqlPipeline/Install-NuGetPackage.ps1 | 4 +++- SqlPipeline/SqlPipeline/SqlPipeline.psd1 | 3 ++- 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/SqlPipeline/Readme.md b/SqlPipeline/Readme.md index 0b9325d..c3a6ddd 100644 --- a/SqlPipeline/Readme.md +++ b/SqlPipeline/Readme.md @@ -24,9 +24,31 @@ Current functionality: Install the required DuckDB.NET NuGet packages into a `./lib` subfolder: ```PowerShell +# PowerShell 7+ (latest DuckDB.NET) Install-SqlPipeline + +# Windows PowerShell 5.1 (pinned compatible versions) +Install-SqlPipeline -WindowsPowerShell ``` +## PowerShell Version Compatibility + +| Feature | PowerShell 7+ | Windows PowerShell 5.1 | +|---|---|---| +| DuckDB.NET version | latest | 1.4.4 (maximum) | +| Extra dependencies | none | `System.Memory` 4.6.0 | + +**Windows PowerShell 5.1** runs on .NET Framework 4.x, which is missing some APIs that newer DuckDB.NET versions require. Use the `-WindowsPowerShell` switch when installing on Windows PowerShell 5.1: + +```PowerShell +Install-SqlPipeline -WindowsPowerShell +``` + +This installs: +- `DuckDB.NET.Bindings.Full` 1.4.4 +- `DuckDB.NET.Data.Full` 1.4.4 +- `System.Memory` 4.6.0 (required polyfill not included in .NET Framework) + ## Quick Start (in-memory, no setup needed) ```PowerShell diff --git a/SqlPipeline/SqlPipeline/Private/SqlPipeline/Install-NuGetPackage.ps1 b/SqlPipeline/SqlPipeline/Private/SqlPipeline/Install-NuGetPackage.ps1 index 83d20f6..9c4c3bb 100644 --- a/SqlPipeline/SqlPipeline/Private/SqlPipeline/Install-NuGetPackage.ps1 +++ b/SqlPipeline/SqlPipeline/Private/SqlPipeline/Install-NuGetPackage.ps1 @@ -23,7 +23,9 @@ function Install-NuGetPackage { $pkgId = $PackageId.ToLower() $url = "https://api.nuget.org/v3-flatcontainer/$pkgId/$Version/$pkgId.$Version.nupkg" - $outFile = Join-Path $OutputDir "$pkgId.$Version.nupkg" + # Download as .zip so Expand-Archive accepts it on Windows PowerShell 5.1, + # which rejects any extension other than .zip even though .nupkg is identical format. + $outFile = Join-Path $OutputDir "$pkgId.$Version.zip" $unzipDir = Join-Path $OutputDir "$pkgId.$Version" Write-Verbose "Downloading $PackageId $Version ..." diff --git a/SqlPipeline/SqlPipeline/SqlPipeline.psd1 b/SqlPipeline/SqlPipeline/SqlPipeline.psd1 index b1b4ad5..dbbf88b 100644 --- a/SqlPipeline/SqlPipeline/SqlPipeline.psd1 +++ b/SqlPipeline/SqlPipeline/SqlPipeline.psd1 @@ -5,7 +5,7 @@ RootModule = 'SqlPipeline.psm1' # Die Versionsnummer dieses Moduls -ModuleVersion = '0.3.3' +ModuleVersion = '0.3.4' # Unterstützte PSEditions # CompatiblePSEditions = @() @@ -124,6 +124,7 @@ PrivateData = @{ # 'ReleaseNotes' des Moduls ReleaseNotes = ' +0.3.4 Fixing package installation with PowerShell 5.1 because Expand-Archive only supports *.zip files 0.3.3 Extending Install-SqlPipeline to install DuckDB.net 1.4.4 when using PowerShell 5.1 (latest supported version), pwsh is supporting all latest versions Fixing to not cancel the module import if SimplySQL does not match the current processor architecture Fixing returned values for last loaded timestamp, duckdb columns, existing table, Get-DuckDB data in general