diff --git a/.gitignore b/.gitignore index 9291c28..56828be 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,6 @@ tests.ps1 *.db *.csv *.txt + +# Nuget packages +**/lib/** \ No newline at end of file diff --git a/ImportDependency/Tests/ImportDependency.Tests.ps1 b/ImportDependency/Tests/ImportDependency.Tests.ps1 new file mode 100644 index 0000000..5a63914 --- /dev/null +++ b/ImportDependency/Tests/ImportDependency.Tests.ps1 @@ -0,0 +1,295 @@ +BeforeAll { + Import-Module "$PSScriptRoot/../ImportDependency" -Force +} + +AfterAll { + Remove-Module ImportDependency -Force -ErrorAction SilentlyContinue + Remove-Module WriteLog -Force -ErrorAction SilentlyContinue +} + + +# --------------------------------------------------------------------------- +Describe "Get-TemporaryPath" { +# --------------------------------------------------------------------------- + + It "Returns a non-null string" { + $result = Get-TemporaryPath + $result | Should -Not -BeNullOrEmpty + } + + It "Returns an existing path" { + $result = Get-TemporaryPath + Test-Path $result | Should -Be $true + } + + It "Returns a string type" { + $result = Get-TemporaryPath + $result | Should -BeOfType [string] + } + +} + + +# --------------------------------------------------------------------------- +Describe "Get-LocalPackage" { +# --------------------------------------------------------------------------- + + BeforeAll { + # Create a temp directory with a mock nuspec file + $script:tempPkgDir = Join-Path ([System.IO.Path]::GetTempPath()) "pester_importdep_$(Get-Random)" + New-Item -ItemType Directory -Path $script:tempPkgDir | Out-Null + + @" + + + + TestPackage + 1.2.3 + A test package for Pester + Test Author + + +"@ | Set-Content -Path (Join-Path $script:tempPkgDir "TestPackage.nuspec") -Encoding UTF8 + + # Second package in a sub-directory + $script:tempPkgSubDir = Join-Path $script:tempPkgDir "subpkg" + New-Item -ItemType Directory -Path $script:tempPkgSubDir | Out-Null + + @" + + + + SubPackage + 0.5.0 + Sub package + Sub Author + + +"@ | Set-Content -Path (Join-Path $script:tempPkgSubDir "SubPackage.nuspec") -Encoding UTF8 + } + + AfterAll { + Remove-Item -Path $script:tempPkgDir -Recurse -Force -ErrorAction SilentlyContinue + } + + It "Returns a package found via nuspec file" { + $result = Get-LocalPackage -NugetRoot $script:tempPkgDir + $result | Should -Not -BeNullOrEmpty + } + + It "Returns the correct package Id" { + $result = Get-LocalPackage -NugetRoot $script:tempPkgDir + $result.Id | Should -Contain "TestPackage" + } + + It "Returns the correct package version" { + $result = Get-LocalPackage -NugetRoot $script:tempPkgDir + ($result | Where-Object { $_.Id -eq "TestPackage" }).Version | Should -Be "1.2.3" + } + + It "Returns the correct package authors" { + $result = Get-LocalPackage -NugetRoot $script:tempPkgDir + ($result | Where-Object { $_.Id -eq "TestPackage" }).Authors | Should -Be "Test Author" + } + + It "Returns the correct package description" { + $result = Get-LocalPackage -NugetRoot $script:tempPkgDir + ($result | Where-Object { $_.Id -eq "TestPackage" }).Description | Should -Be "A test package for Pester" + } + + It "Returns packages from sub-directories" { + $result = Get-LocalPackage -NugetRoot $script:tempPkgDir + $result.Id | Should -Contain "SubPackage" + } + + It "Returns PSCustomObject with all expected properties" { + $result = Get-LocalPackage -NugetRoot $script:tempPkgDir + $pkg = $result | Select-Object -First 1 + $pkg.PSObject.Properties.Name | Should -Contain "Id" + $pkg.PSObject.Properties.Name | Should -Contain "Version" + $pkg.PSObject.Properties.Name | Should -Contain "Description" + $pkg.PSObject.Properties.Name | Should -Contain "Authors" + $pkg.PSObject.Properties.Name | Should -Contain "Path" + $pkg.PSObject.Properties.Name | Should -Contain "SizeMB" + $pkg.PSObject.Properties.Name | Should -Contain "Source" + } + + It "Source property is 'nuspec' for nuspec-based packages" { + $result = Get-LocalPackage -NugetRoot $script:tempPkgDir + ($result | Where-Object { $_.Id -eq "TestPackage" }).Source | Should -Be "nuspec" + } + + It "Returns empty result for non-existent path" { + $nonExistent = Join-Path ([System.IO.Path]::GetTempPath()) "nonexistent_pkg_$(Get-Random)" + $result = Get-LocalPackage -NugetRoot $nonExistent + $result | Should -BeNullOrEmpty + } + + It "Accepts multiple NugetRoot paths" { + $anotherDir = Join-Path ([System.IO.Path]::GetTempPath()) "pester_importdep2_$(Get-Random)" + New-Item -ItemType Directory -Path $anotherDir | Out-Null + @" + +AnotherPkg2.0.0XY +"@ | Set-Content -Path (Join-Path $anotherDir "AnotherPkg.nuspec") -Encoding UTF8 + + $result = Get-LocalPackage -NugetRoot @($script:tempPkgDir, $anotherDir) + $result.Id | Should -Contain "AnotherPkg" + + Remove-Item -Path $anotherDir -Recurse -Force -ErrorAction SilentlyContinue + } + +} + + +# --------------------------------------------------------------------------- +Describe "Get-PSEnvironment" { +# --------------------------------------------------------------------------- + + It "Returns an ordered dictionary" { + $result = Get-PSEnvironment -SkipBackgroundCheck -SkipLocalPackageCheck + $result | Should -BeOfType [System.Collections.Specialized.OrderedDictionary] + } + + It "Contains all required top-level keys" { + $result = Get-PSEnvironment -SkipBackgroundCheck -SkipLocalPackageCheck + foreach ($key in @("PSVersion","PSEdition","OS","IsCore","IsCoreInstalled","DefaultPSCore", + "Architecture","CurrentRuntime","Is64BitOS","Is64BitProcess", + "ExecutingUser","ExecutionPolicy","IsElevated", + "RuntimePreference","FrameworkPreference", + "PackageManagement","PowerShellGet","VcRedist", + "BackgroundCheckCompleted","InstalledModules","InstalledGlobalPackages", + "LocalPackageCheckCompleted","InstalledLocalPackages")) { + $result.Keys | Should -Contain $key -Because "key '$key' must be present" + } + } + + It "PSVersion is a non-empty string" { + $result = Get-PSEnvironment -SkipBackgroundCheck -SkipLocalPackageCheck + $result.PSVersion | Should -Not -BeNullOrEmpty + $result.PSVersion | Should -BeOfType [string] + } + + It "OS is one of the recognised values" { + $result = Get-PSEnvironment -SkipBackgroundCheck -SkipLocalPackageCheck + $result.OS | Should -BeIn @("Windows", "Linux", "MacOS") + } + + It "IsCore is a boolean" { + $result = Get-PSEnvironment -SkipBackgroundCheck -SkipLocalPackageCheck + $result.IsCore | Should -BeOfType [bool] + } + + It "Is64BitOS is a boolean" { + $result = Get-PSEnvironment -SkipBackgroundCheck -SkipLocalPackageCheck + $result.Is64BitOS | Should -BeOfType [bool] + } + + It "Is64BitProcess is a boolean" { + $result = Get-PSEnvironment -SkipBackgroundCheck -SkipLocalPackageCheck + $result.Is64BitProcess | Should -BeOfType [bool] + } + + It "Architecture is a recognised value" { + $result = Get-PSEnvironment -SkipBackgroundCheck -SkipLocalPackageCheck + $result.Architecture | Should -BeIn @("x64", "x32", "ARM64", "ARM", "Unknown") + } + + It "RuntimePreference is a non-empty string" { + $result = Get-PSEnvironment -SkipBackgroundCheck -SkipLocalPackageCheck + $result.RuntimePreference | Should -Not -BeNullOrEmpty + } + + It "FrameworkPreference is a non-empty string" { + $result = Get-PSEnvironment -SkipBackgroundCheck -SkipLocalPackageCheck + $result.FrameworkPreference | Should -Not -BeNullOrEmpty + } + + It "DefaultPSCore contains Version, Is64Bit, Path keys" { + $result = Get-PSEnvironment -SkipBackgroundCheck -SkipLocalPackageCheck + $result.DefaultPSCore.Keys | Should -Contain "Version" + $result.DefaultPSCore.Keys | Should -Contain "Is64Bit" + $result.DefaultPSCore.Keys | Should -Contain "Path" + } + + It "BackgroundCheckCompleted is false when -SkipBackgroundCheck is set" { + $result = Get-PSEnvironment -SkipBackgroundCheck -SkipLocalPackageCheck + $result.BackgroundCheckCompleted | Should -Be $false + } + + It "LocalPackageCheckCompleted is false when -SkipLocalPackageCheck is set" { + $result = Get-PSEnvironment -SkipBackgroundCheck -SkipLocalPackageCheck + $result.LocalPackageCheckCompleted | Should -Be $false + } + + It "BackgroundCheckCompleted is true when background check runs" { + $result = Get-PSEnvironment -SkipLocalPackageCheck + $result.BackgroundCheckCompleted | Should -Be $true + } + + It "LocalPackageCheckCompleted is true when local check runs against a missing folder" { + # Folder does not exist - check still runs (and finds nothing) + $result = Get-PSEnvironment -SkipBackgroundCheck -LocalPackageFolder (Join-Path ([System.IO.Path]::GetTempPath()) "no_pkg_$(Get-Random)") + $result.LocalPackageCheckCompleted | Should -Be $true + } + + It "InstalledLocalPackages is empty for a temp folder with no packages" { + $emptyDir = Join-Path ([System.IO.Path]::GetTempPath()) "pester_empty_$(Get-Random)" + New-Item -ItemType Directory -Path $emptyDir | Out-Null + $result = Get-PSEnvironment -SkipBackgroundCheck -LocalPackageFolder $emptyDir + $result.InstalledLocalPackages | Should -BeNullOrEmpty + Remove-Item $emptyDir -Force + } + + It "InstalledLocalPackages returns packages when a valid lib folder is supplied" { + $libDir = Join-Path ([System.IO.Path]::GetTempPath()) "pester_lib_$(Get-Random)" + New-Item -ItemType Directory -Path $libDir | Out-Null + @" + +LibPkg3.0.0XY +"@ | Set-Content -Path (Join-Path $libDir "LibPkg.nuspec") -Encoding UTF8 + + $result = Get-PSEnvironment -SkipBackgroundCheck -LocalPackageFolder $libDir + $result.InstalledLocalPackages.Id | Should -Contain "LibPkg" + + Remove-Item $libDir -Recurse -Force + } + +} + + +# --------------------------------------------------------------------------- +Describe "Import-Dependency" { +# --------------------------------------------------------------------------- + + It "Runs without errors when called with no arguments" { + { Import-Dependency } | Should -Not -Throw + } + + It "Runs without errors with SuppressWarnings when loading a non-existent module" { + { Import-Dependency -Module "NonExistentModule_$(Get-Random)" -SuppressWarnings } | Should -Not -Throw + } + + It "Runs without errors with LoadWholePackageFolder when the folder does not exist" { + $missingLib = Join-Path ([System.IO.Path]::GetTempPath()) "no_lib_$(Get-Random)" + { Import-Dependency -LoadWholePackageFolder -LocalPackageFolder $missingLib } | Should -Not -Throw + } + + It "Accepts a KeepLogfile switch without throwing" { + { Import-Dependency -KeepLogfile } | Should -Not -Throw + } + + It "Does not load excluded modules (WriteLog and ImportDependency are skipped)" { + # If WriteLog is already loaded, importing it via Import-Dependency is a no-op + # The function itself must not throw + { Import-Dependency -Module "WriteLog" } | Should -Not -Throw + } + + It "Loads a local package folder that exists but is empty without throwing" { + $emptyLib = Join-Path ([System.IO.Path]::GetTempPath()) "empty_lib_$(Get-Random)" + New-Item -ItemType Directory -Path $emptyLib | Out-Null + { Import-Dependency -LocalPackageFolder $emptyLib -LoadWholePackageFolder } | Should -Not -Throw + Remove-Item $emptyLib -Recurse -Force + } + +} diff --git a/InstallDependency/InstallDependency/InstallDependency.psm1 b/InstallDependency/InstallDependency/InstallDependency.psm1 index 05ca63a..3b92951 100644 --- a/InstallDependency/InstallDependency/InstallDependency.psm1 +++ b/InstallDependency/InstallDependency/InstallDependency.psm1 @@ -1,6 +1,24 @@ - + #----------------------------------------------- -# NOTES +#region: VERBOSE OUTPUT +#----------------------------------------------- + +param( + [bool]$Verbose = $false +) + +If ( $Verbose -eq $true ) { + $previousVerbosePreference = $VerbosePreference + $VerbosePreference = "Continue" +} else { + $VerbosePreference = "SilentlyContinue" +} + +#endregion: VERBOSE OUTPUT + + +#----------------------------------------------- +#region: NOTES #----------------------------------------------- <# @@ -12,12 +30,16 @@ https://github.com/RamblingCookieMonster/PSStackExchange/blob/db1277453374cb1668 #> +#endregion: NOTES + #----------------------------------------------- # OS CHECK #----------------------------------------------- -$preCheckisCore = ($PSVersionTable.Keys -contains "PSEdition") -and ($PSVersionTable.PSEdition -ne 'Desktop') +Write-Verbose "Checking the Core and OS" + +$preCheckisCore = $PSVersionTable.Keys -contains "PSEdition" -and $PSVersionTable.PSEdition -eq 'Core' # Check the operating system, if Core if ($preCheckisCore -eq $true) { @@ -31,8 +53,6 @@ if ($preCheckisCore -eq $true) { throw "Unknown operating system" } } else { - # [System.Environment]::OSVersion.VersionString() - # [System.Environment]::Is64BitOperatingSystem $preCheckOs = "Windows" } @@ -41,7 +61,9 @@ if ($preCheckisCore -eq $true) { # ADD MODULE PATH, IF NOT PRESENT #----------------------------------------------- -If ( $preCheckOs -eq "Windows" ) { +If ( $preCheckOs -eq "Windows" -and $preCheckisCore -eq $false ) { + + Write-Verbose "Adding Module path on Windows (when not using Core)" $modulePath = @( [System.Environment]::GetEnvironmentVariable("PSModulePath") -split ";" ) + @( "$( [System.Environment]::GetEnvironmentVariable("ProgramFiles") )\WindowsPowerShell\Modules" @@ -55,29 +77,32 @@ If ( $preCheckOs -eq "Windows" ) { $modulePath += "$( [System.Environment]::GetEnvironmentVariable("ProgramW6432") )\WindowsPowerShell\Modules" } - # Add pwsh core path - If ( $preCheckisCore -eq $true ) { - If ( [System.Environment]::GetEnvironmentVariables().keys -contains "ProgramW6432" ) { - $modulePath += "$( [System.Environment]::GetEnvironmentVariable("ProgramW6432") )\powershell\7\Modules" - } - $modulePath += "$( [System.Environment]::GetEnvironmentVariable("ProgramFiles") )\powershell\7\Modules" - $modulePath += "$( [System.Environment]::GetEnvironmentVariable("ProgramFiles(x86)") )\powershell\7\Modules" - } - # Add all paths # Using $env:PSModulePath for only temporary override $Env:PSModulePath = @( $modulePath | Sort-Object -unique ) -join ";" } +# Check if all module paths are accessible, if not remove them from the path to avoid errors when loading modules +$pathSeparator = if ($preCheckOs -eq 'Windows') { ';' } else { ':' } +$env:PSModulePath = ($env:PSModulePath -split $pathSeparator | Where-Object { + try { + [System.IO.Directory]::GetFiles($_) | Out-Null + $true + } catch { + $false + } +}) -join $pathSeparator + #----------------------------------------------- # ADD SCRIPT PATH, IF NOT PRESENT #----------------------------------------------- -If ( $preCheckOs -eq "Windows" ) { +If ( $preCheckOs -eq "Windows" -and $preCheckisCore -eq $false ) { + + Write-Verbose "Adding Script path on Windows (when not using Core)" - #$envVariables = [System.Environment]::GetEnvironmentVariables() $scriptPath = @( [System.Environment]::GetEnvironmentVariable("Path") -split ";" ) + @( "$( [System.Environment]::GetEnvironmentVariable("ProgramFiles") )\WindowsPowerShell\Scripts" "$( [System.Environment]::GetEnvironmentVariable("ProgramFiles(x86)") )\WindowsPowerShell\Scripts" @@ -89,31 +114,17 @@ If ( $preCheckOs -eq "Windows" ) { $scriptPath += "$( [System.Environment]::GetEnvironmentVariable("ProgramW6432") )\WindowsPowerShell\Scripts" } - # Add pwsh core path - If ( $preCheckisCore -eq $true ) { - If ( [System.Environment]::GetEnvironmentVariables().keys -contains "ProgramW6432" ) { - $scriptPath += "$( [System.Environment]::GetEnvironmentVariable("ProgramW6432") )\powershell\7\Scripts" - } - $scriptPath += "$( [System.Environment]::GetEnvironmentVariable("ProgramFiles") )\powershell\7\Scripts" - $scriptPath += "$( [System.Environment]::GetEnvironmentVariable("ProgramFiles(x86)") )\powershell\7\Scripts" - } - # Using $env:Path for only temporary override $Env:Path = @( $scriptPath | Sort-Object -unique ) -join ";" } -#----------------------------------------------- -# ENUMS -#----------------------------------------------- - - #----------------------------------------------- # LOAD PUBLIC AND PRIVATE FUNCTIONS #----------------------------------------------- -#$PSBoundParameters["Verbose"].IsPresent -eq $true +Write-Verbose "Loading public and private functions" $Public = @( Get-ChildItem -Path "$( $PSScriptRoot )/Public/*.ps1" -ErrorAction SilentlyContinue ) $Private = @( Get-ChildItem -Path "$( $PSScriptRoot )/Private/*.ps1" -ErrorAction SilentlyContinue ) @@ -121,7 +132,7 @@ $Private = @( Get-ChildItem -Path "$( $PSScriptRoot )/Private/*.ps1" -ErrorActio # dot source the files @( $Public + $Private ) | ForEach-Object { $import = $_ - Write-Verbose "Load function $( $import.fullname )" #-verbose + Write-Verbose "Load function $( $import.fullname )" Try { . $import.fullname } Catch { @@ -131,27 +142,38 @@ $Private = @( Get-ChildItem -Path "$( $PSScriptRoot )/Private/*.ps1" -ErrorActio #----------------------------------------------- -# READ IN CONFIG FILES AND VARIABLES +# LOAD WINDOWS SPECIFIC FUNCTIONS #----------------------------------------------- -# ... +Write-Verbose "Loading Windows specific functions" +$WindowsPrivate = @( Get-ChildItem -Path "$( $PSScriptRoot )/Private/Windows/*.ps1" -ErrorAction SilentlyContinue ) -#----------------------------------------------- -# READ IN CONFIG FILES AND VARIABLES -#----------------------------------------------- +If ( $preCheckOs -eq "Windows" ) { + @( $WindowsPrivate ) | ForEach-Object { + $import = $_ + Write-Verbose "Load function $( $import.fullname )" + Try { + . $import.fullname + } Catch { + Write-Error -Message "Failed to import function $( $import.fullname ): $( $_ )" + } + } +} #----------------------------------------------- # SET SOME VARIABLES ONLY VISIBLE TO MODULE AND FUNCTIONS #----------------------------------------------- +Write-Verbose "Define internal module variables" + # Define the variables -#New-Variable -Name execPath -Value $null -Scope Script -Force # Path of the calling script New-Variable -Name psVersion -Value $null -Scope Script -Force # PowerShell version being used New-Variable -Name psEdition -Value $null -Scope Script -Force # Edition of PowerShell (e.g., Desktop, Core) New-Variable -Name platform -Value $null -Scope Script -Force # Platform type (e.g., Windows, Linux, macOS) New-Variable -Name frameworkPreference -Value $null -Scope Script -Force # Preferred .NET framework version +New-Variable -Name runtimePreference -Value $null -Scope Script -Force # Preferred OS native framework version New-Variable -Name isCore -Value $null -Scope Script -Force # Indicates if PowerShell Core is being used (True/False) New-Variable -Name isCoreInstalled -Value $null -Scope Script -Force # Indicates if PowerShell Core is already installed (True/False) New-Variable -Name defaultPsCoreVersion -Value $null -Scope Script -Force # Default version of PowerShell Core that is used @@ -165,65 +187,59 @@ New-Variable -Name isElevated -Value $null -Scope Script -Force # In New-Variable -Name packageManagement -Value $null -Scope Script -Force # Package management system in use (e.g., NuGet, APT) New-Variable -Name powerShellGet -Value $null -Scope Script -Force # Version of PowerShellGet module New-Variable -Name vcredist -Value $null -Scope Script -Force # Indicates if Visual C++ Redistributable is installed (True/False) -New-Variable -Name installedModules -Value $null -Scope Script -Force # Caches all installed PowerShell modules -New-Variable -Name backgroundJobs -Value $null -Scope Script -Force # Hidden variable to store background jobs -New-Variable -Name installedGlobalPackages -Value $null -Scope Script -Force # Caches all installed NuGet Global Packages +New-Variable -Name installedModules -Value $null -Scope Script -Force # Caches all installed PowerShell modules +New-Variable -Name backgroundJobs -Value $null -Scope Script -Force # Hidden variable to store background jobs +New-Variable -Name installedGlobalPackages -Value $null -Scope Script -Force # Caches all installed NuGet global packages +New-Variable -Name executionPolicy -Value $null -Scope Script -Force # Current execution policy # Filling some default values $Script:isCore = $preCheckisCore $Script:os = $preCheckOs $Script:psVersion = $PSVersionTable.PSVersion.ToString() -$Script:powerShellEdition = $PSVersionTable.PSEdition +$Script:powerShellEdition = $PSVersionTable.PSEdition # Need to write that out because psedition is reserved $Script:platform = $PSVersionTable.Platform $Script:is64BitOS = [System.Environment]::Is64BitOperatingSystem $Script:is64BitProcess = [System.Environment]::Is64BitProcess +$Script:executionPolicy = [PSCustomObject]@{ + "LocalMachine" = Get-ExecutionPolicy -Scope LocalMachine + "MachinePolicy" = Get-ExecutionPolicy -Scope MachinePolicy + "Process" = Get-ExecutionPolicy -Scope Process + "CurrentUser" = Get-ExecutionPolicy -Scope CurrentUser + "UserPolicy" = Get-ExecutionPolicy -Scope UserPolicy +} -<# -$Script:frameworkPreference = @( - - # .NET 8+ (future‑proof) - 'net9.0','net8.0','net8.0-windows','net7.0','net7.0-windows', - - # .NET 6 - 'net6.0','net6.0-windows', - - # .NET 5 - 'net5.0','net5.0-windows','netcore50', - - # .NET Standard 2.1 → 2.0 → 1.5 → 1.3 → 1.1 → 1.0 - 'netstandard2.1','netstandard2.0','netstandard1.5', - 'netstandard1.3','netstandard1.1','netstandard1.0', - # Classic .NET Framework descending - 'net48','net47','net462' +#----------------------------------------------- +# CHECKING POWERSHELL CORE DETAILS +#----------------------------------------------- -) -#> +Write-Verbose "Checking more details about PS Core" # Check if pscore is installed $pwshCommand = Get-Command -commandType Application -Name "pwsh*" +$Script:defaultPsCoreVersion = $pwshCommand[0].Version If ( $pwshCommand.Count -gt 0 ) { - If ( ( pwsh { 1+1 } ) -eq 2 ) { - $Script:isCoreInstalled = $true - $Script:defaultPsCoreVersion = pwsh { $PSVersionTable.PSVersion.ToString() } - $Script:defaultPsCoreIs64Bit = pwsh { [System.Environment]::Is64BitProcess } - if ($Script:os -eq "Windows") { - # For Windows - $Script:defaultPsCorePath = ( get-command -name "pwsh*" -CommandType Application | where-object { $_.Source.replace("\pwsh.exe","") -eq ( pwsh { $pshome } ) } ).Source - } elseif ( $Script:os -eq "Linux" ) { - # For Linux - If ( $null -ne (which pwse) ) { - $Script:defaultPsCorePath = (which pwse) - } + $Script:isCoreInstalled = $true + if ($Script:os -eq "Windows") { + # For Windows + $Script:defaultPsCorePath = ( get-command -name "pwsh*" -CommandType Application | where-object { $_.Source.replace("\pwsh.exe","") -eq ( pwsh { $pshome } ) } ).Source + } elseif ( $Script:os -eq "Linux" ) { + # For Linux + If ( $null -ne (which pwsh) ) { + $Script:defaultPsCorePath = (which pwsh) } - } else { - Write-Warning "pwsh command found, but pwsh execution test failed." } - } else { $Script:isCoreInstalled = $false } + +#----------------------------------------------- +# CHECKING PROCESSOR ARCHITECTURE +#----------------------------------------------- + +Write-Verbose "Checking the processor architecture" + # Checking the processor architecture and operating system architecture If ( $null -ne [System.Runtime.InteropServices.RuntimeInformation]::ProcessArchitecture ) { @@ -259,11 +275,18 @@ if ($arch -match "(?i)32") { $Script:architecture = "Unknown" } + +#----------------------------------------------- +# CHECKING .NET PACKAGE RUNTIME PREFERENCE ORDER +#----------------------------------------------- + +Write-Verbose "Checking the .NET package runtime preference order" + # Check which runtimes to prefer $Script:runtimePreference = @() switch ($Script:os) { - 'Windows'{ + 'Windows' { If ($Script:architecture -eq "ARM64") { $Script:runtimePreference = @( "win-arm64", "win-arm", "win-x64" ) @@ -278,10 +301,11 @@ switch ($Script:os) { } $Script:runtimePreference += @( "win-x86" ) + $Script:runtimePreference += @( "win" ) } - 'Linux' { + 'Linux' { If ($Script:architecture -eq "ARM64") { $Script:runtimePreference = @( "linux-arm64", "linux-arm", "linux-x64" ) @@ -296,9 +320,10 @@ switch ($Script:os) { } $Script:runtimePreference += @( "linux-x86" ) + } - 'MacOS' { + 'MacOS' { If ($Script:architecture -eq "ARM64") { $Script:runtimePreference = @( "osx-arm64" ) @@ -309,31 +334,26 @@ switch ($Script:os) { } } - default { - throw "Unsupported OS: $os" + + default { + throw "Unsupported OS: $Script:os" } -} -# Check lib preference -$Script:frameworkPreference = @() -$ver = [System.Environment]::Version +} -if ( $PSVersionTable.PSEdition -eq 'Desktop' ) { - # Desktop PowerShell can load any net4x up to the installed version - $maxFramework = switch ($ver.Major) { - 4 { "net48" } # most common Windows PowerShell 5.1 runs on .NET 4.8 - default { "net48" } - } +#----------------------------------------------- +# CHECKING .NET PACKAGE REF/LIB PREFERENCE ORDER +#----------------------------------------------- - # Add net4x folders descending from the max version - $net4x = @('net48','net47','net462','net461','net45','net40') - $Script:frameworkPreference += $net4x[($net4x.IndexOf($maxFramework))..($net4x.Count-1)] +Write-Verbose "Checking the .NET package ref/lib preference order" - # Then add netstandard (2.0 is the highest fully supported on .NET 4.8) - $Script:frameworkPreference += 'netstandard2.0','netstandard1.5','netstandard1.3','netstandard1.1','netstandard1.0' +# Check lib preference +$Script:frameworkPreference = @() +$ver = [System.Environment]::Version -} else { +# If this is core, add the important framework folders first +If ( $Script:isCore -eq $True ) { # PowerShell 7+ runs on .NET 6, 7, or 8 – pick the highest available $major = $ver.Major # 6,7,8 … @@ -341,7 +361,7 @@ if ( $PSVersionTable.PSEdition -eq 'Desktop' ) { # Add the exact netX.Y folder first $Script:frameworkPreference += "net$( $major ).$( $minor )" - # Add newer “windows” variants if they exist + # Add newer "windows" variants if they exist $Script:frameworkPreference += "net$( $major ).$( $minor )-windows" # Add previous major versions @@ -350,51 +370,159 @@ if ( $PSVersionTable.PSEdition -eq 'Desktop' ) { $Script:frameworkPreference += "net$( $m ).0-windows" } - # Finally netstandard fall‑back - $Script:frameworkPreference += 'netstandard2.1','netstandard2.0','netstandard1.5','netstandard1.3','netstandard1.1','netstandard1.0' + # Finally netcore/netstandard fall-back + $Script:frameworkPreference += 'netcoreapp2.1','netcoreapp2.0','netstandard2.1','netstandard2.0','netstandard1.5','netstandard1.3','netstandard1.1','netstandard1.0' + +} + +# Then add .NET Framework folders for Desktop PowerShell, it could be a try to load them +# Desktop PowerShell can load any net4x up to the installed version +$maxFramework = switch ($ver.Major) { + 4 { "net48" } # most common Windows PowerShell 5.1 runs on .NET 4.8 + default { "net48" } } +# Add net4x folders descending from the max version +$net4x = @('net48','net471','net47','net462','net461','net45','net40') +$Script:frameworkPreference += $net4x[($net4x.IndexOf($maxFramework))..($net4x.Count-1)] + +# Just the fallback for up to .NET 4.8 +if ( $Script:powerShellEdition -eq 'Desktop' ) { + + # Then add netstandard (2.0 is the highest fully supported on .NET 4.8) + $Script:frameworkPreference += 'netstandard2.0','netstandard1.5','netstandard1.3','netstandard1.1','netstandard1.0' + +} + + +#----------------------------------------------- +# CHECKING ELEVATION +#----------------------------------------------- + +Write-Verbose "Checking Elevation" + # Check elevation -# TODO check for MacOS if ($Script:os -eq "Windows") { $identity = [Security.Principal.WindowsIdentity]::GetCurrent() $Script:executingUser = $identity.Name $principal = [Security.Principal.WindowsPrincipal]::new($identity) $Script:isElevated = $principal.IsInRole([Security.Principal.WindowsBuiltinRole]::Administrator) -} elseif ( $Script:os -eq "Linux" ) { +} elseif ( $Script:os -eq "Linux" -or $Script:os -eq "MacOS" ) { $Script:executingUser = whoami $Script:isElevated = -not [String]::IsNullOrEmpty($env:SUDO_USER) } -# Check PowerShellGet and Packagemanagement -Import-Module PowerShellGet -ErrorAction SilentlyContinue -$modules = Get-Module + +#----------------------------------------------- +# CHECKING PACKAGEMANAGEMENT AND POWERSHELLGET VERSIONS +#----------------------------------------------- + +Write-Verbose "Checking PackageManagement and PowerShellGet versions" # Check if PackageManagement and PowerShellGet are available -$modules | where-object { $_.Name -eq "PackageManagement" } | ForEach-Object { - $Script:packageManagement = $_.Version.ToString() -} -$modules | where-object { $_.Name -eq "PowerShellGet" } | ForEach-Object { - $Script:powerShellGet = $_.Version.ToString() -} +$Script:packageManagement = ( Get-Module -Name "PackageManagement" -ListAvailable -ErrorAction SilentlyContinue | Sort-Object Version -Descending | Select-Object -First 1 ).Version.toString() +$Script:powerShellGet = ( Get-Module -Name "PowerShellGet" -ListAvailable -ErrorAction SilentlyContinue | Sort-Object Version -Descending | Select-Object -First 1 ).Version.toString() + + +#----------------------------------------------- +# ADD BACKGROUND JOBS +#----------------------------------------------- + +Write-Verbose "Add background jobs to work out the installed modules and packages" -# Add jobs to find out more about installed modules and packages in the background $Script:backgroundJobs = [System.Collections.ArrayList]@() + +If ( $Script:isCoreInstalled -eq $True ) { + + [void]$Script:backgroundJobs.Add(( + Start-Job -ScriptBlock { + pwsh { [System.Environment]::Is64BitProcess } + } -Name "PwshIs64Bit" + )) + +} + [void]$Script:backgroundJobs.Add(( Start-Job -ScriptBlock { - # Use Get-InstalledModule to retrieve installed modules - Get-InstalledModule -ErrorAction SilentlyContinue - } -Name "InstalledModule" + param($ModuleRoot, $OS) + + $pathSeparator = if ($IsWindows -or $OS -match 'Windows') { ';' } else { ':' } + + $env:PSModulePath -split $pathSeparator | ForEach-Object { + $modulePath = $_ + if (Test-Path $modulePath) { + Get-ChildItem $modulePath -Filter *.psd1 -Recurse -ErrorAction SilentlyContinue | ForEach-Object { + $content = Get-Content $_.FullName -Raw + + if ($content -match "ModuleVersion\s*=\s*(['\`"])(.+?)\1") { + $version = $matches[2] + } else { + $version = 'Unknown' + } + + if ($content -match "Author\s*=\s*(['\`"])(.+?)\1") { + $author = $matches[2] + } else { + $author = 'Unknown' + } + + if ($content -match "CompanyName\s*=\s*(['\`"])(.+?)\1") { + $companyName = $matches[2] + } else { + $companyName = 'Unknown' + } + + [PSCustomObject][Ordered]@{ + Name = $_.BaseName + Version = $version + Author = $author + CompanyName = $companyName + Path = $_.DirectoryName + } + } + } + } + + } -Name "InstalledModule" -ArgumentList $PSScriptRoot.ToString(), $preCheckOs )) + [void]$Script:backgroundJobs.Add(( Start-Job -ScriptBlock { - # Use Get-InstalledModule to retrieve installed modules - PackageManagement\Get-Package -ProviderName NuGet -ErrorAction SilentlyContinue - } -Name "InstalledGlobalPackages" + param($ModuleRoot, $OS) + + Add-Type -AssemblyName System.IO.Compression.FileSystem -ErrorAction Stop + + if ($OS -eq "Windows") { + $pathsToCheck = @( + ( Join-Path $env:USERPROFILE ".nuget\packages" ) + "$( [System.Environment]::GetEnvironmentVariable("ProgramFiles") )\PackageManagement\NuGet\Packages" + "$( [System.Environment]::GetEnvironmentVariable("ProgramFiles(x86)") )\PackageManagement\NuGet\Packages" + ) + } else { + $pathsToCheck = @( + ( Join-Path $HOME ".nuget/packages" ) + ) + } + + # Dot source the needed function from ImportDependency + $importDepPath = Join-Path (Split-Path (Split-Path $ModuleRoot -Parent) -Parent) "ImportDependency/ImportDependency/Public/Get-LocalPackage.ps1" + if ( Test-Path $importDepPath ) { + . $importDepPath + $packages = Get-LocalPackage -NugetRoot $pathsToCheck + $packages + } + + } -Name "InstalledGlobalPackages" -ArgumentList $PSScriptRoot.ToString(), $preCheckOs )) +#----------------------------------------------- +# CHECKING VCREDIST +#----------------------------------------------- + +Write-Verbose "Checking VCRedist" + # Check the vcredist installation $vcredistInstalled = $False $vcredist64 = $False @@ -426,7 +554,6 @@ If ( $Script:os -eq "Windows" ) { } ) ) - } } @@ -434,13 +561,12 @@ If ( $Script:os -eq "Windows" ) { Write-Verbose "VCRedist is not installed" } - } $Script:vcredist = [PSCustomObject]@{ "installed" = $vcredistInstalled "is64bit" = $vcredist64 - "versions" = $vcRedistCollection + "versions" = $vcRedistCollection } @@ -448,6 +574,15 @@ $Script:vcredist = [PSCustomObject]@{ # MAKE PUBLIC FUNCTIONS PUBLIC #----------------------------------------------- -#Write-Verbose "Export public functions: $(($Public.Basename -join ", "))" -verbose -Export-ModuleMember -Function $Public.Basename #-verbose #+ "Set-Logfile" -#Export-ModuleMember -Function $Private.Basename #-verbose #+ "Set-Logfile" +Write-Verbose "Exporting public functions" + +Export-ModuleMember -Function $Public.Basename + + +#----------------------------------------------- +# SET THE VERBOSE PREFERENCE BACK TO THE ORIGINAL VALUE +#----------------------------------------------- + +If ( $Verbose -eq $true ) { + $VerbosePreference = $previousVerbosePreference +} diff --git a/InstallDependency/InstallDependency/Public/Install-Dependency.ps1 b/InstallDependency/InstallDependency/Public/Install-Dependency.ps1 index 4dc23c8..83b3f73 100644 --- a/InstallDependency/InstallDependency/Public/Install-Dependency.ps1 +++ b/InstallDependency/InstallDependency/Public/Install-Dependency.ps1 @@ -1,4 +1,4 @@ - + # TODO make sure to use PowerShellGet v2.2.4 or higher and PackageManagement v1.4 or higher # TODO make heavy use of ImportDependency # TODO always use -allowclobber where possible @@ -56,10 +56,12 @@ Function Install-Dependency { Array of NuGet packages to install in a subfolder of the current folder. Can be changed with parameter LocalPackageFolder. .PARAMETER LocalPackageFolder Folder name of the local package folder. Default is "lib". -.PARAMETER InstallScriptAndModuleForCurrentUser - By default, the modules and scripts will be installed for all users. If you want to install them only for the current user, then set this parameter to $true. .PARAMETER ExcludeDependencies - By default, this script is installing dependencies for every nuget package. This can be deactivated with this switch + By default, this script is installing dependencies for every nuget package. This can be deactivated with this switch. +.PARAMETER SuppressWarnings + Flag to log warnings, but not redirect to the host. +.PARAMETER KeepLogfile + Flag to keep an existing logfile rather than creating a new one. .NOTES Created by : gitfvb .LINK @@ -67,95 +69,46 @@ Function Install-Dependency { #> -[CmdletBinding()] -Param( - - [Parameter(Mandatory=$false)] - [String[]]$Script = [Array]@() - - ,[Parameter(Mandatory=$false)] - [String[]]$Module = [Array]@() - - ,[Parameter(Mandatory=$false)] - [String[]]$GlobalPackage = [Array]@() - - ,[Parameter(Mandatory=$false)] - [String[]]$LocalPackage = [Array]@() - - ,[Parameter(Mandatory=$false)] - [String]$LocalPackageFolder = "lib" - - #,[Parameter(Mandatory=$false)] - # [Switch]$InstallScriptAndModuleForCurrentUser = $false - - ,[Parameter(Mandatory=$false)] - [Switch]$ExcludeDependencies = $false - - # TODO implement - ,[Parameter(Mandatory=$false)][ValidateNotNullOrEmpty()] - [Switch]$SuppressWarnings = $false # Flag to log warnings, but not put redirect to the host - - ,[Parameter(Mandatory=$false)][ValidateNotNullOrEmpty()] - [Switch]$KeepLogfile = $false # Flag to log warnings, but not put redirect to the host - -) - - -#----------------------------------------------- -# DEBUG -#----------------------------------------------- + [CmdletBinding()] + Param( -<# -Set-Location -Path "C:\Users\Florian\Downloads\20230918" - -$Script = [Array]@() -$Module = [Array]@() -$GlobalPackage = [Array]@() -$LocalPackage = [Array]@("npgsql") -$LocalPackageFolder = "lib" -$InstallScriptAndModuleForCurrentUser = $false -$VerbosePreference = "Continue" -#> + [Parameter(Mandatory=$false)] + [String[]]$Script = [Array]@() + ,[Parameter(Mandatory=$false)] + [String[]]$Module = [Array]@() -# TODO check if we can check if this is an admin user rather than enforce it -# TODO use write log instead of write verbose? + ,[Parameter(Mandatory=$false)] + [String[]]$GlobalPackage = [Array]@() -#----------------------------------------------- -# INPUT DEFINITION -#----------------------------------------------- + ,[Parameter(Mandatory=$false)] + [String[]]$LocalPackage = [Array]@() + ,[Parameter(Mandatory=$false)] + [String]$LocalPackageFolder = "lib" -<# + ,[Parameter(Mandatory=$false)] + [Switch]$ExcludeDependencies = $false -$psScripts = @( - #"WriteLogfile" -) - -$psModules = @( - "WriteLog" - "MeasureRows" - "EncryptCredential" - "ExtendFunction" - "ConvertUnixTimestamp" - #"Microsoft.PowerShell.Utility" -) - -# Define either a simple string or provide a pscustomobject with a specific version number -$psPackages = @( - [PSCustomObject]@{ - name="Npgsql" - version = "4.1.12" - includeDependencies = $true - type = "local" # local|global - } -#> + ,[Parameter(Mandatory=$false)][ValidateNotNullOrEmpty()] + [Switch]$SuppressWarnings = $false # Flag to log warnings, but not put redirect to the host + + ,[Parameter(Mandatory=$false)][ValidateNotNullOrEmpty()] + [Switch]$KeepLogfile = $false # Flag to keep existing logfile + + ) + Begin { + # Set explicitly verbose output and remember it + If ( $SuppressWarnings -ne $true -and $PSBoundParameters["Verbose"].IsPresent -eq $true -and $PSBoundParameters["Verbose"] -eq $True) { + $originalVerbosity = $VerbosePreference + $VerbosePreference = 'Continue' + } + Write-Verbose "Proceeding with start settings" - Process { #----------------------------------------------- # START @@ -191,14 +144,14 @@ $psPackages = @( #----------------------------------------------- # Check if this is Pwsh Core - $psEnv = Get-PSEnvironment - $isCore = $psEnv.isCore - $psVersion = $psEnv.PSVersion + $psEnv = Get-PSEnvironment -SkipLocalPackageCheck + $isCore = $psEnv.IsCore Write-Log -Message "Using PowerShell version $( $psEnv.PSVersion ) and $( $psEnv.PSEdition ) edition" - + # Operating system $os = $psEnv.OS Write-Log -Message "Using OS: $( $os )" + Write-Log -Message "Using architecture: $( $psEnv.Architecture )" # Check elevation if ($os -eq "Windows") { @@ -209,12 +162,10 @@ $psPackages = @( } # Check execution policy - $executionPolicy = $psEnv.ExecutionPolicy - Write-Log -Message "Your execution policy is currently: $( $executionPolicy )" -Severity VERBOSE + Write-Log -Message "Your execution policy is currently: $( $psEnv.ExecutionPolicy.Process )" -Severity VERBOSE # Check if elevated rights are needed - #If (( $GlobalPackage.Count -gt 0 -or $Module.Count -gt 0 -or $Script.count -gt 0 ) -and $isElevated -eq $false) { - If ( $GlobalPackage.Count -gt 0 -and $isElevated -eq $false) { + If ( $GlobalPackage.Count -gt 0 -and $psEnv.IsElevated -eq $false) { throw "To install global packages, you need elevated rights, so please restart PowerShell with Administrator privileges!" } @@ -223,7 +174,7 @@ $psPackages = @( # NUGET SETTINGS #----------------------------------------------- - $packageSourceName = "NuGet v2" # otherwise you could create a local repository and put all dependencies in there. You can find more infos here: https://github.com/Apteco/HelperScripts/tree/master/functions/Log#installation-via-local-repository + $packageSourceName = "NuGet v2" $packageSourceLocation = "https://www.nuget.org/api/v2" $packageSourceProviderName = "NuGet" @@ -232,25 +183,28 @@ $psPackages = @( # POWERSHELL GALLERY SETTINGS #----------------------------------------------- - $powerShellSourceName = "PSGallery" # otherwise you could create a local repository and put all dependencies in there. You can find more infos here: https://github.com/Apteco/HelperScripts/tree/master/functions/Log#installation-via-local-repository + $powerShellSourceName = "PSGallery" $powerShellSourceLocation = "https://www.powershellgallery.com/api/v2" $powerShellSourceProviderName = "PowerShellGet" + If ( $psEnv.IsElevated -eq $true ) { - $psScope = "AllUsers" # CurrentUser|AllUsers + $psScope = "AllUsers" } else { - $psScope = "CurrentUser" # CurrentUser|AllUsers + $psScope = "CurrentUser" } Write-Log -Message "Using installation scope: $( $psScope )" -Severity VERBOSE - # TODO GOT UNTIL HERE 2025-11-18 - - - - + # Initialise counters (used across Process and reported in End) + $Script:installCount_s = 0 + $Script:installCount_m = 0 + $Script:installCount_l = 0 + $Script:installCount_g = 0 + } + Process { #----------------------------------------------- # CHECK POWERSHELL GALLERY REPOSITORY @@ -258,25 +212,23 @@ $psPackages = @( # TODO Implement version checks with [System.Version]::Parse("x.y.z") If ( $Script.Count -gt 0 -or $Module.Count -gt 0 ) { - $powershellRepo = @( Get-PackageSource -ProviderName $powerShellSourceProviderName ) #@( Get-PSRepository -ProviderName $powerShellSourceProviderName ) #@( Get-PSRepository | where { $_.SourceLocation -like "https://www.powershellgallery.com*" } ) + $powershellRepo = @( Get-PackageSource -ProviderName $powerShellSourceProviderName ) If ( $powershellRepo.Count -eq 0 ) { Write-Log "No module/script repository found! Please make sure to add a repository to your machine!" -Severity WARNING } } - # Install newer PackageManagement + # Install newer PackageManagement if needed $currentPM = get-installedmodule | where-object { $_.Name -eq "PackageManagement" } If ( $currentPM.Version -eq "1.0.0.1" -or $currentPM.Count -eq 0 ) { Write-Log "PackageManagement is outdated with v$( $currentPM.Version ). This is updating it now." -Severity WARNING - #Install-Module PackageManagement -Force -Verbose -AllowClobber Install-Package -Name PackageManagement -Force } - # Install newer PowerShellGet version when it is the default at 1.0.0.1 + # Install newer PowerShellGet if needed $currentPSGet = get-installedmodule | where-object { $_.Name -eq "PowerShellGet" } If ( $currentPSGet.Version -eq "1.0.0.1" -or $currentPSGet.Count -eq 0 ) { Write-Log "PowerShellGet is outdated with v$( $currentPSGet.Version ). This is updating it now." -Severity WARNING - #Install-Module PowerShellGet -Force -Verbose -AllowClobber Install-Package -Name PowerShellGet -Force } @@ -285,7 +237,7 @@ $psPackages = @( try { # Get PowerShellGet sources - $powershellRepo = @( Get-PackageSource -ProviderName $powerShellSourceProviderName ) #@( Get-PSRepository -ProviderName $powerShellSourceProviderName ) + $powershellRepo = @( Get-PackageSource -ProviderName $powerShellSourceProviderName ) # See if PSRepo needs to get registered If ( $powershellRepo.count -ge 1 ) { @@ -295,15 +247,12 @@ $psPackages = @( $registerPsRepoDecision = $Host.UI.PromptForChoice("", "Register $( $powerShellSourceProviderName ) as repository?", @('&Yes'; '&No'), 1) If ( $registerPsRepoDecision -eq "0" ) { - # Means yes and proceed Register-PSRepository -Name $powerShellSourceName -SourceLocation $powerShellSourceLocation - #Register-PackageSource -Name $packageSourceName -Location $packageSourceLocation -ProviderName $packageSourceProviderName # Load sources again $powershellRepo = @( Get-PSRepository -ProviderName $powerShellSourceProviderName ) } else { - # Means no and leave Write-Log "No package repository found! Please make sure to add a PowerShellGet repository to your machine!" -Severity ERROR exit 0 } @@ -327,17 +276,11 @@ $psPackages = @( } - # TODO [x] ask if you want to trust the new repository - # Do you want to trust that source? If ( $psGetSource.IsTrusted -eq $false ) { Write-Log -Message "Your source is not trusted. Do you want to trust it now?" -Severity WARNING $trustChoice = Request-Choice -title "Trust script/module Source" -message "Do you want to trust $( $psGetSource.Name )?" -choices @("Yes", "No") If ( $trustChoice -eq 1 ) { - # Use - # Set-PSRepository -Name $psGetSource.Name -InstallationPolicy Untrusted - # To get it to the untrusted status again - Set-PSRepository -Name $psGetSource.Name -InstallationPolicy Trusted } } @@ -350,26 +293,17 @@ $psPackages = @( } - # TODO [x] allow local repositories - #----------------------------------------------- # CHECK SCRIPT DEPENDENCIES FOR INSTALLATION AND UPDATE #----------------------------------------------- - $s = 0 If ( $Script.Count -gt 0 ) { - # TODO [ ] Add psgallery possibly, too - try { - #If ( $ScriptsOnly -eq $true -or ( $PackagesOnly -eq $false -and $ScriptsOnly -eq $false -and $ModulesOnly -eq $false) ) { - Write-Log "Checking Script dependencies" -Severity VERBOSE - # SCRIPTS - #$installedScripts = Get-InstalledScript $Script | ForEach-Object { $psScript = $_ @@ -378,16 +312,12 @@ $psPackages = @( $installedScripts = Get-InstalledScript - # TODO [ ] possibly add dependencies on version number - # This is using -force to allow updates - If ( $ExcludeDependencies -eq $true ) { $psScriptDependencies = Find-Script -Name $psScript } else { $psScriptDependencies = Find-Script -Name $psScript -IncludeDependencies } - #$psScriptDependencies | Where-Object { $_.Name -notin $installedScripts.Name } | Install-Script -Scope AllUsers -Verbose -Force $psScriptDependencies | ForEach-Object { $scr = $_ @@ -395,31 +325,28 @@ $psPackages = @( If ( $installedScripts.Name -contains $scr.Name ) { Write-Log -Message "Script $( $scr.Name ) is already installed" -Severity VERBOSE - $alreadyInstalledScript = $installedScripts | Where-Object { $_.Name -eq $scr.Name } #| Select -first 1 + $alreadyInstalledScript = $installedScripts | Where-Object { $_.Name -eq $scr.Name } If ( $scr.Version -gt $alreadyInstalledScript.Version ) { Write-Log -Message "Script $( $scr.Name ) is installed with an older version $( $alreadyInstalledScript.Version ) than the available version $( $scr.Version )" -Severity VERBOSE Update-Script -Name $scr.Name - $s += 1 + $Script:installCount_s += 1 } else { Write-Log -Message "No need to update $( $scr.Name )" -Severity VERBOSE } } else { Write-Log -Message "Installing Script $( $scr.Name )" -Severity VERBOSE - Install-Script -Name $scr.Name -Scope $psScope #-Force - $s += 1 + Install-Script -Name $scr.Name -Scope $psScope + $Script:installCount_s += 1 } } } - #} - } catch { Write-Log -Message "Cannot install scripts!" -Severity WARNING - #$success = $false } @@ -434,16 +361,12 @@ $psPackages = @( # CHECK MODULES DEPENDENCIES FOR INSTALLATION AND UPDATE #----------------------------------------------- - $m = 0 If ( $Module.count -gt 0 ) { try { - # PSGallery should have been added automatically yet - Write-Log "Checking Module dependencies" -Severity VERBOSE - #$installedModules = Get-InstalledModule $Module | Where-Object { $_ -notin @("PowerShellGet","PackageManagement") } | ForEach-Object { $psModule = $_ @@ -452,13 +375,12 @@ $psPackages = @( $installedModules = Get-InstalledModule - # TODO [ ] possibly add dependencies on version number - # This is using -force to allow updates If ( $ExcludeDependencies -eq $true ) { - $psModuleDependencies = Find-Module -Name $psModule #-IncludeDependencies + $psModuleDependencies = Find-Module -Name $psModule } else { $psModuleDependencies = Find-Module -Name $psModule -IncludeDependencies } + $psModuleDependencies | ForEach-Object { $mod = $_ @@ -466,23 +388,22 @@ $psPackages = @( If ( $installedModules.Name -contains $mod.Name ) { Write-Log -Message "Module $( $mod.Name ) is already installed" -Severity VERBOSE - $alreadyInstalledModule = $installedModules | Where-Object { $_.Name -eq $mod.Name } #| Select -first 1 + $alreadyInstalledModule = $installedModules | Where-Object { $_.Name -eq $mod.Name } If ( $mod.Version -gt $alreadyInstalledModule.Version ) { Write-Log -Message "Module $( $mod.Name ) is installed with an older version $( $alreadyInstalledModule.Version ) than the available version $( $mod.Version )" -Severity VERBOSE Update-Module -Name $mod.Name - $m += 1 + $Script:installCount_m += 1 } else { Write-Log -Message "No need to update $( $mod.Name )" -Severity VERBOSE } } else { Write-Log -Message "Installing Module $( $mod.Name )" -Severity VERBOSE - Install-Module -Name $mod.Name -Scope $psScope -AllowClobber #-Force - $m += 1 + Install-Module -Name $mod.Name -Scope $psScope -AllowClobber + $Script:installCount_m += 1 } } - #$psModuleDependencies | where { $_.Name -notin $installedModules.Name } | Install-Module -Scope AllUsers -Verbose -Force } @@ -490,8 +411,6 @@ $psPackages = @( Write-Log -Message "Cannot install modules!" -Severity WARNING - #Write-Error -Message $_.Exception.Message #-Severity ERROR - } } else { @@ -505,29 +424,12 @@ $psPackages = @( # CHECK PACKAGES NUGET REPOSITORY #----------------------------------------------- - <# - - If this module is not installed via nuget, then this makes sense to check again - - # Add nuget first or make sure it is set - - Register-PackageSource -Name "Nuget v2" -Location "https://www.nuget.org/api/v2" –ProviderName Nuget - - # Make nuget trusted - Set-PackageSource -Name NuGet -Trusted - - #> - - # Get-PSRepository - - #Install-Package Microsoft.Data.Sqlite.Core -RequiredVersion 7.0.0-rc.2.22472.11 - If ( $GlobalPackage.Count -gt 0 -or $LocalPackage.Count -gt 0 ) { try { # Get NuGet sources - $sources = @( Get-PackageSource -ProviderName $packageSourceProviderName ) #| where { $_.Location -like "https://www.nuget.org*" } + $sources = @( Get-PackageSource -ProviderName $packageSourceProviderName ) # See if Nuget needs to get registered If ( $sources.count -ge 1 ) { @@ -537,14 +439,12 @@ $psPackages = @( $registerNugetDecision = $Host.UI.PromptForChoice("", "Register $( $packageSourceProviderName ) as repository?", @('&Yes'; '&No'), 1) If ( $registerNugetDecision -eq "0" ) { - # Means yes and proceed Register-PackageSource -Name $packageSourceName -Location $packageSourceLocation -ProviderName $packageSourceProviderName # Load sources again - $sources = @( Get-PackageSource -ProviderName $packageSourceProviderName ) #| where { $_.Location -like "https://www.nuget.org*" } + $sources = @( Get-PackageSource -ProviderName $packageSourceProviderName ) } else { - # Means no and leave Write-Log "No package repository found! Please make sure to add a NuGet repository to your machine!" -Severity ERROR exit 0 } @@ -568,16 +468,11 @@ $psPackages = @( } - # TODO [x] ask if you want to trust the new repository - # Do you want to trust that source? If ( $packageSource.IsTrusted -eq $false ) { Write-Log -Message "Your source is not trusted. Do you want to trust it now?" -Severity WARNING $trustChoice = Request-Choice -title "Trust Package Source" -message "Do you want to trust $( $packageSource.Name )?" -choices @("Yes", "No") If ( $trustChoice -eq 1 ) { - # Use - # Set-PackageSource -Name NuGet - # To get it to the untrusted status again Set-PackageSource -Name $packageSource.Name -Trusted } } @@ -592,11 +487,9 @@ $psPackages = @( #----------------------------------------------- - # CHECK LOCAL PACKAGES DEPENDENCIES FOR INSTALLATION AND UPDATE + # CHECK LOCAL AND GLOBAL PACKAGES FOR INSTALLATION AND UPDATE #----------------------------------------------- - $l = 0 - $g = 0 If ( $LocalPackage.count -gt 0 -or $GlobalPackage.Count -gt 0) { try { @@ -620,79 +513,70 @@ $psPackages = @( $pkg = [System.Collections.ArrayList]@() If ( $GlobalPackage -contains $psPackage ) { $globalFlag = $true - } # TODO [ ] Especially test global and local installation + } Write-Log "Checking package: $( $psPackage )" -severity VERBOSE - # This is using -force to allow updates - <# - Use of continue in case of error because sometimes this happens - AUSFÜHRLICH: Total package yield:'2' for the specified package 'System.ObjectModel'. - Find-Package : Unable to find dependent package(s) (nuget:Microsoft.NETCore.Platforms/3.1.0) - #> - If ( ($psPackage.gettype()).Name -eq "PsCustomObject" ) { If ( $null -eq $psPackage.version ) { Write-Verbose "Looking for $( $psPackage.name ) without specific version." If ( $ExcludeDependencies -eq $true ) { - [void]@( Find-Package $psPackage.name -Source $packageSource.Name -ErrorAction Continue ).foreach({$pkg.add($_)}) # add elements directly instead of saving everything into a variable + [void]@( Find-Package $psPackage.name -Source $packageSource.Name -ErrorAction Continue ).foreach({$pkg.add($_)}) } else { - [void]@( Find-Package $psPackage.name -IncludeDependencies -Source $packageSource.Name -ErrorAction Continue ).foreach({$pkg.add($_)}) # add elements directly instead of saving everything into a variable + [void]@( Find-Package $psPackage.name -IncludeDependencies -Source $packageSource.Name -ErrorAction Continue ).foreach({$pkg.add($_)}) } } else { Write-Verbose "Looking for $( $psPackage.name ) with version $( $psPackage.version )" If ( $ExcludeDependencies -eq $true ) { - [void]@( Find-Package $psPackage.name -Source $packageSource.Name -ErrorAction Continue -RequiredVersion $psPackage.version ).foreach({$pkg.add($_)}) # add elements directly instead of saving everything into a variable + [void]@( Find-Package $psPackage.name -Source $packageSource.Name -ErrorAction Continue -RequiredVersion $psPackage.version ).foreach({$pkg.add($_)}) } else { - [void]@( Find-Package $psPackage.name -IncludeDependencies -Source $packageSource.Name -ErrorAction Continue -RequiredVersion $psPackage.version ).foreach({$pkg.add($_)}) # add elements directly instead of saving everything into a variable + [void]@( Find-Package $psPackage.name -IncludeDependencies -Source $packageSource.Name -ErrorAction Continue -RequiredVersion $psPackage.version ).foreach({$pkg.add($_)}) } } } else { Write-Verbose "Looking for $( $psPackage ) without specific version" If ( $ExcludeDependencies -eq $true ) { - [void]@( Find-Package $psPackage -Source $packageSource.Name -ErrorAction Continue ).foreach({$pkg.add($_)}) # add elements directly instead of saving everything into a variable + [void]@( Find-Package $psPackage -Source $packageSource.Name -ErrorAction Continue ).foreach({$pkg.add($_)}) } else { - [void]@( Find-Package $psPackage -IncludeDependencies -Source $packageSource.Name -ErrorAction Continue ).foreach({$pkg.add($_)}) # add elements directly instead of saving everything into a variable } + [void]@( Find-Package $psPackage -IncludeDependencies -Source $packageSource.Name -ErrorAction Continue ).foreach({$pkg.add($_)}) } } - $pkg | ForEach-Object { # | Where-Object { $_.Name -notin $installedPackages.Name } # | Sort-Object Name, Version -Unique -Descending + $pkg | ForEach-Object { $p = $_ $pd = [PSCustomObject]@{ "GlobalFlag" = $globalFlag - "Package" = $p - "Name" = $p.Name - "Version" = $p.Version + "Package" = $p + "Name" = $p.Name + "Version" = $p.Version } [void]$packagesToInstall.Add($pd) - } } Write-Log -Message "Done with searching for $( $packagesToInstall.Count ) packages" - # Install the packages now, we only use packages of the current repository, so in there if other repositories are used for cross-reference, this won't work at the moment $pack = $packagesToInstall | Where-Object { $_.Package.Summary -notlike "*not reference directly*" -and $_.Package.Name -notlike "Xamarin.*"} | Where-Object { $_.Package.Source -eq $packageSource.Name } | Sort-Object Name, Version -Unique -Descending Write-Log -Message "This is likely to install $( $pack.Count ) packages" - #$packagesToInstall | Where-Object { $_.Source -eq $packageSource.Name -and $_.Name -notin $installedPackages.Name } | Sort-Object Name -Unique | ForEach-Object { #where-object { $_.Source -eq $packageSource.Name } | Select-Object * -Unique | ForEach-Object { - $pack | ForEach-Object { #where-object { $_.Source -eq $packageSource.Name } | Select-Object * -Unique | ForEach-Object { + $i = 0 + $pack | ForEach-Object { $p = $_ If ( $p.GlobalFlag -eq $true ) { Write-Log -message "Installing $( $p.Package.Name ) with version $( $p.Package.version ) from $( $p.Package.Source ) globally" Install-Package -Name $p.Name -Scope $psScope -Source $packageSource.Name -RequiredVersion $p.Version -SkipDependencies -Force - $g += 1 + $Script:installCount_g += 1 } else { Write-Log -message "Installing $( $p.Name ) with version $( $p.version ) from $( $p.Package.Source ) locally" Install-Package -Name $p.Name -Scope $psScope -Source $packageSource.Name -RequiredVersion $p.Version -SkipDependencies -Force -Destination $LocalPackageFolder - $l += 1 + $Script:installCount_l += 1 } - # Write progress Write-Progress -Activity "Package installation in progress" -Status "$( [math]::Round($i/$pack.Count*100) )% Complete:" -PercentComplete ([math]::Round($i/$pack.Count*100)) + $i += 1 } @@ -708,24 +592,45 @@ $psPackages = @( } + # Reset the process ID if another module overrode it + Set-ProcessId -Id $processId + } + + + End { #----------------------------------------------- - # FINISHING + # STATUS #----------------------------------------------- - # Installation Status Write-Log -Message "STATUS:" -Severity INFO - Write-Log -Message " $( $l ) local packages installed into '$( $LocalPackageFolder )'" -Severity INFO - Write-Log -Message " $( $g ) global packages installed" -Severity INFO - Write-Log -Message " $( $m ) modules installed with scope '$( $psScope )'" -Severity INFO - Write-Log -Message " $( $s ) scripts installed with scope '$( $psScope )'" -Severity INFO + Write-Log -Message " $( $Script:installCount_l ) local packages installed into '$( $LocalPackageFolder )'" -Severity INFO + Write-Log -Message " $( $Script:installCount_g ) global packages installed" -Severity INFO + Write-Log -Message " $( $Script:installCount_m ) modules installed with scope '$( $psScope )'" -Severity INFO + Write-Log -Message " $( $Script:installCount_s ) scripts installed with scope '$( $psScope )'" -Severity INFO + + + #----------------------------------------------- + # FINISHING + #----------------------------------------------- - # Performance information $processEnd = [datetime]::now $processDuration = New-TimeSpan -Start $processStart -End $processEnd Write-Log -Message "Done! Needed $( [int]$processDuration.TotalSeconds ) seconds in total" -Severity INFO + Write-Log -Message "Logfile override: $( Get-LogfileOverride )" + + If ( $KeepLogfile -eq $false -and $null -ne $getLogfile -and '' -ne $getLogfile ) { + Write-Log -Message "Changing logfile back to '$( $currentLogfile )'" + Set-Logfile -Path $currentLogfile + } + + # Set explicitly verbose output back + If ( $SuppressWarnings -ne $true -and $PSBoundParameters["Verbose"].IsPresent -eq $true -and $PSBoundParameters["Verbose"] -eq $True) { + $VerbosePreference = $originalVerbosity + } + } -} \ No newline at end of file +} diff --git a/SqlPipeline/Readme.md b/SqlPipeline/Readme.md index 0b9325d..c3a6ddd 100644 --- a/SqlPipeline/Readme.md +++ b/SqlPipeline/Readme.md @@ -24,9 +24,31 @@ Current functionality: Install the required DuckDB.NET NuGet packages into a `./lib` subfolder: ```PowerShell +# PowerShell 7+ (latest DuckDB.NET) Install-SqlPipeline + +# Windows PowerShell 5.1 (pinned compatible versions) +Install-SqlPipeline -WindowsPowerShell ``` +## PowerShell Version Compatibility + +| Feature | PowerShell 7+ | Windows PowerShell 5.1 | +|---|---|---| +| DuckDB.NET version | latest | 1.4.4 (maximum) | +| Extra dependencies | none | `System.Memory` 4.6.0 | + +**Windows PowerShell 5.1** runs on .NET Framework 4.x, which is missing some APIs that newer DuckDB.NET versions require. Use the `-WindowsPowerShell` switch when installing on Windows PowerShell 5.1: + +```PowerShell +Install-SqlPipeline -WindowsPowerShell +``` + +This installs: +- `DuckDB.NET.Bindings.Full` 1.4.4 +- `DuckDB.NET.Data.Full` 1.4.4 +- `System.Memory` 4.6.0 (required polyfill not included in .NET Framework) + ## Quick Start (in-memory, no setup needed) ```PowerShell diff --git a/SqlPipeline/SqlPipeline/Private/SqlPipeline/Install-NuGetPackage.ps1 b/SqlPipeline/SqlPipeline/Private/SqlPipeline/Install-NuGetPackage.ps1 index 83d20f6..9c4c3bb 100644 --- a/SqlPipeline/SqlPipeline/Private/SqlPipeline/Install-NuGetPackage.ps1 +++ b/SqlPipeline/SqlPipeline/Private/SqlPipeline/Install-NuGetPackage.ps1 @@ -23,7 +23,9 @@ function Install-NuGetPackage { $pkgId = $PackageId.ToLower() $url = "https://api.nuget.org/v3-flatcontainer/$pkgId/$Version/$pkgId.$Version.nupkg" - $outFile = Join-Path $OutputDir "$pkgId.$Version.nupkg" + # Download as .zip so Expand-Archive accepts it on Windows PowerShell 5.1, + # which rejects any extension other than .zip even though .nupkg is identical format. + $outFile = Join-Path $OutputDir "$pkgId.$Version.zip" $unzipDir = Join-Path $OutputDir "$pkgId.$Version" Write-Verbose "Downloading $PackageId $Version ..." diff --git a/SqlPipeline/SqlPipeline/Private/duckdb/Get-DuckDBColumns.ps1 b/SqlPipeline/SqlPipeline/Private/duckdb/Get-DuckDBColumns.ps1 index 1b647d0..4559cee 100644 --- a/SqlPipeline/SqlPipeline/Private/duckdb/Get-DuckDBColumns.ps1 +++ b/SqlPipeline/SqlPipeline/Private/duckdb/Get-DuckDBColumns.ps1 @@ -11,9 +11,6 @@ function Get-DuckDBColumns { ) $result = Get-DuckDBData -Connection $Connection -Query "DESCRIBE '$TableName'" - #return @($result.Rows | ForEach-Object { $_['column_name'] }) - - # return - $result.column_name + return @($result.Rows | ForEach-Object { $_['column_name'] }) } diff --git a/SqlPipeline/SqlPipeline/Private/duckdb/Test-DuckDBTableExists.ps1 b/SqlPipeline/SqlPipeline/Private/duckdb/Test-DuckDBTableExists.ps1 index 4978472..ac9e263 100644 --- a/SqlPipeline/SqlPipeline/Private/duckdb/Test-DuckDBTableExists.ps1 +++ b/SqlPipeline/SqlPipeline/Private/duckdb/Test-DuckDBTableExists.ps1 @@ -16,7 +16,6 @@ function Test-DuckDBTableExists { WHERE table_name = '$TableName' AND table_schema = 'main' "@ - #return ([int]$result.Rows[0]['cnt'] -gt 0) - return ([int]$result['cnt'] -gt 0) + return ([int]$result.Rows[0]['cnt'] -gt 0) } \ No newline at end of file diff --git a/SqlPipeline/SqlPipeline/Private/duckdb/Write-DuckDBAppender.ps1 b/SqlPipeline/SqlPipeline/Private/duckdb/Write-DuckDBAppender.ps1 index c04a194..1d9e949 100644 --- a/SqlPipeline/SqlPipeline/Private/duckdb/Write-DuckDBAppender.ps1 +++ b/SqlPipeline/SqlPipeline/Private/duckdb/Write-DuckDBAppender.ps1 @@ -23,6 +23,14 @@ function Write-DuckDBAppender { $appenderRow = $appender.CreateRow() foreach ($name in $propNames) { $val = $row.$name + # Normalize integer subtypes to Int64 before any other check, + # because DuckDB.NET appender has no Int32 overload and PowerShell + # would otherwise fall back to AppendValue(string). + if ($val -is [int] -or $val -is [System.Int16] -or $val -is [byte] -or $val -is [uint16] -or $val -is [uint32]) { + $val = [long]$val + } elseif ($val -is [float]) { + $val = [double]$val + } # Inlined ConvertTo-DuckDBValue if ($null -eq $val) { [void]$appenderRow.AppendValue([DBNull]::Value) @@ -37,7 +45,7 @@ function Write-DuckDBAppender { } $appenderRow.EndRow() - If ( $i % 100 -eq 0 ) { + If ( $i % 10000 -eq 0 ) { Write-Verbose "[$TableName] Appender: Row $i written." } } diff --git a/SqlPipeline/SqlPipeline/Public/SqlPipeline/Install-SqlPipeline.ps1 b/SqlPipeline/SqlPipeline/Public/SqlPipeline/Install-SqlPipeline.ps1 index 8b91bf7..e50a3d1 100644 --- a/SqlPipeline/SqlPipeline/Public/SqlPipeline/Install-SqlPipeline.ps1 +++ b/SqlPipeline/SqlPipeline/Public/SqlPipeline/Install-SqlPipeline.ps1 @@ -3,16 +3,32 @@ function Install-SqlPipeline { [CmdletBinding()] - param() + param( + [Parameter(Mandatory=$false)] + [switch]$WindowsPowerShell + ) process { Write-Verbose "Starting installation of SQLPipeline dependencies..." - If ( $Script:psPackages.Count -gt 0 ) { + # Windows PowerShell 5.1 requires specific older package versions: + # DuckDB.NET 1.4.4 (last version compatible with .NET Framework / WinPS 5.1) + # System.Memory 4.6.0 (required polyfill not included in .NET Framework) + $packagesToInstall = if ($WindowsPowerShell) { + [Array]@( + [PSCustomObject]@{ Name = "DuckDB.NET.Bindings.Full"; Version = "1.4.4" } + [PSCustomObject]@{ Name = "DuckDB.NET.Data.Full"; Version = "1.4.4" } + [PSCustomObject]@{ Name = "System.Memory"; Version = "4.6.0" } + ) + } else { + $Script:psPackages + } + + If ( $packagesToInstall.Count -gt 0 ) { $pse = Get-PSEnvironment - Write-Verbose "There are currently $($Script:psPackages.Count) packages defined as dependencies: $($Script:psPackages -join ", ")" + Write-Verbose "There are currently $($packagesToInstall.Count) packages to install." Write-Verbose "Checking for already installed packages..." Write-Verbose "Installed local packages: $( $pse.InstalledLocalPackages.Id -join ", ")" Write-Verbose "To update already installed packages, please remove them first and then run Install-SqlPipeline again." @@ -20,15 +36,16 @@ function Install-SqlPipeline { $outputDir = Join-Path -Path $PWD.Path -ChildPath "/lib" New-Item -Path $outputDir -ItemType Directory -Force | Out-Null - $psPackages | ForEach-Object { + $packagesToInstall | ForEach-Object { $pkg = $_ - $pkgName = if ( $pkg -is [string] ) { $pkg } elseif ( $pkg -is [pscustomobject] -and $pkg.Name ) { $pkg.Name } else { throw "Invalid package definition: $pkg" } - Write-Verbose "Checking if package $pkg is already installed..." + $pkgName = if ( $pkg -is [string] ) { $pkg } elseif ( $pkg -is [pscustomobject] -and $pkg.Name ) { $pkg.Name } else { throw "Invalid package definition: $pkg" } + $pkgVersion = if ( $pkg -is [pscustomobject] -and $pkg.Version ) { $pkg.Version } else { "" } + Write-Verbose "Checking if package $pkgName is already installed..." If ( -not ( $pse.InstalledLocalPackages.Id -contains $pkgName ) ) { - Write-Verbose "Package $pkg is not installed. Downloading and installing..." - Install-NuGetPackage -PackageId $pkg -OutputDir $outputDir + Write-Verbose "Package $pkgName is not installed. Downloading and installing..." + Install-NuGetPackage -PackageId $pkgName -Version $pkgVersion -OutputDir $outputDir } else { - Write-Verbose "Package $pkg is already installed. Skipping download." + Write-Verbose "Package $pkgName is already installed. Skipping download." } } diff --git a/SqlPipeline/SqlPipeline/Public/duckdb/Add-RowsToDuckDB.ps1 b/SqlPipeline/SqlPipeline/Public/duckdb/Add-RowsToDuckDB.ps1 index f0dd000..5074e15 100644 --- a/SqlPipeline/SqlPipeline/Public/duckdb/Add-RowsToDuckDB.ps1 +++ b/SqlPipeline/SqlPipeline/Public/duckdb/Add-RowsToDuckDB.ps1 @@ -118,7 +118,7 @@ function Add-RowsToDuckDB { Write-Information "[$TableName] $rowCount rows inserted via pipeline." # Force DuckDB to flush changes to disk (important for in-memory connections or when using transactions) - Invoke-DuckDBQuery -Query "FORCE CHECKPOINT" + Invoke-DuckDBQuery -Connection $Connection -Query "FORCE CHECKPOINT" } } diff --git a/SqlPipeline/SqlPipeline/Public/duckdb/Close-SqlPipeline.ps1 b/SqlPipeline/SqlPipeline/Public/duckdb/Close-SqlPipeline.ps1 index 0e0bbfe..e96ac29 100644 --- a/SqlPipeline/SqlPipeline/Public/duckdb/Close-SqlPipeline.ps1 +++ b/SqlPipeline/SqlPipeline/Public/duckdb/Close-SqlPipeline.ps1 @@ -18,5 +18,12 @@ function Close-SqlPipeline { $Connection.Close() Write-Verbose 'DuckDB connection closed.' } - + + # If the closed connection was the active default, restore the in-memory connection + # so that subsequent calls without -Connection still work. + if ([object]::ReferenceEquals($Connection, $Script:DefaultConnection)) { + $Script:DefaultConnection = $Script:InMemoryConnection + Write-Verbose 'Default connection restored to in-memory database.' + } + } \ No newline at end of file diff --git a/SqlPipeline/SqlPipeline/Public/duckdb/Get-DuckDBData.ps1 b/SqlPipeline/SqlPipeline/Public/duckdb/Get-DuckDBData.ps1 index 407638a..6759241 100644 --- a/SqlPipeline/SqlPipeline/Public/duckdb/Get-DuckDBData.ps1 +++ b/SqlPipeline/SqlPipeline/Public/duckdb/Get-DuckDBData.ps1 @@ -20,6 +20,6 @@ function Get-DuckDBData { $reader = $cmd.ExecuteReader() $table = [System.Data.DataTable]::new() $table.Load($reader) - $table + ,$table } diff --git a/SqlPipeline/SqlPipeline/Public/duckdb/Get-LastLoadTimestamp.ps1 b/SqlPipeline/SqlPipeline/Public/duckdb/Get-LastLoadTimestamp.ps1 index fa4cd5f..b289879 100644 --- a/SqlPipeline/SqlPipeline/Public/duckdb/Get-LastLoadTimestamp.ps1 +++ b/SqlPipeline/SqlPipeline/Public/duckdb/Get-LastLoadTimestamp.ps1 @@ -27,6 +27,5 @@ function Get-LastLoadTimestamp { Write-Verbose "[$TableName] No previous load found - performing full load." return [datetime]'2000-01-01' } - #return [datetime]$result.Rows[0]['last_loaded'] - return [datetime]$result['last_loaded'] + return [datetime]$result.Rows[0]['last_loaded'] } diff --git a/SqlPipeline/SqlPipeline/Public/simplysql/Add-RowsToSql.ps1 b/SqlPipeline/SqlPipeline/Public/simplysql/Add-RowsToSql.ps1 index 30e50c4..7bb8027 100644 --- a/SqlPipeline/SqlPipeline/Public/simplysql/Add-RowsToSql.ps1 +++ b/SqlPipeline/SqlPipeline/Public/simplysql/Add-RowsToSql.ps1 @@ -160,6 +160,15 @@ function Add-RowsToSql { begin { + #----------------------------------------------- + # CHECK SIMPLYSQL AVAILABILITY + #----------------------------------------------- + + if (-not $Script:isSimplySqlLoaded) { + throw "SimplySql is not loaded. This may be due to platform incompatibility (e.g. ARM architecture). Use Add-RowsToDuckDB instead." + } + + #----------------------------------------------- # INITIALISE #----------------------------------------------- diff --git a/SqlPipeline/SqlPipeline/SqlPipeline.psd1 b/SqlPipeline/SqlPipeline/SqlPipeline.psd1 index b27bb03..dbbf88b 100644 --- a/SqlPipeline/SqlPipeline/SqlPipeline.psd1 +++ b/SqlPipeline/SqlPipeline/SqlPipeline.psd1 @@ -5,7 +5,7 @@ RootModule = 'SqlPipeline.psm1' # Die Versionsnummer dieses Moduls -ModuleVersion = '0.3.2' +ModuleVersion = '0.3.4' # Unterstützte PSEditions # CompatiblePSEditions = @() @@ -20,7 +20,7 @@ Author = 'florian.von.bracht@apteco.de' CompanyName = 'Apteco GmbH' # Urheberrechtserklärung für dieses Modul -Copyright = '(c) 2025 Apteco GmbH. All rights reserved.' +Copyright = '(c) 2026 Apteco GmbH. All rights reserved.' # Beschreibung der von diesem Modul bereitgestellten Funktionen Description = 'Apteco PS Modules - Wrapper for SimplySQL @@ -124,6 +124,12 @@ PrivateData = @{ # 'ReleaseNotes' des Moduls ReleaseNotes = ' +0.3.4 Fixing package installation with PowerShell 5.1 because Expand-Archive only supports *.zip files +0.3.3 Extending Install-SqlPipeline to install DuckDB.net 1.4.4 when using PowerShell 5.1 (latest supported version), pwsh is supporting all latest versions + Fixing to not cancel the module import if SimplySQL does not match the current processor architecture + Fixing returned values for last loaded timestamp, duckdb columns, existing table, Get-DuckDB data in general + Fixing datatype matching for DuckDB, as there is not Int32 + Fixing the fallback of a closed connection to existing in-memory connection 0.3.2 Renaming Close-DuckDBConnection to Close-SqlPipeline to not clash with Close-DuckDBConnection of AptecoPSFramework (which is used for direct DuckDB connections outside of SqlPipeline) 0.3.1 Skipping comparison with complex datatypes with -SimpleTypesOnly for better import performance Adding a csv importer to export big files first into a temporary file and directly import them via DuckDB into a staging table diff --git a/SqlPipeline/SqlPipeline/SqlPipeline.psm1 b/SqlPipeline/SqlPipeline/SqlPipeline.psm1 index 6f8f26b..3ccbb8d 100644 --- a/SqlPipeline/SqlPipeline/SqlPipeline.psm1 +++ b/SqlPipeline/SqlPipeline/SqlPipeline.psm1 @@ -79,7 +79,9 @@ New-Variable -Name moduleRoot -Value $null -Scope Script -Force # Current lo New-Variable -Name PipelineBuffer -Value $null -Scope Script -Force # Buffer for the incremental load pipeline New-Variable -Name PipelineOptions -Value $null -Scope Script -Force # Options for New-Variable -Name isDuckDBLoaded -Value $null -Scope Script -Force # Flag indicating whether DuckDB.NET is available -New-Variable -Name DefaultConnection -Value $null -Scope Script -Force # Default DuckDB connection (in-memory, auto-initialized on module load) +New-Variable -Name isSimplySqlLoaded -Value $null -Scope Script -Force # Flag indicating whether SimplySql is available +New-Variable -Name DefaultConnection -Value $null -Scope Script -Force # Default DuckDB connection (in-memory, auto-initialized on module load) +New-Variable -Name InMemoryConnection -Value $null -Scope Script -Force # The auto-initialized in-memory connection; used as fallback when a file connection is closed New-Variable -Name psModules -Value $null -Scope Script -Force # Module dependencies New-Variable -Name psPackages -Value $null -Scope Script -Force # NuGet package dependencies New-Variable -Name psAssemblies -Value $null -Scope Script -Force # .NET assembly dependencies @@ -91,6 +93,7 @@ $Script:moduleRoot = $PSScriptRoot.ToString() # Internal pipeline buffer per table $Script:isDuckDBLoaded = $false +$Script:isSimplySqlLoaded = $false $Script:PipelineBuffer = [System.Collections.Generic.Dictionary[string, System.Collections.Generic.List[PSObject]]]::new() $Script:PipelineOptions = [System.Collections.Generic.Dictionary[string, hashtable]]::new() @@ -105,7 +108,12 @@ $Script:PipelineOptions = [System.Collections.Generic.Dictionary[string, hashtab # Load modules Write-Verbose "There are currently $($Script:psModules.Count) modules defined as dependencies: $($Script:psModules -join ", ")" If ( $Script:psModules.Count -gt 0 ) { - Import-Dependency -Module $psModules + try { + Import-Dependency -Module $psModules + $Script:isSimplySqlLoaded = $true + } catch { + Write-Warning "Failed to load one or more module dependencies ($($Script:psModules -join ', ')): $_. SimplySql-based functions will not be available. DuckDB functions will still work if DuckDB.NET is installed." + } } # TODO For future you need in linux maybe this module for outgrid-view, which is also supported on console only: microsoft.powershell.consoleguitools @@ -122,7 +130,8 @@ $Script:psAssemblies | ForEach-Object { # is only required when a persistent file-based database is needed. if ($Script:isDuckDBLoaded) { try { - $Script:DefaultConnection = New-DuckDBConnection -DbPath ':memory:' + $Script:DefaultConnection = New-DuckDBConnection -DbPath ':memory:' + $Script:InMemoryConnection = $Script:DefaultConnection Initialize-PipelineMetadata -Connection $Script:DefaultConnection Write-Verbose "DuckDB in-memory connection initialized automatically. Call Initialize-SQLPipeline -DbPath to switch to a file-based database." } catch { diff --git a/SqlPipeline/Tests/SqlPipeline_DuckDB.Tests.ps1 b/SqlPipeline/Tests/SqlPipeline_DuckDB.Tests.ps1 new file mode 100644 index 0000000..2efe527 --- /dev/null +++ b/SqlPipeline/Tests/SqlPipeline_DuckDB.Tests.ps1 @@ -0,0 +1,319 @@ +# --------------------------------------------------------------------------- +# DuckDB tests +# All tests use the in-memory DuckDB connection that is auto-initialized +# when the SqlPipeline module is imported. The entire Describe block is +# skipped when DuckDB.NET is not available in the current environment. +# --------------------------------------------------------------------------- + +BeforeDiscovery { + # Probe whether DuckDB is usable so we can skip gracefully + $script:duckDBAvailable = $false + try { + Import-Module "$PSScriptRoot/../SqlPipeline" -Force #-ErrorAction Stop 2>$null + Invoke-DuckDBQuery -Query "SELECT 1" -ErrorAction Stop + $script:duckDBAvailable = $true + } catch { + $script:duckDBAvailable = $false + # Windows PowerShell 5.1 requires older pinned versions of DuckDB.NET + System.Memory + if ($PSVersionTable.PSEdition -eq 'Desktop' -or $PSVersionTable.PSVersion.Major -le 5) { + Install-SqlPipeline -WindowsPowerShell + } else { + Install-SqlPipeline + } + # Re-import so the psm1 re-runs and creates $Script:DefaultConnection + # with the newly installed packages. + Import-Module "$PSScriptRoot/../SqlPipeline" -Force + } + + # Try again, if still false + try { + Invoke-DuckDBQuery -Query "SELECT 1" -ErrorAction Stop + $script:duckDBAvailable = $true + } catch { + $script:duckDBAvailable = $false + } + +} + +Describe "Invoke-DuckDBQuery" -Skip:(-not $script:duckDBAvailable) { + + AfterEach { + Invoke-DuckDBQuery -Query "DROP TABLE IF EXISTS dq_test" -ErrorAction SilentlyContinue + } + + It "Executes a CREATE TABLE without throwing" { + { Invoke-DuckDBQuery -Query "CREATE TABLE IF NOT EXISTS dq_test (id INTEGER, val VARCHAR)" } | Should -Not -Throw + } + + It "Executes an INSERT without throwing" { + Invoke-DuckDBQuery -Query "CREATE TABLE IF NOT EXISTS dq_test (id INTEGER, val VARCHAR)" + { Invoke-DuckDBQuery -Query "INSERT INTO dq_test VALUES (1, 'hello')" } | Should -Not -Throw + } + + It "Executes a DROP TABLE without throwing" { + Invoke-DuckDBQuery -Query "CREATE TABLE IF NOT EXISTS dq_test (id INTEGER)" + { Invoke-DuckDBQuery -Query "DROP TABLE dq_test" } | Should -Not -Throw + } + +} + + +Describe "Get-DuckDBData" -Skip:(-not $script:duckDBAvailable) { + + BeforeAll { + Invoke-DuckDBQuery -Query "CREATE TABLE IF NOT EXISTS gd_test (id INTEGER, name VARCHAR)" + Invoke-DuckDBQuery -Query "INSERT INTO gd_test VALUES (1, 'Alice'), (2, 'Bob')" + } + + AfterAll { + Invoke-DuckDBQuery -Query "DROP TABLE IF EXISTS gd_test" + } + + It "Returns a DataTable" { + $result = Get-DuckDBData -Query "SELECT * FROM gd_test" + Should -ActualValue $result -BeOfType [System.Data.DataTable] + } + + It "Returns the correct number of rows" { + $result = Get-DuckDBData -Query "SELECT * FROM gd_test" + $result.Rows.Count | Should -Be 2 + } + + It "Returns expected column names" { + $result = Get-DuckDBData -Query "SELECT * FROM gd_test" + $result.Columns.ColumnName | Should -Contain "id" + $result.Columns.ColumnName | Should -Contain "name" + } + + It "Returns correct values" { + $result = Get-DuckDBData -Query "SELECT name FROM gd_test ORDER BY id" + $result.Rows[0]["name"] | Should -Be "Alice" + $result.Rows[1]["name"] | Should -Be "Bob" + } + + It "Returns empty DataTable for a query with no results" { + $result = Get-DuckDBData -Query "SELECT * FROM gd_test WHERE id = 9999" + Should -ActualValue $result -BeOfType [System.Data.DataTable] + $result.Rows.Count | Should -Be 0 + } + +} + + +Describe "Add-RowsToDuckDB" -Skip:(-not $script:duckDBAvailable) { + + AfterEach { + Invoke-DuckDBQuery -Query "DROP TABLE IF EXISTS ard_people" -ErrorAction SilentlyContinue + Invoke-DuckDBQuery -Query "DROP TABLE IF EXISTS ard_upsert" -ErrorAction SilentlyContinue + Invoke-DuckDBQuery -Query "DROP TABLE IF EXISTS ard_schema" -ErrorAction SilentlyContinue + Invoke-DuckDBQuery -Query "DROP TABLE IF EXISTS ard_tx" -ErrorAction SilentlyContinue + } + + It "Inserts PSCustomObject rows" { + $rows = @( + [PSCustomObject]@{ Name = "Alice"; Age = 30 } + [PSCustomObject]@{ Name = "Bob"; Age = 25 } + ) + $rows | Add-RowsToDuckDB -TableName "ard_people" + + $result = Get-DuckDBData -Query "SELECT * FROM ard_people" + $result.Rows.Count | Should -Be 2 + $result.Rows.Name | Should -Contain "Alice" + $result.Rows.Name | Should -Contain "Bob" + } + + It "Creates the table automatically on first insert" { + [PSCustomObject]@{ Id = 1; Label = "auto" } | Add-RowsToDuckDB -TableName "ard_people" + + $result = Get-DuckDBData -Query "SELECT * FROM ard_people" + $result.Rows.Count | Should -BeGreaterOrEqual 1 + } + + It "Inserts rows with -UseTransaction without throwing" { + $rows = @( + [PSCustomObject]@{ X = 1 } + [PSCustomObject]@{ X = 2 } + ) + { $rows | Add-RowsToDuckDB -TableName "ard_tx" -UseTransaction } | Should -Not -Throw + } + + It "Performs UPSERT when PKColumns are specified" { + # Insert initial row + [PSCustomObject]@{ Id = 1; Val = "original" } | Add-RowsToDuckDB -TableName "ard_upsert" -PKColumns "Id" + # Upsert same PK with updated value + [PSCustomObject]@{ Id = 1; Val = "updated" } | Add-RowsToDuckDB -TableName "ard_upsert" -PKColumns "Id" + + $result = Get-DuckDBData -Query "SELECT * FROM ard_upsert WHERE Id = 1" + $result.Rows.Count | Should -Be 1 + $result.Rows[0]["Val"] | Should -Be "updated" + } + + It "Evolves schema when new columns appear in later rows" { + [PSCustomObject]@{ Col1 = "A" } | Add-RowsToDuckDB -TableName "ard_schema" + [PSCustomObject]@{ Col1 = "B"; Col2 = "extra" } | Add-RowsToDuckDB -TableName "ard_schema" + + $result = Get-DuckDBData -Query "SELECT Col2 FROM ard_schema WHERE Col2 IS NOT NULL" + $result.Rows.Count | Should -BeGreaterOrEqual 1 + $result.Rows[0]["Col2"] | Should -Be "extra" + } + + It "Inserts multiple batches without data loss" { + $rows = 1..25 | ForEach-Object { [PSCustomObject]@{ Num = $_ } } + $rows | Add-RowsToDuckDB -TableName "ard_people" -BatchSize 10 + + $result = Get-DuckDBData -Query "SELECT COUNT(*) AS cnt FROM ard_people" + [int]$result.Rows[0]["cnt"] | Should -Be 25 + } + +} + + +Describe "Set-LoadMetadata and Get-LastLoadTimestamp" -Skip:(-not $script:duckDBAvailable) { + + AfterEach { + # Clean up metadata rows written by these tests + Invoke-DuckDBQuery -Query "DELETE FROM _load_metadata WHERE table_name LIKE 'meta_%'" -ErrorAction SilentlyContinue + } + + It "Set-LoadMetadata does not throw" { + { Set-LoadMetadata -TableName "meta_orders" -RowsLoaded 100 } | Should -Not -Throw + } + + It "Get-LastLoadTimestamp returns 2000-01-01 before any load is recorded" { + $ts = Get-LastLoadTimestamp -TableName "meta_fresh_$(Get-Random)" + $ts | Should -Be ([datetime]"2000-01-01") + } + + It "Get-LastLoadTimestamp returns the timestamp written by Set-LoadMetadata" { + Set-LoadMetadata -TableName "meta_orders" -RowsLoaded 42 -Status "success" + $ts = Get-LastLoadTimestamp -TableName "meta_orders" + $ts | Should -BeGreaterThan ([datetime]"2000-01-01") + } + + It "Set-LoadMetadata stores the correct row count" { + Set-LoadMetadata -TableName "meta_counts" -RowsLoaded 999 + $result = Get-DuckDBData -Query "SELECT rows_loaded FROM _load_metadata WHERE table_name = 'meta_counts'" + [int]$result.Rows[0]["rows_loaded"] | Should -Be 999 + } + + It "Set-LoadMetadata stores the status correctly" { + Set-LoadMetadata -TableName "meta_status" -RowsLoaded 0 -Status "error" -ErrorMessage "Test error" + $result = Get-DuckDBData -Query "SELECT status, error_msg FROM _load_metadata WHERE table_name = 'meta_status'" + $result.Rows[0]["status"] | Should -Be "error" + $result.Rows[0]["error_msg"] | Should -Be "Test error" + } + + It "Set-LoadMetadata upserts on second call for same table" { + Set-LoadMetadata -TableName "meta_upsert" -RowsLoaded 10 + Set-LoadMetadata -TableName "meta_upsert" -RowsLoaded 20 + + $result = Get-DuckDBData -Query "SELECT rows_loaded FROM _load_metadata WHERE table_name = 'meta_upsert'" + $result.Rows.Count | Should -Be 1 + [int]$result.Rows[0]["rows_loaded"] | Should -Be 20 + } + +} + + +Describe "Initialize-SQLPipeline and Close-SqlPipeline" -Skip:(-not $script:duckDBAvailable) { + + BeforeAll { + $script:dbPath = Join-Path ([System.IO.Path]::GetTempPath()) "pester_duck_$(Get-Random).db" + } + + AfterAll { + Remove-Item $script:dbPath -Force -ErrorAction SilentlyContinue + } + + It "Initialize-SQLPipeline returns a DuckDB connection object" { + $conn = Initialize-SQLPipeline -DbPath $script:dbPath + $conn | Should -Not -BeNullOrEmpty + $conn.GetType().Name | Should -Be "DuckDBConnection" + Close-SqlPipeline -Connection $conn + } + + It "Creates the database file on disk" { + $filePath = Join-Path ([System.IO.Path]::GetTempPath()) "pester_duck_file_$(Get-Random).db" + $conn = Initialize-SQLPipeline -DbPath $filePath + Test-Path $filePath | Should -Be $true + Close-SqlPipeline -Connection $conn + Remove-Item $filePath -Force -ErrorAction SilentlyContinue + } + + It "Connection state is Open after Initialize-SQLPipeline" { + $filePath = Join-Path ([System.IO.Path]::GetTempPath()) "pester_duck_open_$(Get-Random).db" + $conn = Initialize-SQLPipeline -DbPath $filePath + $conn.State | Should -Be "Open" + Close-SqlPipeline -Connection $conn + Remove-Item $filePath -Force -ErrorAction SilentlyContinue + } + + It "Close-SqlPipeline closes the connection without throwing" { + $filePath = Join-Path ([System.IO.Path]::GetTempPath()) "pester_duck_close_$(Get-Random).db" + $conn = Initialize-SQLPipeline -DbPath $filePath + { Close-SqlPipeline -Connection $conn } | Should -Not -Throw + Remove-Item $filePath -Force -ErrorAction SilentlyContinue + } + + It "File-based connection persists data across reconnect" { + $filePath = Join-Path ([System.IO.Path]::GetTempPath()) "pester_duck_persist_$(Get-Random).db" + + $conn1 = Initialize-SQLPipeline -DbPath $filePath + [PSCustomObject]@{ Id = 42; Label = "persist" } | Add-RowsToDuckDB -Connection $conn1 -TableName "persist_test" + Close-SqlPipeline -Connection $conn1 + + $conn2 = Initialize-SQLPipeline -DbPath $filePath + $result = Get-DuckDBData -Connection $conn2 -Query "SELECT * FROM persist_test" + Close-SqlPipeline -Connection $conn2 + + $result.Rows.Count | Should -Be 1 + $result.Rows[0]["Id"] | Should -Be 42 + + Remove-Item $filePath -Force -ErrorAction SilentlyContinue + } + +} + + +Describe "Export-DuckDBToParquet" -Skip:(-not $script:duckDBAvailable) { + + BeforeAll { + Invoke-DuckDBQuery -Query "CREATE TABLE IF NOT EXISTS parquet_src (id INTEGER, val VARCHAR)" + Invoke-DuckDBQuery -Query "INSERT INTO parquet_src VALUES (1,'a'), (2,'b'), (3,'c')" + $script:parquetDir = Join-Path ([System.IO.Path]::GetTempPath()) "pester_parquet_$(Get-Random)" + $script:parquetFile = Join-Path $script:parquetDir "output.parquet" + } + + AfterAll { + Invoke-DuckDBQuery -Query "DROP TABLE IF EXISTS parquet_src" + Remove-Item $script:parquetDir -Recurse -Force -ErrorAction SilentlyContinue + } + + It "Creates the output file without throwing" { + { Export-DuckDBToParquet -TableName "parquet_src" -OutputPath $script:parquetFile } | Should -Not -Throw + Test-Path $script:parquetFile | Should -Be $true + } + + It "Creates output directory automatically when it does not exist" { + $newFile = Join-Path ([System.IO.Path]::GetTempPath()) "pester_parquet_newdir_$(Get-Random)/out.parquet" + Export-DuckDBToParquet -TableName "parquet_src" -OutputPath $newFile + Test-Path $newFile | Should -Be $true + Remove-Item (Split-Path $newFile -Parent) -Recurse -Force -ErrorAction SilentlyContinue + } + + It "Accepts SNAPPY compression without throwing" { + $f = Join-Path $script:parquetDir "snappy.parquet" + { Export-DuckDBToParquet -TableName "parquet_src" -OutputPath $f -Compression SNAPPY } | Should -Not -Throw + } + + It "Accepts GZIP compression without throwing" { + $f = Join-Path $script:parquetDir "gzip.parquet" + { Export-DuckDBToParquet -TableName "parquet_src" -OutputPath $f -Compression GZIP } | Should -Not -Throw + } + + It "Re-imports the exported Parquet file via DuckDB" { + $data = Get-DuckDBData -Query "SELECT COUNT(*) AS cnt FROM read_parquet('$($script:parquetFile)')" + [int]$data.Rows[0]["cnt"] | Should -Be 3 + } + +} diff --git a/SqlPipeline/Tests/SqlPipeline.Tests.ps1 b/SqlPipeline/Tests/SqlPipeline_SimplySQL.Tests.ps1 similarity index 99% rename from SqlPipeline/Tests/SqlPipeline.Tests.ps1 rename to SqlPipeline/Tests/SqlPipeline_SimplySQL.Tests.ps1 index 992b59b..7c82899 100644 --- a/SqlPipeline/Tests/SqlPipeline.Tests.ps1 +++ b/SqlPipeline/Tests/SqlPipeline_SimplySQL.Tests.ps1 @@ -1,5 +1,5 @@ BeforeAll { - + Write-Host "Hello World 1" Import-Module "$PSScriptRoot/../SqlPipeline" -Force -Verbose # Create a test SQLite connection @@ -46,7 +46,7 @@ Describe "Add-RowsToSql" { } It "Throws if connection is not valid" { - { + { [PSCustomObject]@{ Name = "Test" } | Add-RowsToSql -TableName "FailTable" -SQLConnectionName "notvalid" } | Should -Throw }