DataParser.psm1

<#
.SYNOPSIS
    This function parses every file in a specified directory. The data from each individual
    file is represented through a HashSet. The Hashtables containing data for each file are 
    stored in an array and returned along with meta data.
.PARAMETER DirName
    Path to the directory with the data files.
.PARAMETER Tool
    Name of the tool that generated the data. (NTTTCP, etc.)
.PARAMETER Mode
    Whether the given directory contains 'Baseline' or 'Test' data
.PARAMETER InnerPivot
    Name of inner pivot property
.PARAMETER OuterPivot
    Name of outer pivot property 
#>
function Get-RawData {
    param (
        [Parameter(Mandatory=$true)] [String] $DirName, 
        [Parameter()] [String] $Tool,
        [Parameter()] [String] $Mode="Baseline",
        [Parameter()] [String] $InnerPivot,
        [Parameter()] [String] $OuterPivot,
        [Parameter()] [Int] $Warmup,
        [Parameter()] [Int] $Cooldown
    )

    $output = @{}
    $files = Get-ChildItem -File $DirName
    if ($files.Count -eq 0) {
        Throw "'$DirName' does not contain any data files."
    }

    switch ($Tool) {
        "NTTTCP" {
            $parseFunc = ${Function:Parse-NTTTCP}

            $output."meta" = @{
                "props" = [Array] @(
                    "throughput",
                    "cycles/byte"
                )
                "units" = @{
                    "cycles/byte"     = "cycles/byte"
                    "throughput" = "Gbps"
                }
                "goal" = @{
                    "throughput" = "increase"
                    "cycles/byte"     = "decrease"
                }
                "format" = @{
                    "throughput" = "0.00"
                    "cycles/byte"     = "0.00"
                }
                "noTable"  = [Array] @("filename", "sessions", "bufferLen", "bufferCount")
            }
        }
        "LATTE" {
            $parseFunc = ${Function:Parse-LATTE}

            $output."meta" = @{
                "props" = [Array] @(
                    "latency"
                )
                "units" = @{
                    "latency"  = "us"
                }
                "goal" = @{
                    "latency"  = "decrease"
                }
                "format" = @{
                    "latency"  = "#.0"
                }
                "noTable"  = [Array]@("filename", "sendMethod", "protocol")
            }
        }
        "LagScope" {
            $parseFunc = ${Function:Parse-LagScope}

            $output."meta" = @{
                "props" = [Array] @(
                    "latency"
                )
                "units" = @{
                    "latency"  = "us"
                }
                "goal" = @{
                    "latency"  = "decrease"
                }
                "format" = @{
                    "latency"  = "#.0"
                }
                "noTable"  = [Array]@("filename", "sendMethod", "protocol")
            }
        }
        "CTStraffic" {
            $parseFunc = ${Function:Parse-CTSTraffic}

            $output."meta" = @{
                "props" = [Array] @(
                    "throughput"
                )
                "units" = @{
                    "throughput" = "Gbps"
                }
                "goal" = @{
                    "throughput" = "increase"
                }
                "format" = @{
                    "throughput" = "0.00"
                }
                "noTable"  = [Array]@("filename", "sessions")
            }
        }
        "CPS" {
            $parseFunc = ${Function:Parse-CPS}

            $output."meta" = @{
                "props" = [Array] @(
                    "conn/s",
                    "close/s"
                )
                "units" = @{
                    "conn/s" = ""
                    "close/s" = ""  
                }
                "goal" = @{
                    "conn/s" = "increase"
                    "close/s" = "increase"    
                }
                "format" = @{ 
                    "conn/s" = "0.0"
                    "close/s" = "0.0"
                }
                "noTable" = [Array]@("filename")
            }
        }
    } 

    $PathCosts = @{
        "meta" = @{
            "props" = @()
        }
    }
    $InnerPivotKeys = @{}
    $OuterPivotKeys = @{}

    $id = if ($Mode -eq "Baseline") {0} else {1}
    $output.data = [Array]@()  
    for($i = 0; $i -lt $files.Count; $i++) { 
        Write-Progress -Activity "Parsing $($Mode) Data Files..." -Status "Parsing..." -Id $id -PercentComplete (100 * (($i) / $files.Count))
        $output.data += , (& $parseFunc -FileName $files[$i].FullName -InnerPivot $InnerPivot -OuterPivot $OuterPivot `
                            -InnerPivotKeys $InnerPivotKeys  -OuterPivotKeys $OuterPivotKeys -PathCosts $PathCosts -Warmup $Warmup -Cooldown $Cooldown)
        if (-not $output.data[-1]) {
            $output.data = $output.data[0..($output.data.Count - 1)] 
        } 
    }

    if ($Tool -in @("CTStraffic", "NTTTCP")) {
        if ($PathCosts.Count -gt 0) {
            # This can be expanded to include the other metrics captured by the pathcosts tool
            Incorporate-PathCosts -Data $output.data -PathCosts $PathCosts   

            if ($PathCosts.meta.props -contains "total root VP utilization") {
                $output.meta.props += "total root VP utilization"
                $output.meta.goal["total root VP utilization"] = "decrease"
                $output.meta.format["total root VP utilization"] = "0.00"
                $output.meta.units["total root VP utilization"] = "% Utilization"
            }
            
            if ($PathCosts.meta.props -contains "vSwitch root VP utilization") {
                $output.meta.props += "vSwitch root VP utilization"
                $output.meta.goal["vSwitch root VP utilization"] = "decrease"
                $output.meta.format["vSwitch root VP utilization"] = "0.00"
                $output.meta.units["vSwitch root VP utilization"] = "% Utilization"
            }

            if ($PathCosts.meta.props -contains "CPU Utilization") { 
                $output.meta.props += "CPU Utilization"
                $output.meta.goal["CPU Utlization"] = "decrease"
                $output.meta.format["CPU Utlization"] = "0.00"
                $output.meta.units["CPU Utlization"] = "% Utilization"
            }

            if ($PathCosts.meta.props -contains "cycles/packet") { 
                $output.meta.props += "cycles/packet"
                $output.meta.goal["cycles/packet"] = "decrease"
                $output.meta.format["cycles/packet"] = "0.00"
            }

            if ($PathCosts.meta.props -contains "cycles/byte") {
                $output.meta.props += "cycles/byte"
                $output.meta.goal["cycles/byte"] = "decrease"
                $output.meta.format["cycles/byte"] = "0.00"
            }

            if ($PathCosts.meta.props -contains "RSS Core Utilization") { 
                $output.meta.props += "RSS Core Utilization"
                $output.meta.goal["RSS Core Utilization"] = "decrease"
                $output.meta.format["RSS Core Utilization"] = "0.00"
                $output.meta.units["cpu utlization"] = "% Utilization"
            }

            if ($PathCosts.meta.props -contains "UVMS Data Core Utilization") { 
                $output.meta.props += "UVMS Data Core Utilization"
                $output.meta.goal["UVMS Data Core Utilization"] = "decrease"
                $output.meta.format["UVMS Data Core Utilization"] = "0.00"
                $output.meta.units["cpu utlization"] = "% Utilization"
            }
        }
        
    }

    Write-Progress -Activity "Parsing $($Mode) Data Files..." -Status "Done" -Id $id -PercentComplete 100
 
    if ($output."data".Count -eq 0) {
        Write-Error "Failed to parse any files in '$DirName'."
    }

    $output.meta.innerPivotKeys = $InnerPivotKeys.Keys 
    $output.meta.outerPivotKeys = $OuterPivotKeys.Keys 

    return $output
}

<#
.SYNOPSIS
    Reads data from HashTable containing pathcost data and writes the values to the DataEntry 
    objects corresponding to each individual file.

.PARAMETER Data
    Array of DataEntry objects which each correspond to a single data file.
.PARAMETER PathCosts
    Hashtable containing a mapping between data filenames and pathcosts data
#>
function Incorporate-PathCosts ($Data, $PathCosts) {
    foreach ($entry in $Data) {
        $file = $entry.filename.Split("\")[-1]
        if ($PathCosts.ContainsKey($file)) {
            
            $cpb = $PathCosts[$file]["cycles/byte"]
            $cpu = $PathCosts[$file]["CPU Utilization"]
            $cpp = $PathCosts[$file]["cycles/packet"]
            $trvp = $PathCosts[$file]["Total Root VP Utilization"]
            $vsrvp = $PathCosts[$file]["vSwitch Root VP Utilization"]
            $udcu = $PathCosts[$file]["UVMS Data Core Utilization"]
            $rcu = $PathCosts[$file]["RSS Core Utilization"]

            # TPUT measures from dedicated tools are more reliable than vswitch counters 
            # (which sometimes return 0 erroneously), thus we perform the cycles/byte calculation
            # using TPUT measures from ntttcp/ctstraffic when possible
            if ($PathCosts[$file].ContainsKey("Total CPU cycles used per second")) {
                $avgTput = ((1000 * 1000 * 1000) / 8) * ($entry["throughput"] | Measure-Object -Average).Average
                if ($avgTput -ne 0) {
                    $cpb = $PathCosts[$file]["Total CPU cycles used per second"] / $avgTput
                }

                if ("cycles/byte" -notin $PathCosts.meta.props) {
                    $PathCosts.meta.props += "cycles/byte"
                }

            }

            # Sometimes counters mess up and record nearly-zero for tput and it causes 
            # cycle/byte calculations to return extremely large numbers. These outliers
            # make visualizations unusable, thus we filter outliers here. We 
            # should look for a more sustainable solution in the future
            
            if ($cpb -lt 100 -and $cpb -gt 0) {
                $entry["cycles/byte"] = $cpb
            } 

            if ($vsrvp -and $vsrvp -le 100) { 
                $entry["vSwitch root VP utilization"] = $vsrvp
            }

            if ($udcu) { 
                $entry["UVMS Data Core Utilization"] = $udcu 
            }

            if ($rcu) { 
                $entry["RSS Core Utilization"] = $rcu 
            }

            if ($cpp -gt 0) {
                $entry["cycles/packet"] = $cpp
            }
            if ($cpu -gt 0) {
                $entry["CPU Utilization"] = $cpu
            }
            if ($trvp -gt 0) {
                $entry["total root VP utilization"] = $trvp
            }
            
        }
    }
}


<#
.SYNOPSIS
    Parses a single XML-formated NTTTCP output data file. Relevant data is collected and returned
    as a Hashtable.
.PARAMETER FileName
    Path of file to be parsed. 
.PARAMETER InnerPivotKeys
    Set containing all inner pivot keys encountered across all data files
.PARAMETER OuterPivotKeys
    Set containing all outer pivot keys encountered across all data files
.PARAMETER InnerPivot
    Name of inner pivot property
.PARAMETER OuterPivot
    Name of outer pivot property
#>
function Parse-NTTTCP ([String] $FileName, $InnerPivot, $OuterPivot, $InnerPivotKeys, $OuterPivotKeys, $PathCosts, $Warmup, $Cooldown) {
    if ($Filename -match "pathcost") {
        Extract-PathCosts -Filename $Filename -PathCosts $PathCosts 
        return
    }
    
    if ($FileName -notlike "*.xml") {
        return
    }

    $file = (Get-Content $FileName) -as [XML]
    if (-not $file) {
        Write-Warning "Skipped '$FileName' because it is not valid XML."
        return
    }

    [Decimal] $cycles = $file.ChildNodes.cycles.'#text'
    [Decimal] $throughput = ($file.ChildNodes.throughput | where {$_.metric -eq "mbps"})."#text" / 1000
    [Int] $sessions = [Int] $filename.split("\")[-1].split(".")[2].substring(1) #should this be .num_processors or .parametes.max_active_threads?
    [Int] $bufferLen = $file.ChildNodes.bufferLen
    [Int] $bufferCount = $file.ChildNodes.io

    $dataEntry = @{
        "sessions"    = $sessions
        "throughput"  = $throughput
        "cycles/byte" = $cycles
        "filename"    = $FileName
        "bufferLen"   = $bufferLen
        "bufferCount" = $bufferCount
    }

    $iPivotKey = if ($dataEntry[$InnerPivot]) {$dataEntry[$InnerPivot]} else {""}
    $oPivotKey = if ($dataEntry[$OuterPivot]) {$dataEntry[$OuterPivot]} else {""}

    $InnerPivotKeys[$iPivotKey] = $true
    $OuterPivotKeys[$oPivotKey] = $true

    return $dataEntry
}

<#
.SYNOPSIS
    Parses a single pathcosts data file, extracts relevant data, and stores the data in a 
    HashTable.
.PARAMETER Filename
    Path of file to be parsed
.PARAMETER PathCosts
    Hashtable to which pathcosts data is written
#>
function Extract-PathCosts ($Filename, $PathCosts) {
    (Get-Content -Path $Filename | ConvertFrom-Json).psobject.properties | Foreach {
        $key = $_.Name     
        $values = @{}
        $obj = $_.Value 
        if ($_.Value[1]) {
            $obj = $_.Value[1]
        }
        $obj.psobject.properties | Foreach {
            try {
                $propName = $_.Name
                if ($propName -Match ".*\((.*)\).*") {
                    $propName = $Matches[1]
                }
                $value = [Decimal]$_.Value
                if ($value -gt 0) {
                    if ($propName -notin $PathCosts.meta.props) {
                        $PathCosts.meta.props += $propName
                    }
                    $values[$propName] = $value
                }
                 
            } catch {}
        }
        $PathCosts[$key] = $values
    }
}
 

<#
.SYNOPSIS
    This function parses a single CTStraffic status log file. Desired data is collected and returned
    as a Hashtable.
.PARAMETER Filename
    Path of the status log file to parse.
.PARAMETER InnerPivotKeys
    Set containing all inner pivot keys encountered across all data files
.PARAMETER OuterPivotKeys
    Set containing all outer pivot keys encountered across all data files
.PARAMETER InnerPivot
    Name of inner pivot property
.PARAMETER OuterPivot
    Name of outer pivot property
#>
function Parse-CTStraffic ( [String] $Filename, $InnerPivot, $OuterPivot , $InnerPivotKeys, $OuterPivotKeys, $PathCosts, $Warmup, $Cooldown) {

    if ($Filename -match "pathcost") {
        Extract-PathCosts -Filename $Filename -PathCosts $PathCosts 
        return
    }

    $data = (Get-Content $Filename) -replace '^"|"$','' | ConvertFrom-Csv


    if (-not ($data | Get-Member -Name "In-Flight" -ErrorAction "SilentlyContinue")) {
        Write-Warning "Skipped '$Filename' because it's not a valid ctsTraffic status log. Please verify that it was generated by the -StatusFilename option."
        return
    }

    $fileSplit = $Filename.Split("\")[-1]
    $fileSplit = $fileSplit.Split("/")[-1]
    $sessionsStr = $fileSplit.Split(".")[2] 
    $bytesToGigabits = [Decimal] 8 / (1000 * 1000 * 1000)

    $throughput = [Array]@()
    $warmupPadding = $Warmup
    $dynamicWarmup = $true
    $dynamicCooldown = $true 
    $cooldownPadding = $Cooldown

    $avgSend = ($data.SendBps | Measure-Object -Average).Average
    $avgRecv = ($data.RecvBps | Measure-Object -Average).Average
    $tputProp = if ($avgSend -gt $avgRecv) {"SendBps"} else {"RecvBps"} 

    for ($i = $warmupPadding; $i -lt $data.Count - $cooldownPadding; $i += 1) { 

        $tputVal = $data[$i].$tputProp
        if ($tputVal -eq 0 -and $dynamicWarmup) {
            continue
        } else {
            $dynamicWarmup = $false
        }
        $throughput += [Decimal] $tputVal * $bytesToGigabits
    }  #($data.SendBps | measure -Average).Average * $bytesToGigabits

    if ($dynamicCooldown) {
        for ($j = $throughput.Length - 1; $j -ge 0; $j -= 1) {
            if ($throughput[$j] -eq 0) {
                $throughput = $throughput[0..($j-1)] 
            } else {
                break
            }
        }
    }

    $maxSessions = ($data."In-Flight" | Measure-Object -Max).Maximum
    
    $sessionsStr = $fileSplit.Split(".")[2]
    $dataEntry = @{
        "sessions"   =  [Int] $maxSessions
        "throughput" = $throughput
        "filename"   = $Filename
    }

    $iPivotKey = if ($dataEntry[$InnerPivot]) {$dataEntry[$InnerPivot]} else {""}
    $oPivotKey = if ($dataEntry[$OuterPivot]) {$dataEntry[$OuterPivot]} else {""}

    $InnerPivotKeys[$iPivotKey] = $true
    $OuterPivotKeys[$oPivotKey] = $true

    return $dataEntry
}

 
<#
.SYNOPSIS
    This function parses a single LATTE data file. Relevant data is collected and returned
    as a Hashtable.
.PARAMETER Filename
    Path of the status log file to parse.
.PARAMETER InnerPivotKeys
    Set containing all inner pivot keys encountered across all data files
.PARAMETER OuterPivotKeys
    Set containing all outer pivot keys encountered across all data files
.PARAMETER InnerPivot
    Name of inner pivot property
.PARAMETER OuterPivot
    Name of outer pivot property
#>
function Parse-LATTE ([string] $FileName, $InnerPivot, $OuterPivot, $InnerPivotKeys, $OuterPivotKeys, $PathCosts, $Warmup, $Cooldown) {

    if ($Filename -match "pathcost") {
        Extract-PathCost -Filename $Filename -PathCosts $PathCosts 
        return
    }

    $file = Get-Content $FileName

    $dataEntry = @{
        "filename" = $FileName
    }

    $splitline = Remove-EmptyStrings -Arr (([Array]$file)[0]).split(' ')
    if ($splitline[0] -eq "Protocol") {
        $histogram = $false

        foreach ($line in $file) {
            $splitLine = Remove-EmptyStrings -Arr $line.split(' ')

            if ($splitLine.Count -eq 0) {
                continue
            }

            if ($splitLine[0] -eq "Protocol") {
                $dataEntry.protocol = $splitLine[-1]
            }
            if ($splitLine[0] -eq "MsgSize") {
                $dataEntry.msgSize = $splitLine[-1]  # Not currently used for anything
            }

            if ($splitLine[0] -eq "Interval(usec)") {
                $dataEntry.latency = [HashTable] @{} 
                $histogram = $true
                continue
            }

            if ($histogram) {
                $dataEntry.latency[[Int32]$splitLine[0]] = [Int32] $splitLine[-1]
            }
        }

        if (-not $histogram) {
            Write-Warning "No histogram in file $filename"
            return
        }

    } 
    else {
        
        [Array] $latency = @()
        foreach ($line in $file) {

            if (-not ($line -match "\d+")) {
                Write-Warning "Error Parsing file $FileName"
                return
            }
            $latency += ,[int]$line
        }
        $dataEntry.latency = $latency[$Warmup..($latency.Count - ($Cooldown + 1))]
        $dataEntry.protocol = (($FileName.Split('\'))[-1].Split('.'))[0].ToUpper()
    }

    $dataEntry.sendMethod = (($FileName.Split('\'))[-1].Split('.'))[2]

    $iPivotKey = if ($dataEntry[$InnerPivot]) {$dataEntry[$InnerPivot]} else {""}
    $oPivotKey = if ($dataEntry[$OuterPivot]) {$dataEntry[$OuterPivot]} else {""}

    $InnerPivotKeys[$iPivotKey] = $true
    $OuterPivotKeys[$oPivotKey] = $true
    return $dataEntry
}

<#
.SYNOPSIS
    This function parses a single LagScope data file. Relevant data is collected and returned
    as a Hashtable. 
.PARAMETER Filename
    Path of the data file to parse.
.PARAMETER InnerPivotKeys
    Set containing all inner pivot keys encountered across all data files
.PARAMETER OuterPivotKeys
    Set containing all outer pivot keys encountered across all data files
.PARAMETER InnerPivot
    Name of inner pivot property
.PARAMETER OuterPivot
    Name of outer pivot property
#>
function Parse-LagScope ([string] $FileName, $InnerPivot, $OuterPivot, $InnerPivotKeys, $OuterPivotKeys, $PathCosts, $Warmup, $Cooldown) {
    
    if ($Filename -match "pathcost") {
        Extract-PathCost -Filename $Filename -PathCosts $PathCosts 
        return
    }

    if ($FileName -notlike "*.json") {return}
    
    $latencies = (Get-Content $Filename | ConvertFrom-Json).latencies 
    $dataEntry = @{
        "filename" = $FileName
        "latency" = [Array] @()
    }

    for ($i = 0; $i -lt $latencies.Count; $i++) {
        $dataEntry.latency += @($latencies[$i].latency) * $latencies[$i].frequency
    }
    
    $iPivotKey = if ($dataEntry[$InnerPivot]) {$dataEntry[$InnerPivot]} else {""}
    $oPivotKey = if ($dataEntry[$OuterPivot]) {$dataEntry[$OuterPivot]} else {""}

    $InnerPivotKeys[$iPivotKey] = $true
    $OuterPivotKeys[$oPivotKey] = $true

    return $dataEntry
} 
 

<#
.SYNOPSIS 
    This function parses a single file containing CPS data. Each line contains
    conn/s and close/s samples which are extracted into arrays, packaged into a 
    HashTable, and returned. 
.PARAMETER Filename
    Path of the status log file to parse.
.PARAMETER InnerPivotKeys
    Set containing all inner pivot keys encountered across all data files
.PARAMETER OuterPivotKeys
    Set containing all outer pivot keys encountered across all data files
.PARAMETER InnerPivot
    Name of inner pivot property
.PARAMETER OuterPivot
    Name of outer pivot property
#>
function Parse-CPS ([string] $FileName, $InnerPivot, $OuterPivot, $InnerPivotKeys, $OuterPivotKeys, $PathCosts, $Warmup, $Cooldown) {
    if ($Filename -match "pathcost") {
        Extract-PathCost -Filename $Filename -PathCosts $PathCosts 
        return
    }

    $file = Get-Content $FileName

    $dataEntry = @{
        "filename" = $FileName
        "conn/s" = [Array] @()
        "close/s" = [Array] @()
    }

    foreach ($line in $file[1..($file.Count - 1)]) {
        $splitLine = Remove-EmptyStrings -Arr $line.split(' ')
         
        if ($splitLine.Count -eq 0) {
            break
        }

        $dataEntry."conn/s" += ,[Decimal]($splitLine[5])
        $dataEntry."close/s" += ,[Decimal]($splitLine[6]) 
    } 

    $iPivotKey = if ($dataEntry[$InnerPivot]) {$dataEntry[$InnerPivot]} else {""}
    $oPivotKey = if ($dataEntry[$OuterPivot]) {$dataEntry[$OuterPivot]} else {""}

    $InnerPivotKeys[$iPivotKey] = $true
    $OuterPivotKeys[$oPivotKey] = $true
    
    return $dataEntry
}


<#
.SYNOPSIS
    This function removes all empty strings from the given array
.PARAMETER Arr
    Array of strings 
#>
function Remove-EmptyStrings ($Arr) {
    $newArr = [Array] @()
    foreach ($val in $arr) {
        $trimVal = $val.Trim()
        if ($trimVal -ne "") {
            $newArr += $trimVal
        }
    }
    return $newArr
}