Feature: Add Powershell Real Time Pipeline Support (#7)

Fixes #6
Also refactored output processing to be cleaner

Author: JustinGrote

PoshNmap/Private/ConvertFromXml.ps1

@@ -0,0 +1,30 @@
+using namespace Newtonsoft.Json
+function ConvertFromXML {
+#TODO: This requires newtonsoft.json. Present in 6.1 but add a check for 5.1
+<#
+.SYNOPSIS
+Uses the newtonsoft.json library to convert XML into an intermediate object. Supports PSObject (Default) and JSON
+#>
+    [CmdletBinding()]
+    param (
+        #The XML to convert to an object
+        [Parameter(ValueFromPipeline)][XML.XMLElement]$Xml,
+        #Output as JSON instead of just PSObject
+        [Switch]$AsJSON,
+        #Use the raw element processings that newtonsoft.json uses. Only need this if you have conflicting names between your attributes and elements
+        [Switch]$Raw
+    )
+
+    $json = [JsonConvert]::SerializeXmlNode($Xml,'Indented')
+
+    if (-not $Raw) {
+        [Regex]$MatchConvertedAmpersand = '(?m)(?<=\s+\")(@)(?=.+\"\:)'
+        $convertedJson = $json -replace $MatchConvertedAmpersand,''
+    }
+
+
+    if ($AsJson) {return $convertedJson}
+
+    return ConvertFrom-Json $convertedJson
+
+}

PoshNmap/Private/FormatNmapOutput.ps1

@@ -0,0 +1,36 @@
+
+
+function FormatNmapOutput {
+<#
+.SYNOPSIS
+Takes the raw formatting from ConvertFrom-NmapXML and makes a useful Powershell Object out of the output. Meant to be called from ConvertFrom-NmapXml
+.INPUTS
+[PSCustomObject]
+.OUTPUTS
+[PoshNmapResult]
+.NOTES
+The raw formatting is still available as the nmaprun property on the object, to maintain compatibility
+#>
+
+    [CmdletBinding()]
+    param (
+        #Nmaprun output from ConvertFrom-NmapXml. Should basically be XML -> Json -> PSObject output
+        [Parameter(ValueFromPipeline)][PSCustomObject]$InputNmapObject,
+        #Return a summary of the scan rather than individual hosts
+        [Switch]$Summary
+    )
+
+    if (-not $inputNmapObject.nmaprun) {throwUser "This is not a valid Object output from Convert-NmapXML"}
+    $nmaprun = $inputNmapObject.nmaprun
+
+    #Only return a summary if that was requested
+    if ($summary) {return (FormatNmapOutputSummary $nmapRun)}
+
+    #Generate nicer host entries
+    $i=1
+    $itotal = $nmaprun.host | measure | % count
+    foreach ($hostnode in $nmaprun.host) {
+        write-progress -Activity "Parsing NMAP Result" -Status "Processing Scan Entries" -CurrentOperation "Processing $i of $itotal" -PercentComplete (($i/$itotal)*100)
+        FormatPoshNmapHost $hostnode
+    }
+}

PoshNmap/Private/FormatNmapXmlSummary.ps1 …Nmap/Private/FormatNmapOutputSummary.ps1PoshNmap/Private/FormatNmapXmlSummary.ps1 renamed to PoshNmap/Private/FormatNmapOutputSummary.ps1 PoshNmap/Private/FormatNmapXmlSummary.ps1 renamed to PoshNmap/Private/FormatNmapOutputSummary.ps1

@@ -1,5 +1,5 @@
 
-function FormatNmapXmlSummary ($nmapRun) {
+function FormatNmapOutputSummary ($nmapRun) {
 
     #Parse the scanned services list
     foreach ($scanInfoItem in $nmaprun.scaninfo) {

PoshNmap/Private/FormatNmapXml.ps1 PoshNmap/Private/FormatPoshNmapHost.ps1PoshNmap/Private/FormatNmapXml.ps1 renamed to PoshNmap/Private/FormatPoshNmapHost.ps1

@@ -1,37 +1,16 @@
+using namespace Management.Automation
 Update-TypeData -TypeName PoshNmapHost -DefaultDisplayPropertySet IPv4,FQDN,Status,OpenPorts -Force
-
-function FormatNmapXml {
-<#
-.SYNOPSIS
-Takes the raw formatting from ConvertFrom-NmapXML and makes a useful Powershell Object out of the output. Meant to be called from ConvertFrom-NmapXml
-.INPUTS
-[Hashtable]
-.OUTPUTS
-[PoshNmapResult]
-.NOTES
-The raw formatting is still available as the nmaprun property on the object, to maintain compatibility
-#>
-
+function FormatPoshNmapHost {
     [CmdletBinding()]
     param (
-        #Nmaprun output from ConvertFrom-NmapXml. We use hashtable because it's the easiest to manipulate quickly
-        [Parameter(ValueFromPipeline)]$InputNmapXml,
-        #Return a summary of the scan rather than individual hosts
-        [Switch]$Summary
+        [Parameter(ValueFromPipeline)][pscustomobject]$PoshNmapHost
     )
 
-    if (-not $inputNmapXml.nmaprun) {throwUser "This is not a valid Hashtable output from Convert-NmapXML"}
-
-    $nmaprun = $inputNmapXml.nmaprun
-
-    #Only return a summary if that was requested
-    if ($summary) {return (FormatNmapXmlSummary $nmapRun)}
+    process {
+        $hostnode = $PoshNmapHost
 
-    #Generate nicer host entries
-    $i=1
-    $itotal = $nmaprun.host | measure | % count
-    foreach ($hostnode in $nmaprun.host) {
-        write-progress -Activity "Parsing NMAP Result" -Status "Processing Scan Entries" -CurrentOperation "Processing $i of $itotal" -PercentComplete (($i/$itotal)*100)
+        #Deep copy the nmap result so when we output it as the nmapresult property, it is "unchanged"
+        $nmapResult = [PSSerializer]::Serialize($hostnode, [int32]::MaxValue)
 
         # Init variables, with $entry being the custom object for each <host>.
         $service = $null
@@ -46,7 +25,7 @@ The raw formatting is still available as the nmaprun property on the object, to
             MAC = $null
             #Arraylist used for performance as this can get large quickly
             Ports = New-Object Collections.ArrayList
-            OpenPorts = $hostnode.ports | measure | % count
+            OpenPorts = $hostnode.ports.port | measure | % count
         }
         $entry.FQDN = $entry.FQDNs | select -first 1
         $entry.Hostname = $entry.FQDN -replace '^(\w+)\..*$','$1'
@@ -72,12 +51,12 @@ The raw formatting is still available as the nmaprun property on the object, to
             }
             $portResult | FormatStringOut -scriptblock {$this.protocol,$this.port -join ':'}
             $portResult.State | FormatStringOut -scriptblock {$this.state}
-            $portResult.Services | FormatStringOut -scriptblock {($this.name,$this.product -join ':') + " ($($this.conf * 10)%)"}
+            $portResult.Services | FormatStringOut -scriptblock {($this.name,$this.product -join ':') + " ($([int]($this.conf) * 10)%)"}
 
             #TODO: Refactor this now that I'm better at Powershell :)
             # Build Services property. What a mess...but exclude non-open/non-open|filtered ports and blank service info, and exclude servicefp too for the sake of tidiness.
             if ($_.state.state -like "open*" -and ($_.service.tunnel.length -gt 2 -or $_.service.product.length -gt 2 -or $_.service.proto.length -gt 2)) {
-
+                $OutputDelimiter = ', '
                 $entry.Services += ($_.protocol,$_.portid,$service -join ':')+
                     ':'+
                     ($_.service.product,$_.service.version,$_.service.tunnel,$_.service.proto,$_.service.rpcnum -join " ").Trim() +
@@ -107,7 +86,6 @@ The raw formatting is still available as the nmaprun property on the object, to
 
                 $portResult.scriptResult[$scriptItem.id] = [pscustomobject]$scriptResultEntry
             }
-
             $entry.Ports.Add($portResult) > $null
         }
 
@@ -119,16 +97,13 @@ The raw formatting is still available as the nmaprun property on the object, to
         $entry.OSGuesses = $hostnode.os.osmatch
         if (@($entry.OSGuesses).count -lt 1) { $entry.OS = $null }
 
-
         if ($hostnode.hostscript -ne $null) {
             $hostnode.hostscript.script | foreach-object {
                 $entry.Script += '<HostScript id="' + $_.id + '">' + $OutputDelimiter + ($_.output.replace("`n","$OutputDelimiter")) + "$OutputDelimiter</HostScript> $OutputDelimiter $OutputDelimiter"
             }
         }
-        $i++  #Progress counter...
 
-        #Add raw host reference
-        $entry.nmapResult = $hostnode
+        $entry.NmapResult = [PSSerializer]::Deserialize($nmapResult)
 
         [PSCustomObject]$entry
     }

PoshNmap/Private/InvokeNmapExe.ps1

@@ -0,0 +1,44 @@
+function InvokeNmapExe ($argumentList,$computerName) {
+<#
+.SYNOPSIS
+Starts Nmap with supplied argument list and computername list
+.NOTES
+This function primarily exists to be mocked by Pester
+#>
+    $nmapexe = (Get-Command nmap).path
+    & "$nmapexe" $argumentList $computerName
+}
+
+function StartNmap ($argumentList,$computerName,[int]$updateInterval=200,[Switch]$Raw) {
+    [Collections.Arraylist]$nmapExeOutput = New-Object Collections.ArrayList
+    [Collections.Arraylist]$hostEntry = New-Object Collections.ArrayList
+    if (-not $Raw) {
+        $ArgumentList += "-oX","-",'--stats-every',"${updateInterval}ms"
+    }
+
+    write-verbose "Invoking nmap $argumentList $computerName"
+    InvokeNmapExe $argumentList $computerName | Foreach-Object {
+        if ($Raw) {$PSItem} else {
+            #Strip taskprogress items and report them as progress instead
+            if ($PSItem -match '^<taskprogress') {
+                $taskprogress = ([xml]$PSItem).taskprogress
+                #write-debug "Task $($taskProgress.task) is $([int]($taskProgress.percent))% complete with $($taskProgress.remaining) items left"
+
+                $ETA = [TimeSpan]::FromSeconds($taskProgress.etc - $taskProgress.time)
+                $WriteProgressParams = @{
+                    Id=10
+                    Activity = "Invoke-Nmap Scan of $($computername -join ',')"
+                    Status = "$($taskProgress.task)"
+                    CurrentOperation = "$($taskProgress.remaining) items remaining. ETA $ETA"
+                    PercentComplete = $taskProgress.percent
+                }
+                Write-Progress @WriteProgressParams
+            } else {
+                $PSItem
+            }
+        }
+    }
+
+    write-progress -id 10 -Activity 'Invoke-Nmap Scan' -Completed
+}
+

PoshNmap/Private/StartNmap.ps1

@@ -4,68 +4,89 @@ function ConvertFrom-NmapXml {
 <#
 .SYNOPSIS
 Converts NmapXML into various formats. Currently supported are JSON, PSObject, NmapReport
+.NOTES
+Only supports nmap reports piped from nmap directly. In the future will support existing full nmap reports
 .EXAMPLE
 nmap localhost -oX - | ConvertFrom-NmapXML -OutFormat JSON
 Takes an NMAP run output and converts it into JSON
-
 #>
-    [CmdletBinding(DefaultParameterSetName="String")]
+    [CmdletBinding()]
     param (
-        #The NMAPXML content
-        [Parameter(ParameterSetName='String',ValueFromPipeline)][String[]]$InputString,
-        [Parameter(ParameterSetName='XML',ValueFromPipeline)][XML[]]$InputObject,
+        #Reads XML "Strings" one by one
+        [Parameter(ValueFromPipeline)][String[]]$InputObject,
 
         #Choose the format that you want to convert the NMAPXML to. Valid options are: JSON or HashTable
         [ValidateSet('JSON','HashTable','PSObject','PoshNmap','Summary')]
-        $OutFormat = 'JSON'
+        $OutFormat = 'PoshNmap'
     )
 
-    process {
-        #If strings were passed via pipeline, assume it is output from nmap XML which is multiple lines and coalesce them into one large document.
-        $InputObjectBundle += $InputString
+    begin {
+        $xmlDocument = [Collections.ArrayList]@()
+        $hostEntry = [Collections.ArrayList]@()
     }
 
-    end {
-        try {
-            [XML]$CombinedDocument = $InputObjectBundle
-        } catch [InvalidCastException] {
-            $exception = [System.Management.Automation.PSInvalidCastException]::New("The input provided is not valid XML. If you are piping from nmap, did you use 'nmap -oX -'?")
-            throwUser $exception
-        }
+    process {
+        #Unwrap $InputObject if it was passed as an array
+        foreach ($nmapLineItem in $inputObject) {
+            #If the output format is not PoshNmap, we will coalesce into a single document and process at the end, otherwise we will do it in real time for the pipeline
+            if ($OutFormat -ne 'PoshNmap') {
+                $xmlDocument += $nmapLineItem
+            #If this is a host entry, start capturing a host buffer
+            } elseif ($nmapLineItem -match '^<host') {
+                $hostEntry += $nmapLineItem
+            } elseif ($hostentry.count -ge 1) {
+                if ($nmapLineItem -match '^</host>$') {
+                    $hostEntry += $nmapLineItem
+                    try {
+                        (ConvertFromXml ([xml]$hostEntry).host).host | FormatPoshNmapHost
+                    } finally {
+                        $hostEntry = [Collections.ArrayList]@()
+                    }
+                } else {
+                    $hostEntry += $nmapLineItem
+                }
+            }
 
-        if ($CombinedDocument) {$inputObject = $CombinedDocument}
-        $jsonResult = foreach ($nmapXmlItem in $InputObject) {
+            #If we are making a host entry, keep adding lines until we hit a </host> entry and then process it
 
-            #Selecting NmapRun required for PS5.1 compatibility due to Newtonsoft.Json bug
-            $nmapRunItem = ($nmapXmlItem).SelectSingleNode('nmaprun')
+        }
+    }
 
-            #Indented JSON is important as we will use a regex to clean up the @ elements
-            $convertedJson = [JsonConvert]::SerializeXmlNode($nmapRunItem,'Indented')
+    end {
+        #If we don't have any post-processing, don't worry about it
+        if (-not $xmlDocument) {continue}
 
-            #Remove @ symbols from xml attributes. There are no element/attribute collisions in the nmap xml (that we know of) so this should be OK.
-            [Regex]$MatchConvertedAmpersand = '(?m)(?<=\s+\")(@)(?=.+\"\:)'
-            $convertedJson = $convertedJson -replace $MatchConvertedAmpersand,''
-            $convertedJson
+        if ($xmlDocument -isnot [xml]) {
+            try {
+                $xmlDocument = [XML]$xmlDocument
+            } catch [InvalidCastException] {
+                $exception = [System.Management.Automation.PSInvalidCastException]::New("The input provided is not valid XML. If you are piping from nmap, did you use 'nmap -oX -'?")
+                throwUser $exception
+            }
+        }
+
+        if (-not $xmlDocument.nmaprun) {
+            throwUser "The provided document is not a valid NMAP XML document (doesn't have an nmaprun element)"
         }
 
+        $nmapRun = $xmlDocument.selectSingleNode('nmaprun')
+
         switch ($OutFormat) {
             'JSON' {
-                return $jsonResult
+                ConvertFromXml $nmapRun -AsJSON
             }
             'PSObject' {
-                return $jsonResult | ConvertFrom-Json
+                (ConvertFromXml $nmapRun).nmaprun
             }
-            'HashTable' {
-                #TODO: PSCore Method, add as potential feature flag but for now use same method for both to avoid incompatibilities
-                #$jsonResult | ConvertFrom-Json -AsHashtable
-                return $jsonResult | ConvertFrom-Json | ConvertPSObjectToHashtable
+            'Summary' {
+                FormatNmapOutputSummary -nmaprun (ConvertFromXml $nmapRun).nmaprun
             }
-            'PoshNmap' {
-                return $jsonResult | ConvertFrom-Json | FormatNmapXml
+            'HashTable' {
+                (ConvertFromXml $nmapRun).nmaprun | ConvertPSObjectToHashtable
             }
-            'Summary' {
-                return $jsonResult | ConvertFrom-Json | FormatNmapXml -Summary
+            Default {
+                throwUser "Outformat $Outformat is not valid. This should not happen, file as an issue if you see this"
             }
         }
     }
-}
+}