Bug fixes, performance, and reliability improvements (#65)

* update role assignment collection logic

* Update pbit to match pwsh output

* Fix module version check

* Fix bug on tenants with no P2 licenses

* Fix warning on tenants with no AAD licenses

* Remove resource not found warnings

* Remove auth method registration warning

* Remove posh test gallery from cd pipeline

* Stop auto trigger of CD pipeline on preview branch

* Test break on error

* Add ScriptStackTrace to exception telemetry

* Add data to exception telemetry

* Remove old comments

* update ms graph batching and telemetry

* update mg calls to v1.0 and general clean up

* Add ordereddictionary param to all appinsight func

* Fix error when run as different user

* Move scopes definition to module variable

* Add comments and fix error retry

* Add warning to upgrade PowerShell version

* Remove WinPoSh warning until device comp support

* Additional telemetry for troubleshooting

* Add retry for Timeout exception

* Fix relative uri code path

* limit downloading membership of large groups

* Fix RequiredAssemblies on publish, find type error

* Update retry logic and progress bar text

* shorten progress bar activity text

* fix assignedPlans filter

* move zip assembly load from manifest to script

build/Launch-PSModule.ps1

@@ -18,6 +18,7 @@ param
 )
 
 if ($NoNewWindow) {
+    #$ErrorActionPreference = [System.Management.Automation.ActionPreference]::Break
     Import-Module $ModuleManifestPath -PassThru -Force
     if ($PostImportScriptBlock) { Invoke-Command -ScriptBlock $PostImportScriptBlock -NoNewScope }
 }
@@ -26,6 +27,7 @@ else {
         param ([string]$ModulePath, [scriptblock]$PostImportScriptBlock)
         ## Force WindowsPowerShell to load correct version of built-in modules when launched from PowerShell 6+
         if ($PSVersionTable.PSEdition -eq 'Desktop') { Import-Module 'Microsoft.PowerShell.Management', 'Microsoft.PowerShell.Utility', 'CimCmdlets' -MaximumVersion 5.9.9.9 }
+        #$ErrorActionPreference = [System.Management.Automation.ActionPreference]::Break
         Import-Module $ModulePath -PassThru -ArgumentList @{
             'ai.instrumentationKey' = 'f7c43a96-9493-41e3-ad62-4320f5835ce2'
         }

build/Update-PSModuleManifest.ps1

@@ -12,22 +12,23 @@ param
     # Indicates the module is prerelease.
     [Parameter(Mandatory = $false)]
     [string] $Prerelease,
-    # Skip Update of RequiredAssemblies
+    # Skip automatic additions to RequiredAssemblies from module file list.
     [Parameter(Mandatory = $false)]
-    [switch] $SkipRequiredAssemblies
+    [switch] $SkipRequiredAssembliesDetection
 )
 
 ## Initialize
 Import-Module "$PSScriptRoot\CommonFunctions.psm1" -Force -WarningAction SilentlyContinue -ErrorAction Stop
 [hashtable] $paramUpdateModuleManifest = @{ }
 if ($Guid) { $paramUpdateModuleManifest['Guid'] = $Guid }
 if ($ModuleVersion) { $paramUpdateModuleManifest['ModuleVersion'] = $ModuleVersion }
-if ($Prerelease) { $paramUpdateModuleManifest['Prerelease'] = $Prerelease }
 
 [System.IO.FileInfo] $ModuleManifestFileInfo = Get-PathInfo $ModuleManifestPath -DefaultFilename "*.psd1" -ErrorAction Stop
 
 ## Read Module Manifest
 $ModuleManifest = Import-PowerShellDataFile $ModuleManifestFileInfo.FullName
+if ($ModuleManifest.PrivateData.PSData['Prerelease'] -eq 'source') { $paramUpdateModuleManifest['Prerelease'] = "" }
+if ($Prerelease) { $paramUpdateModuleManifest['Prerelease'] = $Prerelease }
 if ($ModuleManifest.NestedModules) { $paramUpdateModuleManifest['NestedModules'] = $ModuleManifest.NestedModules }
 $paramUpdateModuleManifest['FunctionsToExport'] = $ModuleManifest.FunctionsToExport
 $paramUpdateModuleManifest['CmdletsToExport'] = $ModuleManifest.CmdletsToExport
@@ -43,10 +44,13 @@ $ModuleFileList = Get-RelativePath $ModuleFileListFileInfo.FullName -WorkingDire
 $ModuleFileList = $ModuleFileList -replace '\\net45\\', '\!!!\' -replace '\\netcoreapp2.1\\', '\net45\' -replace '\\!!!\\', '\netcoreapp2.1\'  # PowerShell Core fails to load assembly if net45 dll comes before netcoreapp2.1 dll in the FileList.
 $paramUpdateModuleManifest['FileList'] = $ModuleFileList
 
-if (!$SkipRequiredAssemblies -and $ModuleRequiredAssembliesFileInfo) {
+## Generate RequiredAssemblies list based on existing items and file list
+$paramUpdateModuleManifest['RequiredAssemblies'] += $ModuleManifest.RequiredAssemblies | Where-Object { $_ -notin $ModuleFileListFileInfo.Name }
+if (!$SkipRequiredAssembliesDetection -and $ModuleRequiredAssembliesFileInfo) {
     $ModuleRequiredAssemblies = Get-RelativePath $ModuleRequiredAssembliesFileInfo.FullName -WorkingDirectory $ModuleOutputDirectoryInfo.FullName -ErrorAction Stop
-    $paramUpdateModuleManifest['RequiredAssemblies'] = $ModuleRequiredAssemblies
+    $paramUpdateModuleManifest['RequiredAssemblies'] += $ModuleRequiredAssemblies
 }
+if (!$paramUpdateModuleManifest['RequiredAssemblies']) { $paramUpdateModuleManifest.Remove('RequiredAssemblies') }
 
 ## Clear RequiredAssemblies
 (Get-Content $ModuleManifestFileInfo.FullName -Raw) -replace "(?s)RequiredAssemblies\ =\ @\([^)]*\)", "# RequiredAssemblies = @()" | Set-Content $ModuleManifestFileInfo.FullName

build/azure-pipelines/azure-pipelines-cd.yml

@@ -9,7 +9,7 @@ resources:
       branches:
         include:
         - master
-        - preview
+        #- preview
 
 parameters:
 - name: vmImage

build/azure-pipelines/template-psmodule-build.yml

@@ -81,7 +81,7 @@ steps:
   displayName: 'Update PowerShell Module Manifest'
   inputs:
     filePath: '$(System.DefaultWorkingDirectory)/build/Update-PSModuleManifest.ps1'
-    arguments: '-ModuleManifestPath "$(Pipeline.Workspace)/${{ parameters.artifactOutput }}/${{ coalesce(parameters.moduleRename,parameters.moduleName) }}/${{ coalesce(parameters.moduleRename,parameters.moduleName) }}.psd1" -Guid "${{ parameters.moduleGuid }}" -ModuleVersion "${{ parameters.moduleVersion }}" -Prerelease "${{ parameters.prereleaseTag }}" -SkipRequiredAssemblies'
+    arguments: '-ModuleManifestPath "$(Pipeline.Workspace)/${{ parameters.artifactOutput }}/${{ coalesce(parameters.moduleRename,parameters.moduleName) }}/${{ coalesce(parameters.moduleRename,parameters.moduleName) }}.psd1" -Guid "${{ parameters.moduleGuid }}" -ModuleVersion "${{ parameters.moduleVersion }}" -Prerelease "${{ parameters.prereleaseTag }}"'
     pwsh: true
 
 - ${{ if parameters.GenerateManifest }}:

src/AzureADAssessment.psd1

@@ -56,7 +56,7 @@ RequiredModules = @(
 )
 
 # Assemblies that must be loaded prior to importing this module
-RequiredAssemblies = @("System.IO.Compression.FileSystem.dll")
+# RequiredAssemblies = @()
 
 # Script files (.ps1) that are run in the caller's environment prior to importing this module.
 # ScriptsToProcess = @()
@@ -86,6 +86,7 @@ NestedModules = @(
     '.\internal\Export-JsonArray.ps1'
     '.\internal\Get-MsGraphResults.ps1'
     '.\internal\Format-Csv.ps1'
+    '.\internal\Format-NumberWithUnit.ps1'
     '.\internal\Get-AadObjectById.ps1'
     '.\internal\Get-ObjectPropertyValue.ps1'
     '.\internal\Get-SpreadsheetJson.ps1'
@@ -179,7 +180,7 @@ PrivateData = @{
     PSData = @{
 
         # Tags applied to this module. These help with module discovery in online galleries.
-            Tags   = 'Microsoft', 'Identity', 'Azure', 'AzureActiveDirectory', 'AzureAD', 'AAD', 'PSEdition_Desktop', 'PSEdition_Core', 'Windows', 'Linux', 'MacOS'
+        Tags   = 'Microsoft', 'Identity', 'Azure', 'AzureActiveDirectory', 'AzureAD', 'AAD', 'PSEdition_Desktop', 'PSEdition_Core', 'Windows', 'Linux', 'MacOS'
 
         # A URL to the license for this module.
         LicenseUri = 'https://raw.githubusercontent.com/AzureAD/AzureADAssessment/master/LICENSE'
@@ -193,6 +194,9 @@ PrivateData = @{
         # ReleaseNotes of this module
         # ReleaseNotes = ''
 
+        # Prerelease string of this module
+        Prerelease = 'source'
+
         # External dependent modules of this module
         # ExternalModuleDependencies = @()
 

src/AzureADAssessment.psm1

@@ -17,6 +17,12 @@ param (
 ## Set Strict Mode for Module. https://docs.microsoft.com/en-us/powershell/module/microsoft.powershell.core/set-strictmode
 Set-StrictMode -Version 3.0
 
+## Display Warning on old PowerShell versions. https://docs.microsoft.com/en-us/powershell/scripting/install/PowerShell-Support-Lifecycle#powershell-end-of-support-dates
+# ToDo: Only Windows PowerShell can currently satify device compliance CA requirement. Look at adding Windows Broker (WAM) support to support device compliance on PowerShell 7.
+# if ($PSVersionTable.PSVersion -lt [version]'7.0') {
+#     Write-Warning 'It is recommended to use this module with the latest version of PowerShell which can be downloaded here: https://aka.ms/install-powershell'
+# }
+
 ## Initialize Module Configuration
 $script:ModuleConfigDefault = Import-Config -Path (Join-Path $PSScriptRoot 'config.json')
 $script:ModuleConfig = $script:ModuleConfigDefault.psobject.Copy()
@@ -25,6 +31,11 @@ Import-Config | Set-Config
 if ($PSBoundParameters.ContainsKey('ModuleConfiguration')) { Set-Config $ModuleConfiguration }
 #Export-Config
 
+# Load zip dll on Windows PowerShell
+if ($PSVersionTable.PSEdition -eq 'Desktop') {
+    Add-Type -AssemblyName System.IO.Compression.FileSystem -ErrorAction Stop
+}
+
 ## Initialize Module Variables
 $script:ConnectState = @{
     ClientApplication = $null
@@ -41,6 +52,20 @@ $script:MsGraphSession.UserAgent += ' AzureADAssessment'
 #     UseDefaultCredentials = $true
 # }
 
+[string[]] $script:MsGraphScopes = @(
+    'Organization.Read.All'
+    'RoleManagement.Read.Directory'
+    'Application.Read.All'
+    'User.Read.All'
+    'Group.Read.All'
+    'Policy.Read.All'
+    'Directory.Read.All'
+    'SecurityEvents.Read.All'
+    'UserAuthenticationMethod.Read.All'
+    'AuditLog.Read.All'
+    'Reports.Read.All'
+)
+
 $script:mapMgEnvironmentToAzureCloudInstance = @{
     'Global'   = 'AzurePublic'
     'China'    = 'AzureChina'

src/AzureADAssessmentPortable.psm1

@@ -156,7 +156,10 @@ function Export-AADAssessADFSConfiguration {
     #Add-Type -assembly "system.io.compression.filesystem"
     #[io.compression.zipfile]::CreateFromDirectory($filePathBase, $zipfileName)
 
-    #Invoke-Item $zipfileBase
+    # try {
+    #     Invoke-Item $zipfileBase -ErrorAction SilentlyContinue
+    # }
+    # catch {}
 }
 
 

src/Complete-AADAssessmentReports.ps1

@@ -162,7 +162,10 @@ function Complete-AADAssessmentReports {
             Assert-DirectoryExists $PowerBIWorkingDirectory
             Copy-Item -Path (Join-Path $OutputDirectoryAAD '*') -Destination $PowerBIWorkingDirectory -Force
             Copy-Item -LiteralPath $PBITemplateAssessmentPath, $PBITemplateConditionalAccessPath -Destination $PowerBIWorkingDirectory -Force
-            #Invoke-Item $PowerBIWorkingDirectory
+            # try {
+            #     Invoke-Item $PowerBIWorkingDirectory -ErrorAction SilentlyContinue
+            # }
+            # catch {}
         }
 
         ## Expand AAD Connect
@@ -171,9 +174,12 @@ function Complete-AADAssessmentReports {
 
         ## Complete
         Write-Progress -Id 0 -Activity ('Microsoft Azure AD Assessment Complete Reports - {0}' -f $AssessmentDetail.AssessmentTenantDomain) -Completed
-        Invoke-Item $OutputDirectoryData
+        try {
+            Invoke-Item $OutputDirectoryData -ErrorAction SilentlyContinue
+        }
+        catch {}
 
     }
-    catch { if ($MyInvocation.CommandOrigin -eq 'Runspace') { Write-AppInsightsException $_.Exception }; throw }
+    catch { if ($MyInvocation.CommandOrigin -eq 'Runspace') { Write-AppInsightsException -ErrorRecord $_ -IncludeProcessStatistics }; throw }
     finally { Complete-AppInsightsRequest $MyInvocation.MyCommand.Name -Success $? }
 }

src/Connect-AADAssessment.ps1

@@ -63,11 +63,11 @@ function Connect-AADAssessment {
         }
         $script:ConnectState.CloudEnvironment = $CloudEnvironment
 
-        Confirm-ModuleAuthentication $script:ConnectState.ClientApplication -CloudEnvironment $script:ConnectState.CloudEnvironment -User $User -ErrorAction Stop
+        Confirm-ModuleAuthentication $script:ConnectState.ClientApplication -CloudEnvironment $script:ConnectState.CloudEnvironment -User $User -CorrelationId $script:AppInsightsRuntimeState.OperationStack.Peek().Id -ErrorAction Stop
         #Get-MgContext
         #Get-AzureADCurrentSessionInfo
         Write-Debug ($script:ConnectState.MsGraphToken.Scopes -join ' ')
     }
-    catch { if ($MyInvocation.CommandOrigin -eq 'Runspace') { Write-AppInsightsException $_.Exception }; throw }
+    catch { if ($MyInvocation.CommandOrigin -eq 'Runspace') { Write-AppInsightsException -ErrorRecord $_ -IncludeProcessStatistics }; throw }
     finally { Complete-AppInsightsRequest $MyInvocation.MyCommand.Name -Success $? }
 }

src/Disconnect-AADAssessment.ps1

@@ -20,6 +20,6 @@ function Disconnect-AADAssessment {
         }
 
     }
-    catch { if ($MyInvocation.CommandOrigin -eq 'Runspace') { Write-AppInsightsException $_.Exception }; throw }
+    catch { if ($MyInvocation.CommandOrigin -eq 'Runspace') { Write-AppInsightsException -ErrorRecord $_ -IncludeProcessStatistics }; throw }
     finally { Complete-AppInsightsRequest $MyInvocation.MyCommand.Name -Success $? }
 }

src/Expand-AADAssessAADConnectConfig.ps1

@@ -75,6 +75,6 @@ function Expand-AADAssessAADConnectConfig {
         Write-Output $report.FullName
 
     }
-    catch { if ($MyInvocation.CommandOrigin -eq 'Runspace') { Write-AppInsightsException $_.Exception }; throw }
+    catch { if ($MyInvocation.CommandOrigin -eq 'Runspace') { Write-AppInsightsException -ErrorRecord $_ -IncludeProcessStatistics }; throw }
     finally { Complete-AppInsightsRequest $MyInvocation.MyCommand.Name -Success $? }
 }

src/Export-AADAssessConditionalAccessData.ps1

@@ -56,6 +56,6 @@ function Export-AADAssessConditionalAccessData {
         #| Export-JsonArray (Join-Path $OutputDirectory "servicePrincipals.json") -Depth 5 -Compress
 
     }
-    catch { if ($MyInvocation.CommandOrigin -eq 'Runspace') { Write-AppInsightsException $_.Exception }; throw }
+    catch { if ($MyInvocation.CommandOrigin -eq 'Runspace') { Write-AppInsightsException -ErrorRecord $_ -IncludeProcessStatistics }; throw }
     finally { Complete-AppInsightsRequest $MyInvocation.MyCommand.Name -Success $? }
 }

src/Export-AADAssessmentPortableModule.ps1

@@ -27,6 +27,6 @@ function Export-AADAssessmentPortableModule {
         #Invoke-WebRequest -Uri 'https://github.com/AzureAD/Deployment-Plans/raw/master/ADFS%20to%20AzureAD%20App%20Migration/ADFSAADMigrationUtils.psm1' -UseBasicParsing -OutFile $AdfsAadMigrationModulePath
 
     }
-    catch { if ($MyInvocation.CommandOrigin -eq 'Runspace') { Write-AppInsightsException $_.Exception }; throw }
+    catch { if ($MyInvocation.CommandOrigin -eq 'Runspace') { Write-AppInsightsException -ErrorRecord $_ -IncludeProcessStatistics }; throw }
     finally { Complete-AppInsightsRequest $MyInvocation.MyCommand.Name -Success $? }
 }