[Bug]: Export User file empty when Source = DevOps Server 2022

[LudekStipal]

Version

• I confirm that I am using the latest version

Source Version

Azure DevOps Server 2022

Target Version

Azure DevOps Service

Relevant configuration

{
  "ChangeSetMappingFile": null,
  "Source": {
    "$type": "TfsTeamProjectConfig",
    "Collection": "https://devops/tfs/VWFSAG/",
    "Project": "IBATeam",
    "ReflectedWorkItemIDFieldName": "ReflectedWorkItemId",
    "AllowCrossProjectLinking": false,
    "AuthenticationMode": "AccessToken",
    "PersonalAccessToken": "xxx",
    "PersonalAccessTokenVariableName": "",
    "LanguageMaps": {
      "AreaPath": "Area",
      "IterationPath": "Iteration"
    }
  },
  "Target": {
    "$type": "TfsTeamProjectConfig",
    "Collection": "https://dev.azure.com/vwfs",
    "Project": "Azubis.AuroraTeam",
    "ReflectedWorkItemIDFieldName": "ReflectedWorkItemId",
    "AllowCrossProjectLinking": false,
    "AuthenticationMode": "AccessToken",
    "PersonalAccessToken": "xxx",
    "PersonalAccessTokenVariableName": "",
    "LanguageMaps": {
      "AreaPath": "Area",
      "IterationPath": "Iteration"
    }
  },
  "FieldMaps": [],
  "GitRepoMapping": null,
  "LogLevel": "Debug",
  "CommonEnrichersConfig": [
    {
      "$type": "TfsNodeStructureOptions",
      "RefName": "TfsNodeStructure",
      "Enabled": false,
      "NodeBasePaths": [],
      "AreaMaps": {},
      "IterationMaps": {},
      "ShouldCreateMissingRevisionPaths": true,
      "ReplicateAllExistingNodes": true
    },
    {
      "$type": "TfsTeamSettingsEnricherOptions",
      "Enabled": false,
      "MigrateTeamSettings": true,
      "UpdateTeamSettings": true,
      "MigrateTeamCapacities": true,
      "Teams": null
    },
    {
      "$type": "TfsWorkItemLinkEnricherOptions",
      "RefName": "TfsWorkItemLinkEnricher",
      "Enabled": false,
      "FilterIfLinkCountMatches": true,
      "SaveAfterEachLinkIsAdded": false
    },
    {
      "$type": "TfsRevisionManagerOptions",
      "RefName": "TfsRevisionManager",
      "Enabled": false,
      "ReplayRevisions": true,
      "MaxRevisions": 0
    },
    {
      "$type": "TfsAttachmentEnricherOptions",
      "RefName": "TfsAttachmentEnricher",
      "Enabled": false,
      "ExportBasePath": "c:\\temp\\WorkItemAttachmentExport",
      "MaxRevisions": 480000000
    },
    {
      "$type": "StringManipulatorEnricherOptions",
      "RefName": "StringManipulator",
      "Enabled": false,
      "MaxStringLength": 1000000,
      "Manipulators": [
        {
          "$type": "RegexStringManipulator",
          "Enabled": true,
          "Pattern": "[^( -~)\n\r\t]+",
          "Replacement": "",
          "Description": "Remove invalid characters from the end of the string"
        }
      ]
    },
    {
      "$type": "TfsUserMappingEnricherOptions",
      "RefName": "TfsUserMappingEnricher",
      "Enabled": true,
      "UserMappingFile": "C:\\Users\\dkx8myt\\downloads\\Users_IBATeam.json",
      "IdentityFieldsToCheck": [
        "System.AssignedTo",
        "System.ChangedBy",
        "System.CreatedBy",
        "Microsoft.VSTS.Common.ActivatedBy",
        "Microsoft.VSTS.Common.ResolvedBy",
        "Microsoft.VSTS.Common.ClosedBy"
      ]
    }
  ],
  "Processors": [
    {
      "$type": "WorkItemMigrationConfig",
      "Enabled": false,
      "UpdateCreatedDate": true,
      "UpdateCreatedBy": true,
      "WIQLQuery": "SELECT [System.Id] FROM WorkItems WHERE [System.TeamProject] = @TeamProject AND [System.WorkItemType] NOT IN ('Test Suite', 'Test Plan','Shared Steps','Shared Parameter','Feedback Request') ORDER BY [System.ChangedDate] desc",
      "FixHtmlAttachmentLinks": false,
      "WorkItemCreateRetryLimit": 5,
      "FilterWorkItemsThatAlreadyExistInTarget": false,
      "PauseAfterEachWorkItem": false,
      "AttachRevisionHistory": false,
      "GenerateMigrationComment": true,
      "WorkItemIDs": null,
      "MaxGracefulFailures": 0,
      "SkipRevisionWithInvalidIterationPath": false,
      "SkipRevisionWithInvalidAreaPath": false
    },
    {
      "$type": "WorkItemPostProcessingConfig",
      "Enabled": false,
      "WIQLQuery": "SELECT [System.Id] FROM WorkItems WHERE [System.TeamProject] = @TeamProject AND [System.WorkItemType] NOT IN ('Test Suite', 'Test Plan','Shared Steps','Shared Parameter','Feedback Request') ORDER BY [System.ChangedDate] desc",
      "WorkItemCreateRetryLimit": 5,
      "FilterWorkItemsThatAlreadyExistInTarget": false,
      "PauseAfterEachWorkItem": false
    },
    {
      "$type": "ExportUsersForMappingConfig",
      "Enabled": true,
      "WIQLQuery": "SELECT [System.Id] FROM WorkItems WHERE [System.TeamProject] = @TeamProject AND [System.WorkItemType] NOT IN ('Test Suite', 'Test Plan','Shared Steps','Shared Parameter','Feedback Request') AND [System.ChangedDate] > '02-06-2024' ORDER BY [System.ChangedDate] desc",
      "OnlyListUsersInWorkItems": true
    }
  ],
  "Version": "15.0",
  "workaroundForQuerySOAPBugEnabled": false,
  "WorkItemTypeDefinition": {
    "User Story": "Product Backlog Item"
  },
  "Endpoints": {
    "InMemoryWorkItemEndpoints": [
      {
        "Name": "Source",
        "EndpointEnrichers": null
      },
      {
        "Name": "Target",
        "EndpointEnrichers": null
      }
    ]
  }
}

Relevant log output

[12:22:15 DBG] [v15.0.5] TfsUserMappingEnricher::GetUsersInSourceMappedToTarget [targetUsersCount|1316]
[12:22:15 DBG] [v15.0.5] TfsUserMappingEnricher::GetUsersInSourceMappedToTargetForWorkItems [mappedUsers|0]
[12:22:15 INF] [v15.0.5] Found 0 total mapped
[12:22:15 INF] [v15.0.5] Filtered to 0 total viable mappings
[12:22:15 INF] [v15.0.5] Writen to: C:\Users\dkx8myt\downloads\Users_IBATeam.json
[12:22:15 INF] [v15.0.5] DONE in {Elapsed} seconds
[12:22:15 INF] [v15.0.5]  Migration Context Complete ExportUsersForMappingContext
[12:22:15 INF] [v15.0.5] Application is shutting down...
[12:22:15 DBG] [v15.0.5] Hosting stopping
[12:22:15 DBG] [v15.0.5] Exiting with return code: 0
[12:22:15 INF] [v15.0.5] Application Ending
[12:22:15 INF] [v15.0.5] The application ran in 00:02:46.3304176 and finished at 06/10/2024 12:22:15
[12:22:15 DBG] [v15.0.5] Hosting stopped

What happened?

All versions of Tool with current TfsUserMappingEnricher (tried 15.0.4 Release, 1.0.5 Preview 72, 1.0.5 Preview 168) are incapable of exporting any Users from DevOps Server (Domain logins). Even if only TfsUserMappingEnricherOptions and ExportUsersForMappingConfig are enabled, it always "filters to 0 viable mappings".

I do not expect the EXPORT of SOURCE users to do any "mapping" to target identities. All I want is a JSON of all Users present in Source WorkItems, which can be used as basis for preparing our own Mapping file (from DevOps Server AD identity do DevOps Services e-mail identity). This worked in Tool version 14.x, where all that happened was:

src/VstsSyncMigrator.Core/Execution/MigrationContext/ExportUsersForMapping.cs

Dictionary<string,string> usersToMap = _TfsUserMappingEnricher.findUsersToMap(sourceWorkItems, _config.IdentityFieldsToCheck);
System.IO.File.WriteAllText(_config.LocalExportJsonFile, Newtonsoft.Json.JsonConvert.SerializeObject(usersToMap));

However, in current implementation, no matter what I configure (I even tried setting the Target DevOps to the same Server instance, no luck - the Enricher is unable to work with AD identities properly, and output is always an empty file).

Debug in Visual Studio

• Visual Studio Debug

[LudekStipal]

Still present in current Preview 15.1.1-Preview.11:

[09:35:07 DBG] [v15.1.1] TfsUserMappingEnricher::GetUsersInSourceMappedToTarget [targetUsersCount|1342]
[09:35:07 DBG] [v15.1.1] TfsUserMappingEnricher::GetUsersInSourceMappedToTargetForWorkItems [mappedUsers|0]
[09:35:07 INF] [v15.1.1] Found 0 total mapped
[09:35:07 INF] [v15.1.1] Filtered to 0 total viable mappings
[09:35:07 INF] [v15.1.1] Writen to: C:\Users\dkx8myt\downloads\Users_IBATeam.json
[09:35:07 INF] [v15.1.1] DONE in {Elapsed} seconds
[09:35:07 INF] [v15.1.1]  Migration Context Complete ExportUsersForMappingContext
[09:35:07 INF] [v15.1.1] Application is shutting down...
[09:35:07 DBG] [v15.1.1] Hosting stopping
[09:35:07 DBG] [v15.1.1] Exiting with return code: 0
[09:35:07 INF] [v15.1.1] Application Ending
[09:35:07 INF] [v15.1.1] The application ran in 00:05:00.0167004 and finished at 07/01/2024 09:35:07
[09:35:07 DBG] [v15.1.1] Hosting stopped

[LudekStipal]

The same behaviour in 15.1.3-Preview.2:

[11:39:46 DBG] [v15.1.3.0-Preview.2] TfsUserMappingEnricher::GetUsersInSourceMappedToTarget [targetUsersCount|1343]
[11:39:46 DBG] [v15.1.3.0-Preview.2] TfsUserMappingEnricher::GetUsersInSourceMappedToTargetForWorkItems [mappedUsers|0]
[11:39:46 INF] [v15.1.3.0-Preview.2] Found 0 total mapped
[11:39:46 INF] [v15.1.3.0-Preview.2] Filtered to 0 total viable mappings
[11:39:46 INF] [v15.1.3.0-Preview.2] Writen to: C:\Users\dkx8myt\downloads\Users_IBATeam.json
[11:39:46 INF] [v15.1.3.0-Preview.2] DONE in {Elapsed} seconds
[11:39:46 INF] [v15.1.3.0-Preview.2]  Migration Context Complete ExportUsersForMappingContext
[11:39:46 INF] [v15.1.3.0-Preview.2] Application is shutting down...
[11:39:46 DBG] [v15.1.3.0-Preview.2] Hosting stopping
[11:39:46 DBG] [v15.1.3.0-Preview.2] Exiting with return code: 0
[11:39:46 INF] [v15.1.3.0-Preview.2] Application Ending
[11:39:46 INF] [v15.1.3.0-Preview.2] The application ran in 00:06:10.1517818 and finished at 07/15/2024 11:39:46

and 15.1.4-Preview.6:

[10:29:28 DBG] [v15.1.4.0-Preview.6] TfsUserMappingEnricher::GetUsersInSourceMappedToTarget [targetUsersCount|1343]
[10:29:28 DBG] [v15.1.4.0-Preview.6] TfsUserMappingEnricher::GetUsersInSourceMappedToTargetForWorkItems [mappedUsers|0]
[10:29:28 INF] [v15.1.4.0-Preview.6] Found 0 total mapped
[10:29:28 INF] [v15.1.4.0-Preview.6] Filtered to 0 total viable mappings
[10:29:28 INF] [v15.1.4.0-Preview.6] Writen to: C:\Users\dkx8myt\downloads\Users_IBATeam.json
[10:29:28 INF] [v15.1.4.0-Preview.6] DONE in {Elapsed} seconds
[10:29:28 INF] [v15.1.4.0-Preview.6]  Migration Context Complete ExportUsersForMappingContext
[10:29:28 INF] [v15.1.4.0-Preview.6] Application is shutting down...
[10:29:28 DBG] [v15.1.4.0-Preview.6] Hosting stopping
[10:29:28 DBG] [v15.1.4.0-Preview.6] Exiting with return code: 0
[10:29:28 INF] [v15.1.4.0-Preview.6] Application Ending
[10:29:28 INF] [v15.1.4.0-Preview.6] The application ran in 00:04:00.5554693 and finished at 07/18/2024 10:29:28
[10:29:28 DBG] [v15.1.4.0-Preview.6] Hosting stopped

[MrHinsh]

@LudekStipal I think you are going to have to debug. I do not have access to a TFS server connected to AD!

We can run a debug session together if you need!

[LudekStipal]

Hi @MrHinsh , sorry for the late reply, I was on planned holiday and just returned. I ran Debug session using the latest Code version (Repo clone, reports itself as [15.1.8-Local.9]) and was able to narrow down the issue: the Identities from Source (DevOps Server 2022) are never loaded (empty), so there can be no mapping file produced.

The problem lies within:

\src\MigrationTools.Clients.AzureDevops.ObjectModel\ProcessorEnrichers\TfsUserMappingEnricher.cs

private List<IdentityItemData> GetUsersListFromServer(IGroupSecurityService gss)

This works fine for Azure DevOps Services (Cloud); However, when executing gss.ReadIdentity() against Azure DevOps Server 2022 (Update 2, but same behaviour was in Update 1), the filter:

var people = SIDS.Members.ToList().Where(x => x.Contains("\\")).Select(x => x);

always returns empty array, so nothing can be mapped / exported.

The SIDS return from DevOps Server do not contain a single entry with "backslash", they are all SIDs - either starting with S-1-5-xxx, S-1-9-xxx, or in [Guid]:Build:[Guid] Format:

So the following logic - ReadIdentity with SaerchFactor.AccountName - is also not usable for this case.

I am currently not sure which "SIDs" belong tu Users, but it seems this modified function works for both DevOps Services and DevOps Server:

        private List<IdentityItemData> GetUsersListFromServer(IGroupSecurityService gss)
        {
            Identity SIDS = gss.ReadIdentity(SearchFactor.AccountName, "Project Collection Valid Users", QueryMembership.Expanded);
            var people = SIDS.Members.ToList().Where(x => x.Contains("\\") || x.StartsWith("S-1-5-")).Select(x => x);

            List<IdentityItemData> foundUsers = new List<IdentityItemData>();
            Log.LogTrace("TfsUserMappingEnricher::GetUsersListFromServer:foundUsers\\ {@foundUsers}", foundUsers);
            foreach (string user in people)
            {
                Log.LogDebug("TfsUserMappingEnricher::GetUsersListFromServer::[user:{user}] Atempting to load user", user);
                try
                {
                    var bits = user.Split('\\');
                    Identity sids = bits.Length > 1 ? gss.ReadIdentity(SearchFactor.AccountName, bits[1], QueryMembership.Expanded) : gss.ReadIdentity(SearchFactor.Sid, bits[0], QueryMembership.Expanded);
 
                    if (sids != null)
                    {
                        if (!foundUsers.Where(x => x.FriendlyName == sids.DisplayName).Any())
                            foundUsers.Add(new IdentityItemData() { FriendlyName = sids.DisplayName, AccountName = sids.AccountName });
                    }
                    else
                    {
                        Log.LogDebug("TfsUserMappingEnricher::GetUsersListFromServer::[user:{user}] ReadIdentity returned null for {@bits}", user, bits);
                    }

                }
                catch (Exception ex)
                {
                    Telemetry.TrackException(ex, null, null);
                    Log.LogWarning("TfsUserMappingEnricher::GetUsersListFromServer::[user:{user}] Failed With {Exception}", user, ex.Message);
                }

            }
            return foundUsers;
        }

• People filter: added x.StartsWith("S-1-5-")
• Read Identities either by AccountName, or SID:

Identity sids = bits.Length > 1 ? gss.ReadIdentity(SearchFactor.AccountName, bits[1], QueryMembership.Expanded) : gss.ReadIdentity(SearchFactor.Sid, bits[0], QueryMembership.Expanded);

• Also adding a simple "distinct" filter when adding to output array was necessary (most likely due to technical accounts / users in multiple Domains with the same DisplayName), otherwise following mapping to targetUsers failed:

if (!foundUsers.Where(x => x.FriendlyName == sids.DisplayName).Any())

With this modification, the output file contains expected result (DisplayNames of Users in WorkItems mapped to "null": this is what we need, as the Projects need to supply their own correct Identities (e-mail addresses) for the Mapping).

[LudekStipal]

New Path for file containing GetUsersListFromServer function in MigrationTools v16.0.5-Preview.4

src\MigrationTools.Clients.TfsObjectModel\Tools\TfsUserMappingTool.cs

The problem / possible solution remains the same as in my previous post.

[satano]

@LudekStipal I have fixed this in #2522 Not sure if it is released out in latest preview version, but from the sources it will work for you.

@MrHinsh I think you can close this issue.

[MrHinsh]

It's in preview. Not sure if it's shipped yet as Winget are quite slow... But choco is usually quick.

[github-actions]

This issue is stale because it has been open 30 days with no activity. Remove stale label or comment or this will be closed in 10 days