Hotfix Use Python Version on m133 (#7149)

Author: brcrista

Tasks/UsePythonVersion/Strings/resources.resjson/en-US/resources.resjson

@@ -3,10 +3,13 @@
   "loc.helpMarkDown": "[More Information](https://go.microsoft.com/fwlink/?linkid=871498)",
   "loc.description": "Retrieves the specified version of Python from the tool cache. Optionally add it to PATH.",
   "loc.instanceNameFormat": "Use Python $(versionSpec)",
+  "loc.group.displayName.advanced": "Advanced",
   "loc.input.label.versionSpec": "Version spec",
   "loc.input.help.versionSpec": "Version range or exact version of a Python version to use.",
   "loc.input.label.addToPath": "Add to PATH",
   "loc.input.help.addToPath": "Whether to prepend the retrieved Python version to the PATH environment variable to make it available in subsequent tasks or scripts without using the output variable.",
+  "loc.input.label.architecture": "Architecture",
+  "loc.input.help.architecture": "The target architecture (x86, x64) of the Python interpreter.",
   "loc.messages.ListAvailableVersions": "Available versions:",
   "loc.messages.PlatformNotRecognized": "Platform not recognized",
   "loc.messages.PrependPath": "Prepending PATH environment variable with directory: %s",

Tasks/UsePythonVersion/Tests/L0.ts

@@ -82,15 +82,16 @@ describe('UsePythonVersion L0 Suite', function () {
         };
         mockery.registerMock('vsts-task-lib/task', Object.assign({}, mockTask, mockBuildVariables));
 
-        const toolPath = path.join('/', 'Python', '3.6.4');
+        const toolPath = path.join('/', 'Python', '3.6.4', 'x64');
         mockery.registerMock('vsts-task-tool-lib/tool', {
             findLocalTool: () => toolPath
         });
 
         const uut = reload();
         const parameters = {
             versionSpec: '3.6',
-            addToPath: false
+            addToPath: false,
+            architecture: 'x64'
         };
 
         assert.strictEqual(buildVariables['pythonLocation'], undefined);
@@ -103,13 +104,14 @@ describe('UsePythonVersion L0 Suite', function () {
         mockery.registerMock('vsts-task-lib/task', mockTask);
         mockery.registerMock('vsts-task-tool-lib/tool', {
             findLocalTool: () => null,
-            findLocalToolVersions: () => ['2.7.13']
+            findLocalToolVersions: () => ['2.6.0', '2.7.13']
         });
 
         const uut = reload();
         const parameters = {
             versionSpec: '3.x',
-            addToPath: false
+            addToPath: false,
+            architecture: 'x64'
         };
 
         try {
@@ -119,14 +121,52 @@ describe('UsePythonVersion L0 Suite', function () {
             const expectedMessage = [
                 'loc_mock_VersionNotFound 3.x',
                 'loc_mock_ListAvailableVersions',
-                '2.7.13'
+                '2.6.0 (x86)',
+                '2.7.13 (x86)',
+                '2.6.0 (x64)',
+                '2.7.13 (x64)'
             ].join(EOL);
 
             assert.strictEqual(e.message, expectedMessage);
             done();
         }
     });
 
+    it('selects architecture passed as input', async function () {
+        let buildVariables: { [key: string]: string } = {};
+        const mockBuildVariables = {
+            setVariable: (variable: string, value: string) => {
+                buildVariables[variable] = value;
+            },
+            getVariable: (variable: string) => buildVariables[variable]
+        };
+        mockery.registerMock('vsts-task-lib/task', Object.assign({}, mockTask, mockBuildVariables));
+
+        const x86ToolPath = path.join('/', 'Python', '3.6.4', 'x86');
+        const x64ToolPath = path.join('/', 'Python', '3.6.4', 'x64');
+        mockery.registerMock('vsts-task-tool-lib/tool', {
+            findLocalTool: (toolName: string, versionSpec: string, arch?: string) => {
+                if (arch === 'x86') {
+                    return x86ToolPath;
+                } else {
+                    return x64ToolPath;
+                }
+            }
+        });
+
+        const uut = reload();
+        const parameters = {
+            versionSpec: '3.6',
+            addToPath: false,
+            architecture: 'x86'
+        };
+
+        assert.strictEqual(buildVariables['pythonLocation'], undefined);
+
+        await uut.usePythonVersion(parameters, Platform.Linux);
+        assert.strictEqual(buildVariables['pythonLocation'], x86ToolPath);
+    });
+
     it('sets PATH correctly on Linux', async function () {
         mockery.registerMock('vsts-task-lib/task', mockTask);
 
@@ -145,12 +185,12 @@ describe('UsePythonVersion L0 Suite', function () {
         const uut = reload();
         const parameters = {
             versionSpec: '3.6',
-            outputVariable: 'Python',
-            addToPath: true
+            addToPath: true,
+            architecture: 'x64'
         };
 
         await uut.usePythonVersion(parameters, Platform.Linux);
-        assert.strictEqual(`${toolPath}:`, mockPath);
+        assert.strictEqual(`${path.join(toolPath, 'bin')}:${toolPath}:`, mockPath);
     });
 
     it('sets PATH correctly on Windows', async function () {
@@ -173,7 +213,8 @@ describe('UsePythonVersion L0 Suite', function () {
         const uut = reload();
         const parameters = {
             versionSpec: '3.6',
-            addToPath: true
+            addToPath: true,
+            architecture: 'x64'
         };
 
         await uut.usePythonVersion(parameters, Platform.Windows);

Tasks/UsePythonVersion/main.ts

@@ -8,7 +8,8 @@ import { usePythonVersion } from './usepythonversion';
         task.setResourcePath(path.join(__dirname, 'task.json'));
         await usePythonVersion({
             versionSpec: task.getInput('versionSpec', true),
-            addToPath: task.getBoolInput('addToPath', true)
+            addToPath: task.getBoolInput('addToPath', true),
+            architecture: task.getInput('architecture', true)
         },
         getPlatform());
         task.setResult(task.TaskResult.Succeeded, "");

Tasks/UsePythonVersion/task.json

@@ -12,12 +12,19 @@
     "author": "Microsoft Corporation",
     "version": {
         "Major": 0,
-        "Minor": 133,
+        "Minor": 134,
         "Patch": 2
     },
     "preview": true,
     "demands": [],
     "instanceNameFormat": "Use Python $(versionSpec)",
+    "groups": [
+        {
+            "name": "advanced",
+            "displayName": "Advanced",
+            "isExpanded": false
+        }
+    ],
     "inputs": [
         {
             "name": "versionSpec",
@@ -34,6 +41,19 @@
             "required": true,
             "defaultValue": "true",
             "helpMarkDown": "Whether to prepend the retrieved Python version to the PATH environment variable to make it available in subsequent tasks or scripts without using the output variable."
+        },
+        {
+            "name": "architecture",
+            "type": "pickList",
+            "label": "Architecture",
+            "defaultValue": "x64",
+            "required": true,
+            "helpMarkDown": "The target architecture (x86, x64) of the Python interpreter.",
+            "groupName": "advanced",
+            "options": {
+                "x86": "x86",
+                "x64": "x64"
+            }
         }
     ],
     "outputVariables": [

Tasks/UsePythonVersion/task.loc.json

@@ -12,12 +12,19 @@
   "author": "Microsoft Corporation",
   "version": {
     "Major": 0,
-    "Minor": 133,
+    "Minor": 134,
     "Patch": 2
   },
   "preview": true,
   "demands": [],
   "instanceNameFormat": "ms-resource:loc.instanceNameFormat",
+  "groups": [
+    {
+      "name": "advanced",
+      "displayName": "ms-resource:loc.group.displayName.advanced",
+      "isExpanded": false
+    }
+  ],
   "inputs": [
     {
       "name": "versionSpec",
@@ -34,6 +41,19 @@
       "required": true,
       "defaultValue": "true",
       "helpMarkDown": "ms-resource:loc.input.help.addToPath"
+    },
+    {
+      "name": "architecture",
+      "type": "pickList",
+      "label": "ms-resource:loc.input.label.architecture",
+      "defaultValue": "x64",
+      "required": true,
+      "helpMarkDown": "ms-resource:loc.input.help.architecture",
+      "groupName": "advanced",
+      "options": {
+        "x86": "x86",
+        "x64": "x64"
+      }
     }
   ],
   "outputVariables": [

Tasks/UsePythonVersion/usepythonversion.ts

@@ -11,48 +11,61 @@ import { Platform } from './taskutil';
 import * as toolUtil  from './toolutil';
 
 interface TaskParameters {
-    readonly versionSpec: string,
-    readonly addToPath: boolean
+    versionSpec: string,
+    addToPath: boolean,
+    architecture: string
 }
 
 export function pythonVersionToSemantic(versionSpec: string) {
     const prereleaseVersion = /(\d+\.\d+\.\d+)([a|b|rc]\d*)/g;
     return versionSpec.replace(prereleaseVersion, '$1-$2');
 }
 
-export async function usePythonVersion(parameters: TaskParameters, platform: Platform): Promise<void> {
+export async function usePythonVersion(parameters: Readonly<TaskParameters>, platform: Platform): Promise<void> {
     // Python's prelease versions look like `3.7.0b2`.
     // This is the one part of Python versioning that does not look like semantic versioning, which specifies `3.7.0-b2`.
     // If the version spec contains prerelease versions, we need to convert them to the semantic version equivalent
     const semanticVersionSpec = pythonVersionToSemantic(parameters.versionSpec);
     task.debug(`Semantic version spec of ${parameters.versionSpec} is ${semanticVersionSpec}`);
 
-    const installDir: string | null = tool.findLocalTool('Python', semanticVersionSpec);
+    const installDir: string | null = tool.findLocalTool('Python', semanticVersionSpec, parameters.architecture);
     if (!installDir) {
         // Fail and list available versions
+        const x86Versions = tool.findLocalToolVersions('Python', 'x86')
+            .map(s => `${s} (x86)`)
+            .join(os.EOL);
+
+        const x64Versions = tool.findLocalToolVersions('Python', 'x64')
+            .map(s => `${s} (x64)`)
+            .join(os.EOL);
+
         throw new Error([
             task.loc('VersionNotFound', parameters.versionSpec),
             task.loc('ListAvailableVersions'),
-            tool.findLocalToolVersions('Python')
+            x86Versions,
+            x64Versions
         ].join(os.EOL));
     }
 
     task.setVariable('pythonLocation', installDir);
     if (parameters.addToPath) {
         toolUtil.prependPathSafe(installDir);
 
-        // Python has "scripts" directories where command-line tools that come with packages are installed.
-        // There are different directories for `pip install` and `pip install --user`.
-        // On Linux and macOS, pip will create the scripts directories and add them to PATH as needed.
-        // On Windows, these directories do not get added to PATH, so we will add them ourselves.
+        // Make sure Python's "bin" directories are in PATH.
+        // Python has "scripts" or "bin" directories where command-line tools that come with packages are installed.
+        // This is where pip is, along with anything that pip installs.
+        // There is a seperate directory for `pip install --user`.
+        //
         // For reference, these directories are as follows:
         //   macOS / Linux:
-        //      /usr/local/bin
+        //      <sys.prefix>/bin (by default /usr/local/bin, but not on hosted agents -- see the `else`)
         //      (--user) ~/.local/bin
         //   Windows:
         //      <Python installation dir>\Scripts
         //      (--user) %APPDATA%\Python\PythonXY\Scripts
+        // See https://docs.python.org/3/library/sysconfig.html
         if (platform === Platform.Windows) {
+            // On Windows, these directories do not get added to PATH, so we will add them ourselves.
             const scriptsDir = path.join(installDir, 'Scripts');
             toolUtil.prependPathSafe(scriptsDir);
 
@@ -65,6 +78,13 @@ export async function usePythonVersion(parameters: TaskParameters, platform: Pla
 
             const userScriptsDir = path.join(process.env['APPDATA'], 'Python', `Python${major}${minor}`, 'Scripts');
             toolUtil.prependPathSafe(userScriptsDir);
+        } else {
+            // On Linux and macOS, tools cache should be set up so that each Python version has its own "bin" directory.
+            // We do this so that the tool cache can just be dropped on an agent with minimal installation (no copying to /usr/local).
+            // This allows us side-by-side the same minor version of Python with different patch versions or architectures (since Python uses /usr/local/lib/python3.6, etc.).
+            toolUtil.prependPathSafe(path.join(installDir, 'bin'));
+
+            // On Linux and macOS, pip will create the --user directory and add it to PATH as needed.
         }
     }
 }