Change tool metadata file format to JSON + support arm64 host fetches (…

…#1553) Co-authored-by: data-queue <[email protected]> Co-authored-by: Billy Robert O'Neal III <[email protected]>
microsoft · Jan 3, 2025 · f351510 · f351510
1 parent 18a617c
commit f351510
Show file tree

Hide file tree

Showing 31 changed files with 955 additions and 392 deletions.
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -80,7 +80,6 @@ jobs:
         shell: pwsh
         run: |
           cd out/build/${{ matrix.preset }}
-          Import-Module Pester -Force -MinimumVersion '5.6.1' -MaximumVersion '5.99'
           ${{ github.workspace }}/azure-pipelines/end-to-end-tests.ps1 -RunArtifactsTests
         env:
           VCPKG_ROOT: ${{ github.workspace }}/vcpkg-root
diff --git a/azure-pipelines/end-to-end-tests-dir/fetch.ps1 b/azure-pipelines/end-to-end-tests-dir/fetch.ps1
@@ -1,95 +1,123 @@
 . $PSScriptRoot/../end-to-end-tests-prelude.ps1
 
+$VcpkgToolsJsonSchemaFile = (Get-Item "$PSScriptRoot/../../docs/vcpkg-tools.schema.json").FullName
+if (-not (Test-Json -ea:0 -LiteralPath "$VcpkgRoot/scripts/vcpkg-tools.json" -SchemaFile $VcpkgToolsJsonSchemaFile)) {
+    throw "real vcpkg-tools.json doesn't conform to schema"
+}
+
 if (-not $IsMacOS -and -not $IsLinux) {
     "" | Out-File -enc ascii $(Join-Path $TestingRoot .vcpkg-root)
 
     $Scripts = Join-Path $TestingRoot "scripts"
     mkdir $Scripts | Out-Null
+
+    $VcpkgToolsJson = Join-Path $Scripts "vcpkg-tools.json"
+
+    $7zip_version = "19.00"
+    $ninja_version = "1.10.2"
 
-@"
-<?xml version="1.0"?>
-<tools version="2">
-    <tool name="7zip" os="windows">
-        <version>19.00</version>
-        <exeRelativePath>Files\7-Zip\7z.exe</exeRelativePath>
-        <url>https://www.7-zip.org/a/7z1900-x64.msi</url>
-        <sha512>7837a8677a01eed9c3309923f7084bc864063ba214ee169882c5b04a7a8b198ed052c15e981860d9d7952c98f459a4fab87a72fd78e7d0303004dcb86f4324c8</sha512>
-        <archiveName>7z1900-x64.msi</archiveName>
-    </tool>
-    <tool name="ninja-testing" os="windows">
-        <version>1.10.2</version>
-        <exeRelativePath>ninja.exe</exeRelativePath>
-        <url>https://github.com/ninja-build/ninja/releases/download/v1.10.2/ninja-win.zip</url>
-        <sha512>6004140d92e86afbb17b49c49037ccd0786ce238f340f7d0e62b4b0c29ed0d6ad0bab11feda2094ae849c387d70d63504393714ed0a1f4d3a1f155af7a4f1ba3</sha512>
-        <archiveName>ninja-win-1.10.2.zip</archiveName>
-    </tool>
-    <tool name="ninja" os="windows">
-        <version>1.10.2</version>
-        <exeRelativePath>ninja.exe</exeRelativePath>
-        <url>https://github.com/ninja-build/ninja/releases/download/v1.10.2/ninja-win.zip</url>
-        <sha512>6004140d92e86afbb17b49c49037ccd0786ce238f340f7d0e62b4b0c29ed0d6ad0bab11feda2094ae849c387d70d63504393714ed0a1f4d3a1f155af7a4f1ba3</sha512>
-        <archiveName>ninja-win-1.10.2.zip</archiveName>
-    </tool>
-    <tool name="cmake" os="windows">
-        <version>3.22.2</version>
-        <exeRelativePath>cmake-3.22.2-windows-i386\bin\cmake.exe</exeRelativePath>
-        <url>https://github.com/Kitware/CMake/releases/download/v3.22.2/cmake-3.22.2-windows-i386.zip</url>
-        <sha512>969d3d58d56d8fa3cc3acae2b949bf58abab945f70ae292ff20c9060d845dfc094c613c367a924abff47f307cc33af1467cdb9b75bb857868e38b2c7cdc72f79</sha512>
-        <archiveName>cmake-3.22.2-windows-i386.zip</archiveName>
-    </tool>
-    <tool name="cmake" os="osx">
-        <version>3.22.2</version>
-        <exeRelativePath>cmake-3.22.2-macos-universal/CMake.app/Contents/bin/cmake</exeRelativePath>
-        <url>https://github.com/Kitware/CMake/releases/download/v3.22.2/cmake-3.22.2-macos-universal.tar.gz</url>
-        <sha512>08104f608ecb9a5cfef38e79f0957d21e425616c0677781445492f82cbfec805113e3b5eb4bc737b707bb26a00678e7bd55e17555a5611c08b0b9b44ac5136ac</sha512>
-        <archiveName>cmake-3.22.2-macos-universal.tar.gz</archiveName>
-    </tool>
-    <tool name="cmake" os="linux">
-        <version>3.22.2</version>
-        <exeRelativePath>cmake-3.22.2-linux-x86_64/bin/cmake</exeRelativePath>
-        <url>https://github.com/Kitware/CMake/releases/download/v3.22.2/cmake-3.22.2-linux-x86_64.tar.gz</url>
-        <sha512>579e08b086f6903ef063697fca1dc2692f68a7341dd35998990b772b4221cdb5b1deecfa73bad9d46817ef09e58882b2adff9d64f959c01002c11448a878746b</sha512>
-        <archiveName>cmake-3.22.2linux-x86_64.tar.gz</archiveName>
-    </tool>
-    <tool name="cmake" os="freebsd">
-        <version>3.20.4</version>
-        <exeRelativePath>usr/local/bin/cmake</exeRelativePath>
-        <url>https://pkg.freebsd.org/FreeBSD:13:amd64/quarterly/All/cmake-3.20.4.txz</url>
-        <sha512>3e5b675d7ff924f92996d912e2365582e687375109ef99c9073fb8196bb329243a406b218cf1358d7cc518988b311ce9e5bf87de4d64f2e6377b7c2bc8894475</sha512>
-        <archiveName>cmake-3.20.4.txz</archiveName>
-    </tool>
-</tools>
-"@ | % { $_ -replace "`r","" } | Out-File -enc ascii $(Join-Path $Scripts "vcpkgTools.xml")
+    @'
+{
+    "schema-version": 1,
+    "tools": [{
+        "name": "7zip",
+        "os": "windows",
+        "version": "19.00",
+        "executable": "Files\\7-Zip\\7z.exe",
+        "url": "https://www.7-zip.org/a/7z1900-x64.msi",
+        "sha512": "7837a8677a01eed9c3309923f7084bc864063ba214ee169882c5b04a7a8b198ed052c15e981860d9d7952c98f459a4fab87a72fd78e7d0303004dcb86f4324c8",
+        "archive": "7z1900-x64.msi"
+    },
+    {
+        "name": "ninja-testing",
+        "os": "windows",
+        "version": "1.10.2",
+        "executable": "ninja.exe",
+        "url": "https://github.com/ninja-build/ninja/releases/download/v1.10.2/ninja-win.zip",
+        "sha512": "6004140d92e86afbb17b49c49037ccd0786ce238f340f7d0e62b4b0c29ed0d6ad0bab11feda2094ae849c387d70d63504393714ed0a1f4d3a1f155af7a4f1ba3",
+        "archive": "ninja-win-1.10.2.zip"
+    },
+    {
+        "name": "ninja",
+        "os": "windows",
+        "version": "1.10.2",
+        "executable": "ninja.exe",
+        "url": "https://github.com/ninja-build/ninja/releases/download/v1.10.2/ninja-win.zip",
+        "sha512": "6004140d92e86afbb17b49c49037ccd0786ce238f340f7d0e62b4b0c29ed0d6ad0bab11feda2094ae849c387d70d63504393714ed0a1f4d3a1f155af7a4f1ba3",
+        "archive": "ninja-win-1.10.2.zip"
+    },
+    {
+        "name": "cmake",
+        "os": "windows",
+        "version": "3.22.2",
+        "executable": "cmake-3.22.2-windows-i386\\bin\\cmake.exe",
+        "url": "https://github.com/Kitware/CMake/releases/download/v3.22.2/cmake-3.22.2-windows-i386.zip",
+        "sha512": "969d3d58d56d8fa3cc3acae2b949bf58abab945f70ae292ff20c9060d845dfc094c613c367a924abff47f307cc33af1467cdb9b75bb857868e38b2c7cdc72f79",
+        "archive": "cmake-3.22.2-windows-i386.zip"
+    },
+    {
+        "name": "cmake",
+        "os": "osx",
+        "version": "3.22.2",
+        "executable": "cmake-3.22.2-macos-universal/CMake.app/Contents/bin/cmake",
+        "url": "https://github.com/Kitware/CMake/releases/download/v3.22.2/cmake-3.22.2-macos-universal.tar.gz",
+        "sha512": "08104f608ecb9a5cfef38e79f0957d21e425616c0677781445492f82cbfec805113e3b5eb4bc737b707bb26a00678e7bd55e17555a5611c08b0b9b44ac5136ac",
+        "archive": "cmake-3.22.2-macos-universal.tar.gz"
+    },
+    {
+        "name": "cmake",
+        "os": "linux",
+        "version": "3.22.2",
+        "executable": "cmake-3.22.2-linux-x86_64/bin/cmake",
+        "url": "https://github.com/Kitware/CMake/releases/download/v3.22.2/cmake-3.22.2-linux-x86_64.tar.gz",
+        "sha512": "579e08b086f6903ef063697fca1dc2692f68a7341dd35998990b772b4221cdb5b1deecfa73bad9d46817ef09e58882b2adff9d64f959c01002c11448a878746b",
+        "archive": "cmake-3.22.2linux-x86_64.tar.gz"
+    },
+    {
+        "name": "cmake",
+        "os": "freebsd",
+        "version": "3.20.4",
+        "executable": "/usr/local/bin/cmake",
+        "url": "https://pkg.freebsd.org/FreeBSD:13:amd64/quarterly/All/cmake-3.20.4.txz",
+        "sha512": "3e5b675d7ff924f92996d912e2365582e687375109ef99c9073fb8196bb329243a406b218cf1358d7cc518988b311ce9e5bf87de4d64f2e6377b7c2bc8894475",
+        "archive": "cmake-3.20.4.txz"
+    }]
+}
+'@ | % { $_ -replace "`r","" } | Out-File -enc ascii $VcpkgToolsJson
+
+    if (-not (Test-Json -ea:0 -LiteralPath $VcpkgToolsJson -SchemaFile $VcpkgToolsJsonSchemaFile)) {
+        throw "testing vcpkg-tools.json doesn't conform to schema"
+    }
 
     $env:VCPKG_DOWNLOADS = Join-Path $TestingRoot 'down loads'
     Run-Vcpkg -TestArgs ($commonArgs + @("fetch", "7zip", "--vcpkg-root=$TestingRoot"))
     Throw-IfFailed
-    Require-FileExists "$TestingRoot/down loads/tools/7zip-19.00-windows/Files/7-Zip/7z.exe"
+    Require-FileExists "$env:VCPKG_DOWNLOADS/tools/7zip-19.00-windows/Files/7-Zip/7z.exe"
 
     Run-Vcpkg -TestArgs ($commonArgs + @("fetch", "ninja-testing", "--vcpkg-root=$TestingRoot"))
     Throw-IfFailed
-    Require-FileExists "$TestingRoot/down loads/tools/ninja-testing-1.10.2-windows/ninja.exe"
+    Require-FileExists "$env:VCPKG_DOWNLOADS/tools/ninja-testing-1.10.2-windows/ninja.exe"
 
     $path = $env:PATH
 
-    $env:PATH = "$path;$TestingRoot/down loads/tools/ninja-testing-1.10.2-windows"
+    $env:PATH = "$path;$env:VCPKG_DOWNLOADS/tools/ninja-testing-1.10.2-windows"
     Run-Vcpkg -TestArgs ($commonArgs + @("fetch", "ninja", "--vcpkg-root=$TestingRoot"))
     Throw-IfFailed
-    Require-FileNotExists "$TestingRoot/down loads/tools/ninja-1.10.2-windows/ninja.exe"
+    Require-FileNotExists "$env:VCPKG_DOWNLOADS/tools/ninja-1.10.2-windows/ninja.exe"
 
     $env:VCPKG_FORCE_DOWNLOADED_BINARIES = "1"
     Run-Vcpkg -TestArgs ($commonArgs + @("fetch", "ninja", "--vcpkg-root=$TestingRoot"))
     Throw-IfFailed
-    Require-FileExists "$TestingRoot/down loads/tools/ninja-1.10.2-windows/ninja.exe"
+    Require-FileExists "$env:VCPKG_DOWNLOADS/tools/ninja-1.10.2-windows/ninja.exe"
 
-    Remove-Item -Recurse -Force "$TestingRoot/down loads/tools/ninja-1.10.2-windows" -ErrorAction SilentlyContinue
+    Remove-Item -Recurse -Force "$env:VCPKG_DOWNLOADS/tools/ninja-1.10.2-windows" -ErrorAction SilentlyContinue
     Remove-Item env:VCPKG_FORCE_DOWNLOADED_BINARIES
 
     $env:VCPKG_FORCE_SYSTEM_BINARIES = "1"
     $env:PATH = "$PSScriptRoot\..\e2e-assets\fetch;$path"
     Run-Vcpkg -TestArgs ($commonArgs + @("fetch", "ninja", "--vcpkg-root=$TestingRoot"))
     Throw-IfFailed
-    Require-FileNotExists "$TestingRoot/down loads/tools/ninja-1.10.2-windows/ninja.exe"
+    Require-FileNotExists "$env:VCPKG_DOWNLOADS/tools/ninja-1.10.2-windows/ninja.exe"
 
     Remove-Item env:VCPKG_FORCE_SYSTEM_BINARIES
     $out = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("fetch", "ninja", "--vcpkg-root=$TestingRoot", "--x-stderr-status"))

diff --git a/azure-pipelines/end-to-end-tests.ps1 b/azure-pipelines/end-to-end-tests.ps1
@@ -41,6 +41,10 @@ if ($PSVersionTable.PSVersion.Major -lt 7) {
     Write-Error "vcpkg end to end tests must use pwsh rather than Windows PowerShell"
 }
 
+# If you get an error on the next line, install Pester from an administrative command prompt with:
+# Install-Module -Name Pester -Force -MinimumVersion '5.6.1' -MaximumVersion '5.99' -Scope AllUsers
+Import-Module Pester -Force -MinimumVersion '5.6.1' -MaximumVersion '5.99'
+
 if ($IsLinux) {
     $Triplet = 'x64-linux'
 } elseif ($IsMacOS) {

diff --git a/docs/vcpkg-tools.schema.json b/docs/vcpkg-tools.schema.json
@@ -0,0 +1,69 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "https://raw.githubusercontent.com/microsoft/vcpkg-tool/main/docs/vcpkg-tools.schema.json",
+  "type": "object",
+  "properties": {
+    "schema-version": {
+      "description": "Identifies the version of the tools database format; currently always 1.",
+      "type": "number",
+      "minimum": 1,
+      "maximum": 1
+    },
+    "tools": {
+      "type": "array",
+      "description": "The set of known tools.",
+      "items": {
+        "type": "object",
+        "properties": {
+          "name": {
+            "type": "string",
+            "description": "The name of the tool as exposed to `vcpkg fetch`."
+          },
+          "os": {
+            "type": "string",
+            "description": "The platform where the record is valid.",
+            "enum": [ "windows", "osx", "linux", "freebsd", "openbsd" ]
+          },
+          "version": {
+            "type": "string",
+            "description": "The version to match against against a system provided copy of this tool.",
+            "pattern": "\\d+(\\.\\d+(\\.\\d+)?)?"
+          },
+          "arch": {
+            "type": "string",
+            "description": "The architecture where the record is valid.",
+            "enum": [ "x86", "x64", "amd64", "arm", "arm64", "arm64ec", "s390x", "ppc64le", "riscv32", "riscv64", "loongarch32", "loongarch64", "mips64" ]
+          },
+          "executable": {
+            "type": "string",
+            "description": "The relative path to the executable for the tool within the downloaded archive, if any."
+          },
+          "url": {
+            "type": "string",
+            "description": "The URL to download the tool from.",
+            "format": "uri"
+          },
+          "sha512": {
+            "type": "string",
+            "description": "The SHA-512 hash of the downloaded archive.",
+            "pattern": "^[0-9a-fA-F]{128}$"
+          },
+          "archive": {
+            "type": "string",
+            "description": "The name of the downloaded archive."
+          }
+        },
+        "patternProperties": {
+          "^\\$": {}
+        },
+        "required": ["name", "os", "version"],
+        "additionalProperties": false
+      }
+    }
+  },
+  "patternProperties": {
+    "^\\$": {}
+  },
+  "additionalProperties": false,
+  "required": [ "schema-version", "tools" ]
+}
diff --git a/include/vcpkg/base/contractual-constants.h b/include/vcpkg/base/contractual-constants.h
@@ -12,9 +12,11 @@ namespace vcpkg
     inline constexpr StringLiteral JsonIdAcquiredArtifacts = "acquired-artifacts";
     inline constexpr StringLiteral JsonIdActivatedArtifacts = "activated-artifacts";
     inline constexpr StringLiteral JsonIdAlgorithm = "algorithm";
+    inline constexpr StringLiteral JsonIdArchive = "archive";
     inline constexpr StringLiteral JsonIdAllCapsSHA256 = "SHA256";
     inline constexpr StringLiteral JsonIdAllCapsSHA512 = "SHA512";
     inline constexpr StringLiteral JsonIdApply = "apply";
+    inline constexpr StringLiteral JsonIdArch = "arch";
     inline constexpr StringLiteral JsonIdArchiveCapitalLocation = "archiveLocation";
     inline constexpr StringLiteral JsonIdArtifact = "artifact";
     inline constexpr StringLiteral JsonIdBaseline = "baseline";
@@ -46,6 +48,7 @@ namespace vcpkg
     inline constexpr StringLiteral JsonIdDollarSchema = "$schema";
     inline constexpr StringLiteral JsonIdDownloads = "downloads";
     inline constexpr StringLiteral JsonIdError = "error";
+    inline constexpr StringLiteral JsonIdExecutable = "executable";
     inline constexpr StringLiteral JsonIdFeatures = "features";
     inline constexpr StringLiteral JsonIdFiles = "files";
     inline constexpr StringLiteral JsonIdFilesystem = "filesystem";
@@ -68,6 +71,7 @@ namespace vcpkg
     inline constexpr StringLiteral JsonIdMessage = "message";
     inline constexpr StringLiteral JsonIdMicrosoft = "microsoft";
     inline constexpr StringLiteral JsonIdName = "name";
+    inline constexpr StringLiteral JsonIdOS = "os";
     inline constexpr StringLiteral JsonIdOverlayPorts = "overlay-ports";
     inline constexpr StringLiteral JsonIdOverlayTriplets = "overlay-triplets";
     inline constexpr StringLiteral JsonIdOverrides = "overrides";
@@ -87,8 +91,10 @@ namespace vcpkg
     inline constexpr StringLiteral JsonIdRequires = "requires";
     inline constexpr StringLiteral JsonIdResolved = "resolved";
     inline constexpr StringLiteral JsonIdScanned = "scanned";
+    inline constexpr StringLiteral JsonIdSchemaVersion = "schema-version";
     inline constexpr StringLiteral JsonIdSettings = "settings";
     inline constexpr StringLiteral JsonIdSha = "sha";
+    inline constexpr StringLiteral JsonIdSha512 = "sha512";
     inline constexpr StringLiteral JsonIdState = "state";
     inline constexpr StringLiteral JsonIdSummary = "summary";
     inline constexpr StringLiteral JsonIdSupports = "supports";
@@ -290,6 +296,7 @@ namespace vcpkg
     inline constexpr StringLiteral SwitchTargetX86 = "target:x86";
     inline constexpr StringLiteral SwitchTLogFile = "tlog-file";
     inline constexpr StringLiteral SwitchTools = "tools";
+    inline constexpr StringLiteral SwitchToolDataFile = "tool-data-file";
     inline constexpr StringLiteral SwitchTriplet = "triplet";
     inline constexpr StringLiteral SwitchUrl = "url";
     inline constexpr StringLiteral SwitchVcpkgRoot = "vcpkg-root";

diff --git a/include/vcpkg/base/jsonreader.h b/include/vcpkg/base/jsonreader.h
@@ -1,6 +1,7 @@
 #pragma once
 
 #include <vcpkg/base/fwd/json.h>
+#include <vcpkg/base/fwd/system.h>
 
 #include <vcpkg/base/chrono.h>
 #include <vcpkg/base/json.h>
@@ -30,7 +31,7 @@ namespace vcpkg::Json
         virtual Optional<Type> visit_string(Reader&, StringView) const;
         virtual Optional<Type> visit_array(Reader&, const Array&) const;
         virtual Optional<Type> visit_object(Reader&, const Object&) const;
-        virtual View<StringView> valid_fields() const;
+        virtual View<StringLiteral> valid_fields() const noexcept;
 
         virtual ~IDeserializer() = default;
 
@@ -100,7 +101,7 @@ namespace vcpkg::Json
         // * are not in `valid_fields`
         // if known_fields.empty(), then it's treated as if all field names are valid
         void check_for_unexpected_fields(const Object& obj,
-                                         View<StringView> valid_fields,
+                                         View<StringLiteral> valid_fields,
                                          const LocalizedString& type_name);
 
         template<class Type>
@@ -238,7 +239,7 @@ namespace vcpkg::Json
     }
 
     template<class Type>
-    View<StringView> IDeserializer<Type>::valid_fields() const
+    View<StringLiteral> IDeserializer<Type>::valid_fields() const noexcept
     {
         return {};
     }
@@ -361,4 +362,18 @@ namespace vcpkg::Json
         virtual Optional<std::string> visit_string(Json::Reader&, StringView sv) const override;
         static const FeatureNameDeserializer instance;
     };
+
+    struct ArchitectureDeserializer final : Json::IDeserializer<Optional<CPUArchitecture>>
+    {
+        virtual LocalizedString type_name() const override;
+        virtual Optional<Optional<CPUArchitecture>> visit_string(Json::Reader&, StringView sv) const override;
+        static const ArchitectureDeserializer instance;
+    };
+
+    struct Sha512Deserializer final : Json::IDeserializer<std::string>
+    {
+        virtual LocalizedString type_name() const override;
+        virtual Optional<std::string> visit_string(Json::Reader&, StringView sv) const override;
+        static const Sha512Deserializer instance;
+    };
 }