Skip to content

Commit f351510

Browse files
vicromsdata-queueBillyONeal
authored
Change tool metadata file format to JSON + support arm64 host fetches (microsoft#1553)
Co-authored-by: data-queue <[email protected]> Co-authored-by: Billy Robert O'Neal III <[email protected]>
1 parent 18a617c commit f351510

31 files changed

+955
-392
lines changed

‎.github/workflows/build.yaml

-1
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,6 @@ jobs:
8080
shell: pwsh
8181
run: |
8282
cd out/build/${{ matrix.preset }}
83-
Import-Module Pester -Force -MinimumVersion '5.6.1' -MaximumVersion '5.99'
8483
${{ github.workspace }}/azure-pipelines/end-to-end-tests.ps1 -RunArtifactsTests
8584
env:
8685
VCPKG_ROOT: ${{ github.workspace }}/vcpkg-root

‎azure-pipelines/end-to-end-tests-dir/fetch.ps1

+89-61
Original file line numberDiff line numberDiff line change
@@ -1,95 +1,123 @@
11
. $PSScriptRoot/../end-to-end-tests-prelude.ps1
22

3+
$VcpkgToolsJsonSchemaFile = (Get-Item "$PSScriptRoot/../../docs/vcpkg-tools.schema.json").FullName
4+
if (-not (Test-Json -ea:0 -LiteralPath "$VcpkgRoot/scripts/vcpkg-tools.json" -SchemaFile $VcpkgToolsJsonSchemaFile)) {
5+
throw "real vcpkg-tools.json doesn't conform to schema"
6+
}
7+
38
if (-not $IsMacOS -and -not $IsLinux) {
49
"" | Out-File -enc ascii $(Join-Path $TestingRoot .vcpkg-root)
510

611
$Scripts = Join-Path $TestingRoot "scripts"
712
mkdir $Scripts | Out-Null
13+
14+
$VcpkgToolsJson = Join-Path $Scripts "vcpkg-tools.json"
15+
16+
$7zip_version = "19.00"
17+
$ninja_version = "1.10.2"
818

9-
@"
10-
<?xml version="1.0"?>
11-
<tools version="2">
12-
<tool name="7zip" os="windows">
13-
<version>19.00</version>
14-
<exeRelativePath>Files\7-Zip\7z.exe</exeRelativePath>
15-
<url>https://www.7-zip.org/a/7z1900-x64.msi</url>
16-
<sha512>7837a8677a01eed9c3309923f7084bc864063ba214ee169882c5b04a7a8b198ed052c15e981860d9d7952c98f459a4fab87a72fd78e7d0303004dcb86f4324c8</sha512>
17-
<archiveName>7z1900-x64.msi</archiveName>
18-
</tool>
19-
<tool name="ninja-testing" os="windows">
20-
<version>1.10.2</version>
21-
<exeRelativePath>ninja.exe</exeRelativePath>
22-
<url>https://github.com/ninja-build/ninja/releases/download/v1.10.2/ninja-win.zip</url>
23-
<sha512>6004140d92e86afbb17b49c49037ccd0786ce238f340f7d0e62b4b0c29ed0d6ad0bab11feda2094ae849c387d70d63504393714ed0a1f4d3a1f155af7a4f1ba3</sha512>
24-
<archiveName>ninja-win-1.10.2.zip</archiveName>
25-
</tool>
26-
<tool name="ninja" os="windows">
27-
<version>1.10.2</version>
28-
<exeRelativePath>ninja.exe</exeRelativePath>
29-
<url>https://github.com/ninja-build/ninja/releases/download/v1.10.2/ninja-win.zip</url>
30-
<sha512>6004140d92e86afbb17b49c49037ccd0786ce238f340f7d0e62b4b0c29ed0d6ad0bab11feda2094ae849c387d70d63504393714ed0a1f4d3a1f155af7a4f1ba3</sha512>
31-
<archiveName>ninja-win-1.10.2.zip</archiveName>
32-
</tool>
33-
<tool name="cmake" os="windows">
34-
<version>3.22.2</version>
35-
<exeRelativePath>cmake-3.22.2-windows-i386\bin\cmake.exe</exeRelativePath>
36-
<url>https://github.com/Kitware/CMake/releases/download/v3.22.2/cmake-3.22.2-windows-i386.zip</url>
37-
<sha512>969d3d58d56d8fa3cc3acae2b949bf58abab945f70ae292ff20c9060d845dfc094c613c367a924abff47f307cc33af1467cdb9b75bb857868e38b2c7cdc72f79</sha512>
38-
<archiveName>cmake-3.22.2-windows-i386.zip</archiveName>
39-
</tool>
40-
<tool name="cmake" os="osx">
41-
<version>3.22.2</version>
42-
<exeRelativePath>cmake-3.22.2-macos-universal/CMake.app/Contents/bin/cmake</exeRelativePath>
43-
<url>https://github.com/Kitware/CMake/releases/download/v3.22.2/cmake-3.22.2-macos-universal.tar.gz</url>
44-
<sha512>08104f608ecb9a5cfef38e79f0957d21e425616c0677781445492f82cbfec805113e3b5eb4bc737b707bb26a00678e7bd55e17555a5611c08b0b9b44ac5136ac</sha512>
45-
<archiveName>cmake-3.22.2-macos-universal.tar.gz</archiveName>
46-
</tool>
47-
<tool name="cmake" os="linux">
48-
<version>3.22.2</version>
49-
<exeRelativePath>cmake-3.22.2-linux-x86_64/bin/cmake</exeRelativePath>
50-
<url>https://github.com/Kitware/CMake/releases/download/v3.22.2/cmake-3.22.2-linux-x86_64.tar.gz</url>
51-
<sha512>579e08b086f6903ef063697fca1dc2692f68a7341dd35998990b772b4221cdb5b1deecfa73bad9d46817ef09e58882b2adff9d64f959c01002c11448a878746b</sha512>
52-
<archiveName>cmake-3.22.2linux-x86_64.tar.gz</archiveName>
53-
</tool>
54-
<tool name="cmake" os="freebsd">
55-
<version>3.20.4</version>
56-
<exeRelativePath>usr/local/bin/cmake</exeRelativePath>
57-
<url>https://pkg.freebsd.org/FreeBSD:13:amd64/quarterly/All/cmake-3.20.4.txz</url>
58-
<sha512>3e5b675d7ff924f92996d912e2365582e687375109ef99c9073fb8196bb329243a406b218cf1358d7cc518988b311ce9e5bf87de4d64f2e6377b7c2bc8894475</sha512>
59-
<archiveName>cmake-3.20.4.txz</archiveName>
60-
</tool>
61-
</tools>
62-
"@ | % { $_ -replace "`r","" } | Out-File -enc ascii $(Join-Path $Scripts "vcpkgTools.xml")
19+
@'
20+
{
21+
"schema-version": 1,
22+
"tools": [{
23+
"name": "7zip",
24+
"os": "windows",
25+
"version": "19.00",
26+
"executable": "Files\\7-Zip\\7z.exe",
27+
"url": "https://www.7-zip.org/a/7z1900-x64.msi",
28+
"sha512": "7837a8677a01eed9c3309923f7084bc864063ba214ee169882c5b04a7a8b198ed052c15e981860d9d7952c98f459a4fab87a72fd78e7d0303004dcb86f4324c8",
29+
"archive": "7z1900-x64.msi"
30+
},
31+
{
32+
"name": "ninja-testing",
33+
"os": "windows",
34+
"version": "1.10.2",
35+
"executable": "ninja.exe",
36+
"url": "https://github.com/ninja-build/ninja/releases/download/v1.10.2/ninja-win.zip",
37+
"sha512": "6004140d92e86afbb17b49c49037ccd0786ce238f340f7d0e62b4b0c29ed0d6ad0bab11feda2094ae849c387d70d63504393714ed0a1f4d3a1f155af7a4f1ba3",
38+
"archive": "ninja-win-1.10.2.zip"
39+
},
40+
{
41+
"name": "ninja",
42+
"os": "windows",
43+
"version": "1.10.2",
44+
"executable": "ninja.exe",
45+
"url": "https://github.com/ninja-build/ninja/releases/download/v1.10.2/ninja-win.zip",
46+
"sha512": "6004140d92e86afbb17b49c49037ccd0786ce238f340f7d0e62b4b0c29ed0d6ad0bab11feda2094ae849c387d70d63504393714ed0a1f4d3a1f155af7a4f1ba3",
47+
"archive": "ninja-win-1.10.2.zip"
48+
},
49+
{
50+
"name": "cmake",
51+
"os": "windows",
52+
"version": "3.22.2",
53+
"executable": "cmake-3.22.2-windows-i386\\bin\\cmake.exe",
54+
"url": "https://github.com/Kitware/CMake/releases/download/v3.22.2/cmake-3.22.2-windows-i386.zip",
55+
"sha512": "969d3d58d56d8fa3cc3acae2b949bf58abab945f70ae292ff20c9060d845dfc094c613c367a924abff47f307cc33af1467cdb9b75bb857868e38b2c7cdc72f79",
56+
"archive": "cmake-3.22.2-windows-i386.zip"
57+
},
58+
{
59+
"name": "cmake",
60+
"os": "osx",
61+
"version": "3.22.2",
62+
"executable": "cmake-3.22.2-macos-universal/CMake.app/Contents/bin/cmake",
63+
"url": "https://github.com/Kitware/CMake/releases/download/v3.22.2/cmake-3.22.2-macos-universal.tar.gz",
64+
"sha512": "08104f608ecb9a5cfef38e79f0957d21e425616c0677781445492f82cbfec805113e3b5eb4bc737b707bb26a00678e7bd55e17555a5611c08b0b9b44ac5136ac",
65+
"archive": "cmake-3.22.2-macos-universal.tar.gz"
66+
},
67+
{
68+
"name": "cmake",
69+
"os": "linux",
70+
"version": "3.22.2",
71+
"executable": "cmake-3.22.2-linux-x86_64/bin/cmake",
72+
"url": "https://github.com/Kitware/CMake/releases/download/v3.22.2/cmake-3.22.2-linux-x86_64.tar.gz",
73+
"sha512": "579e08b086f6903ef063697fca1dc2692f68a7341dd35998990b772b4221cdb5b1deecfa73bad9d46817ef09e58882b2adff9d64f959c01002c11448a878746b",
74+
"archive": "cmake-3.22.2linux-x86_64.tar.gz"
75+
},
76+
{
77+
"name": "cmake",
78+
"os": "freebsd",
79+
"version": "3.20.4",
80+
"executable": "/usr/local/bin/cmake",
81+
"url": "https://pkg.freebsd.org/FreeBSD:13:amd64/quarterly/All/cmake-3.20.4.txz",
82+
"sha512": "3e5b675d7ff924f92996d912e2365582e687375109ef99c9073fb8196bb329243a406b218cf1358d7cc518988b311ce9e5bf87de4d64f2e6377b7c2bc8894475",
83+
"archive": "cmake-3.20.4.txz"
84+
}]
85+
}
86+
'@ | % { $_ -replace "`r","" } | Out-File -enc ascii $VcpkgToolsJson
87+
88+
if (-not (Test-Json -ea:0 -LiteralPath $VcpkgToolsJson -SchemaFile $VcpkgToolsJsonSchemaFile)) {
89+
throw "testing vcpkg-tools.json doesn't conform to schema"
90+
}
6391

6492
$env:VCPKG_DOWNLOADS = Join-Path $TestingRoot 'down loads'
6593
Run-Vcpkg -TestArgs ($commonArgs + @("fetch", "7zip", "--vcpkg-root=$TestingRoot"))
6694
Throw-IfFailed
67-
Require-FileExists "$TestingRoot/down loads/tools/7zip-19.00-windows/Files/7-Zip/7z.exe"
95+
Require-FileExists "$env:VCPKG_DOWNLOADS/tools/7zip-19.00-windows/Files/7-Zip/7z.exe"
6896

6997
Run-Vcpkg -TestArgs ($commonArgs + @("fetch", "ninja-testing", "--vcpkg-root=$TestingRoot"))
7098
Throw-IfFailed
71-
Require-FileExists "$TestingRoot/down loads/tools/ninja-testing-1.10.2-windows/ninja.exe"
99+
Require-FileExists "$env:VCPKG_DOWNLOADS/tools/ninja-testing-1.10.2-windows/ninja.exe"
72100

73101
$path = $env:PATH
74102

75-
$env:PATH = "$path;$TestingRoot/down loads/tools/ninja-testing-1.10.2-windows"
103+
$env:PATH = "$path;$env:VCPKG_DOWNLOADS/tools/ninja-testing-1.10.2-windows"
76104
Run-Vcpkg -TestArgs ($commonArgs + @("fetch", "ninja", "--vcpkg-root=$TestingRoot"))
77105
Throw-IfFailed
78-
Require-FileNotExists "$TestingRoot/down loads/tools/ninja-1.10.2-windows/ninja.exe"
106+
Require-FileNotExists "$env:VCPKG_DOWNLOADS/tools/ninja-1.10.2-windows/ninja.exe"
79107

80108
$env:VCPKG_FORCE_DOWNLOADED_BINARIES = "1"
81109
Run-Vcpkg -TestArgs ($commonArgs + @("fetch", "ninja", "--vcpkg-root=$TestingRoot"))
82110
Throw-IfFailed
83-
Require-FileExists "$TestingRoot/down loads/tools/ninja-1.10.2-windows/ninja.exe"
111+
Require-FileExists "$env:VCPKG_DOWNLOADS/tools/ninja-1.10.2-windows/ninja.exe"
84112

85-
Remove-Item -Recurse -Force "$TestingRoot/down loads/tools/ninja-1.10.2-windows" -ErrorAction SilentlyContinue
113+
Remove-Item -Recurse -Force "$env:VCPKG_DOWNLOADS/tools/ninja-1.10.2-windows" -ErrorAction SilentlyContinue
86114
Remove-Item env:VCPKG_FORCE_DOWNLOADED_BINARIES
87115

88116
$env:VCPKG_FORCE_SYSTEM_BINARIES = "1"
89117
$env:PATH = "$PSScriptRoot\..\e2e-assets\fetch;$path"
90118
Run-Vcpkg -TestArgs ($commonArgs + @("fetch", "ninja", "--vcpkg-root=$TestingRoot"))
91119
Throw-IfFailed
92-
Require-FileNotExists "$TestingRoot/down loads/tools/ninja-1.10.2-windows/ninja.exe"
120+
Require-FileNotExists "$env:VCPKG_DOWNLOADS/tools/ninja-1.10.2-windows/ninja.exe"
93121

94122
Remove-Item env:VCPKG_FORCE_SYSTEM_BINARIES
95123
$out = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("fetch", "ninja", "--vcpkg-root=$TestingRoot", "--x-stderr-status"))

‎azure-pipelines/end-to-end-tests.ps1

+4
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,10 @@ if ($PSVersionTable.PSVersion.Major -lt 7) {
4141
Write-Error "vcpkg end to end tests must use pwsh rather than Windows PowerShell"
4242
}
4343

44+
# If you get an error on the next line, install Pester from an administrative command prompt with:
45+
# Install-Module -Name Pester -Force -MinimumVersion '5.6.1' -MaximumVersion '5.99' -Scope AllUsers
46+
Import-Module Pester -Force -MinimumVersion '5.6.1' -MaximumVersion '5.99'
47+
4448
if ($IsLinux) {
4549
$Triplet = 'x64-linux'
4650
} elseif ($IsMacOS) {

‎docs/vcpkg-tools.schema.json

+69
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
{
2+
"$schema": "http://json-schema.org/draft-07/schema#",
3+
"$id": "https://raw.githubusercontent.com/microsoft/vcpkg-tool/main/docs/vcpkg-tools.schema.json",
4+
"type": "object",
5+
"properties": {
6+
"schema-version": {
7+
"description": "Identifies the version of the tools database format; currently always 1.",
8+
"type": "number",
9+
"minimum": 1,
10+
"maximum": 1
11+
},
12+
"tools": {
13+
"type": "array",
14+
"description": "The set of known tools.",
15+
"items": {
16+
"type": "object",
17+
"properties": {
18+
"name": {
19+
"type": "string",
20+
"description": "The name of the tool as exposed to `vcpkg fetch`."
21+
},
22+
"os": {
23+
"type": "string",
24+
"description": "The platform where the record is valid.",
25+
"enum": [ "windows", "osx", "linux", "freebsd", "openbsd" ]
26+
},
27+
"version": {
28+
"type": "string",
29+
"description": "The version to match against against a system provided copy of this tool.",
30+
"pattern": "\\d+(\\.\\d+(\\.\\d+)?)?"
31+
},
32+
"arch": {
33+
"type": "string",
34+
"description": "The architecture where the record is valid.",
35+
"enum": [ "x86", "x64", "amd64", "arm", "arm64", "arm64ec", "s390x", "ppc64le", "riscv32", "riscv64", "loongarch32", "loongarch64", "mips64" ]
36+
},
37+
"executable": {
38+
"type": "string",
39+
"description": "The relative path to the executable for the tool within the downloaded archive, if any."
40+
},
41+
"url": {
42+
"type": "string",
43+
"description": "The URL to download the tool from.",
44+
"format": "uri"
45+
},
46+
"sha512": {
47+
"type": "string",
48+
"description": "The SHA-512 hash of the downloaded archive.",
49+
"pattern": "^[0-9a-fA-F]{128}$"
50+
},
51+
"archive": {
52+
"type": "string",
53+
"description": "The name of the downloaded archive."
54+
}
55+
},
56+
"patternProperties": {
57+
"^\\$": {}
58+
},
59+
"required": ["name", "os", "version"],
60+
"additionalProperties": false
61+
}
62+
}
63+
},
64+
"patternProperties": {
65+
"^\\$": {}
66+
},
67+
"additionalProperties": false,
68+
"required": [ "schema-version", "tools" ]
69+
}

‎include/vcpkg/base/contractual-constants.h

+7
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,11 @@ namespace vcpkg
1212
inline constexpr StringLiteral JsonIdAcquiredArtifacts = "acquired-artifacts";
1313
inline constexpr StringLiteral JsonIdActivatedArtifacts = "activated-artifacts";
1414
inline constexpr StringLiteral JsonIdAlgorithm = "algorithm";
15+
inline constexpr StringLiteral JsonIdArchive = "archive";
1516
inline constexpr StringLiteral JsonIdAllCapsSHA256 = "SHA256";
1617
inline constexpr StringLiteral JsonIdAllCapsSHA512 = "SHA512";
1718
inline constexpr StringLiteral JsonIdApply = "apply";
19+
inline constexpr StringLiteral JsonIdArch = "arch";
1820
inline constexpr StringLiteral JsonIdArchiveCapitalLocation = "archiveLocation";
1921
inline constexpr StringLiteral JsonIdArtifact = "artifact";
2022
inline constexpr StringLiteral JsonIdBaseline = "baseline";
@@ -46,6 +48,7 @@ namespace vcpkg
4648
inline constexpr StringLiteral JsonIdDollarSchema = "$schema";
4749
inline constexpr StringLiteral JsonIdDownloads = "downloads";
4850
inline constexpr StringLiteral JsonIdError = "error";
51+
inline constexpr StringLiteral JsonIdExecutable = "executable";
4952
inline constexpr StringLiteral JsonIdFeatures = "features";
5053
inline constexpr StringLiteral JsonIdFiles = "files";
5154
inline constexpr StringLiteral JsonIdFilesystem = "filesystem";
@@ -68,6 +71,7 @@ namespace vcpkg
6871
inline constexpr StringLiteral JsonIdMessage = "message";
6972
inline constexpr StringLiteral JsonIdMicrosoft = "microsoft";
7073
inline constexpr StringLiteral JsonIdName = "name";
74+
inline constexpr StringLiteral JsonIdOS = "os";
7175
inline constexpr StringLiteral JsonIdOverlayPorts = "overlay-ports";
7276
inline constexpr StringLiteral JsonIdOverlayTriplets = "overlay-triplets";
7377
inline constexpr StringLiteral JsonIdOverrides = "overrides";
@@ -87,8 +91,10 @@ namespace vcpkg
8791
inline constexpr StringLiteral JsonIdRequires = "requires";
8892
inline constexpr StringLiteral JsonIdResolved = "resolved";
8993
inline constexpr StringLiteral JsonIdScanned = "scanned";
94+
inline constexpr StringLiteral JsonIdSchemaVersion = "schema-version";
9095
inline constexpr StringLiteral JsonIdSettings = "settings";
9196
inline constexpr StringLiteral JsonIdSha = "sha";
97+
inline constexpr StringLiteral JsonIdSha512 = "sha512";
9298
inline constexpr StringLiteral JsonIdState = "state";
9399
inline constexpr StringLiteral JsonIdSummary = "summary";
94100
inline constexpr StringLiteral JsonIdSupports = "supports";
@@ -290,6 +296,7 @@ namespace vcpkg
290296
inline constexpr StringLiteral SwitchTargetX86 = "target:x86";
291297
inline constexpr StringLiteral SwitchTLogFile = "tlog-file";
292298
inline constexpr StringLiteral SwitchTools = "tools";
299+
inline constexpr StringLiteral SwitchToolDataFile = "tool-data-file";
293300
inline constexpr StringLiteral SwitchTriplet = "triplet";
294301
inline constexpr StringLiteral SwitchUrl = "url";
295302
inline constexpr StringLiteral SwitchVcpkgRoot = "vcpkg-root";

‎include/vcpkg/base/jsonreader.h

+18-3
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
#pragma once
22

33
#include <vcpkg/base/fwd/json.h>
4+
#include <vcpkg/base/fwd/system.h>
45

56
#include <vcpkg/base/chrono.h>
67
#include <vcpkg/base/json.h>
@@ -30,7 +31,7 @@ namespace vcpkg::Json
3031
virtual Optional<Type> visit_string(Reader&, StringView) const;
3132
virtual Optional<Type> visit_array(Reader&, const Array&) const;
3233
virtual Optional<Type> visit_object(Reader&, const Object&) const;
33-
virtual View<StringView> valid_fields() const;
34+
virtual View<StringLiteral> valid_fields() const noexcept;
3435

3536
virtual ~IDeserializer() = default;
3637

@@ -100,7 +101,7 @@ namespace vcpkg::Json
100101
// * are not in `valid_fields`
101102
// if known_fields.empty(), then it's treated as if all field names are valid
102103
void check_for_unexpected_fields(const Object& obj,
103-
View<StringView> valid_fields,
104+
View<StringLiteral> valid_fields,
104105
const LocalizedString& type_name);
105106

106107
template<class Type>
@@ -238,7 +239,7 @@ namespace vcpkg::Json
238239
}
239240

240241
template<class Type>
241-
View<StringView> IDeserializer<Type>::valid_fields() const
242+
View<StringLiteral> IDeserializer<Type>::valid_fields() const noexcept
242243
{
243244
return {};
244245
}
@@ -361,4 +362,18 @@ namespace vcpkg::Json
361362
virtual Optional<std::string> visit_string(Json::Reader&, StringView sv) const override;
362363
static const FeatureNameDeserializer instance;
363364
};
365+
366+
struct ArchitectureDeserializer final : Json::IDeserializer<Optional<CPUArchitecture>>
367+
{
368+
virtual LocalizedString type_name() const override;
369+
virtual Optional<Optional<CPUArchitecture>> visit_string(Json::Reader&, StringView sv) const override;
370+
static const ArchitectureDeserializer instance;
371+
};
372+
373+
struct Sha512Deserializer final : Json::IDeserializer<std::string>
374+
{
375+
virtual LocalizedString type_name() const override;
376+
virtual Optional<std::string> visit_string(Json::Reader&, StringView sv) const override;
377+
static const Sha512Deserializer instance;
378+
};
364379
}

0 commit comments

Comments
 (0)