Skip to content

Commit 49725e3

Browse files
ViktorHoferViktorHofer
committed
Remove VMR inner-build concept
1. Remove the repo source-build infrastructure: - The outer/inner-build concept - The SB intermediate package creation 2. Remove the DotNetBuildPhase property and replace its usage with the now only two inclusive properties. Update docs. 3. Move the ArPow inner clone target into the VMR orchestrator and integrate it into the repo infra (still conditional only for SBE). 4. Move the ArPow target that handles creating the package usage data and the package report into the VMR orchestrator 5. Update the remaining DotNetBuild.props files with the new property, item and target names. Extra work: - Move the `WritePrebuiltUsageData` and `ReportPrebuiltUsage` targets that were executed only for dotnet.proj into finish-source-only.proj to better distinguish them from the newly added ArPow targets that got moved into the VMR orchestrator. - Place the "prebuilt-report" logs under "artifacts\log\<config>\" and avoid the now unnecessary copy step in the YML. - Remove unnecessary SourceBuild.props / DotNetBuild.props which only contained properties that were used for repo source-build. - Rename the remaining few SourceBuild.props to DotNetBuild.props
1 parent 0f66536 commit 49725e3

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

52 files changed

+380
-1660
lines changed

Diff for: ‎Directory.Build.props

+4-5
Original file line numberDiff line numberDiff line change
@@ -174,14 +174,13 @@
174174
<PrebuiltSourceBuiltPackagesPath>$([MSBuild]::NormalizeDirectory('$(PrereqsPackagesDir)', 'previously-source-built'))</PrebuiltSourceBuiltPackagesPath>
175175
<PrebuiltSourceBuiltPackagesPath Condition="'$(CustomPrebuiltSourceBuiltPackagesPath)' != ''">$([MSBuild]::EnsureTrailingSlash('$(CustomPrebuiltSourceBuiltPackagesPath)'))</PrebuiltSourceBuiltPackagesPath>
176176

177-
<PackageReportDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'prebuilt-report'))</PackageReportDir>
178-
<ResultingPrebuiltPackagesDir>$([MSBuild]::NormalizeDirectory('$(PackageReportDir)', 'prebuilt-packages'))</ResultingPrebuiltPackagesDir>
179177
<SbrpRepoSrcDir>$([MSBuild]::NormalizeDirectory('$(SrcDir)', 'source-build-reference-packages', 'src'))</SbrpRepoSrcDir>
180178
<ReferencePackagesDir>$([MSBuild]::NormalizeDirectory('$(PrereqsPackagesDir)', 'reference'))</ReferencePackagesDir>
181179
<SourceBuiltArtifactsTarballName>Private.SourceBuilt.Artifacts</SourceBuiltArtifactsTarballName>
182180
<SourceBuiltPrebuiltsTarballName>Private.SourceBuilt.Prebuilts</SourceBuiltPrebuiltsTarballName>
183181

184182
<BaselineDataFile>$(ToolsDir)prebuilt-baseline.xml</BaselineDataFile>
183+
<PackageVersionsDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsObjDir)', 'PackageVersions'))</PackageVersionsDir>
185184
</PropertyGroup>
186185

187186
<!-- Build task assembly paths -->
@@ -192,9 +191,9 @@
192191

193192
<PropertyGroup Condition="'$(EnablePoison)' == 'true'">
194193
<PoisonMarkerFile>.prebuilt.xml</PoisonMarkerFile>
195-
<PoisonReportDataFile>$(PackageReportDir)poison-catalog.xml</PoisonReportDataFile>
196-
<PoisonedReportFile>$(PackageReportDir)poisoned.txt</PoisonedReportFile>
197-
<PoisonUsageReportFile>$(PackageReportDir)poison-usage.xml</PoisonUsageReportFile>
194+
<PoisonReportDataFile>$(ArtifactsLogDir)poison-catalog.xml</PoisonReportDataFile>
195+
<PoisonedReportFile>$(ArtifactsLogDir)poisoned.txt</PoisonedReportFile>
196+
<PoisonUsageReportFile>$(ArtifactsLogDir)poison-usage.xml</PoisonUsageReportFile>
198197
</PropertyGroup>
199198

200199
<PropertyGroup>

Diff for: ‎eng/finish-source-only.proj

+134-17
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,16 @@
22
<Project Sdk="Microsoft.Build.NoTargets">
33

44
<PropertyGroup>
5+
<TargetFramework>$(NetCurrent)</TargetFramework>
6+
57
<!-- Need to set to false to calculate RepositoryCommit. -->
68
<EnableSourceControlManagerQueries>false</EnableSourceControlManagerQueries>
7-
<TargetFramework>$(NetCurrent)</TargetFramework>
9+
10+
<PackageReportDataFile>$(ArtifactsLogDir)prebuilt-usage.xml</PackageReportDataFile>
11+
<ProjectAssetsJsonArchiveFile>$(ArtifactsLogDir)all-project-assets-json-files.zip</ProjectAssetsJsonArchiveFile>
12+
<ProdConManifestFile>$(ArtifactsLogDir)prodcon-build.xml</ProdConManifestFile>
13+
14+
<ResultingPrebuiltPackagesDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsLogDir)', 'prebuilt-packages'))</ResultingPrebuiltPackagesDir>
815
</PropertyGroup>
916

1017
<ItemGroup>
@@ -14,11 +21,132 @@
1421
<ProjectReference Include="$(RepositoryEngineeringDir)merge-asset-manifests.proj" />
1522
</ItemGroup>
1623

17-
<!-- After building, generate a prebuilt usage report. -->
18-
<Target Name="ReportPrebuiltUsage"
24+
<UsingTask TaskName="Microsoft.DotNet.UnifiedBuild.Tasks.UsageReport.WritePackageUsageData" AssemblyFile="$(MicrosoftDotNetUnifiedBuildTasksAssembly)" TaskFactory="TaskHostFactory" />
25+
<Target Name="WritePrebuiltUsageData"
1926
BeforeTargets="Build"
20-
Condition="'$(SkipReportPrebuiltUsage)' != 'true'">
21-
<MSBuild Projects="$(RepoProjectsDir)$(RootRepo).proj" Targets="WritePrebuiltUsageData;ReportPrebuiltUsage" />
27+
Inputs="$(MSBuildProjectFullPath)"
28+
Outputs="$(BaseIntermediateOutputPath)WritePrebuiltUsageData.complete">
29+
<ItemGroup>
30+
<!--
31+
Note the usage of */*/*.nupkg and not **/*.nupkg. The latter would include nupkgs
32+
carried as content, i.e. in Microsoft.FSharp.Compiler nupkg, and cause false positives.
33+
Content nupkgs were not restored and should not be used in prebuit detection.
34+
35+
Example of a restored package:
36+
$(NuGetPackageRoot)microsoft.fsharp.compiler/12.8.300-beta.24203.1/microsoft.fsharp.compiler.12.8.300-beta.24203.1.nupkg
37+
38+
Example of a content package:
39+
$(NuGetPackageRoot)microsoft.fsharp.compiler/12.8.300-beta.24203.1/contentFiles/Release/FSharp.Core.8.0.300.nupkg
40+
-->
41+
<RestoredPackageFile Include="$(NuGetPackageRoot)*/*/*.nupkg"
42+
Exclude="$(NuGetPackageRoot)tests/**/*.nupkg" />
43+
44+
<!-- Only contains packages when building. -->
45+
<TarballPrebuiltPackageFile Include="$(PrebuiltPackagesPath)*.nupkg" />
46+
47+
<SourceBuiltPackageFile Include="$(ArtifactsPackagesDir)**/*.nupkg" />
48+
<SourceBuiltPackageFile Include="$(PrebuiltSourceBuiltPackagesPath)*.nupkg" />
49+
<ReferencePackageFile Include="$(ReferencePackagesDir)**/*.nupkg" />
50+
51+
<!-- Check all RIDs from all restored Microsoft.NETCore.Platforms packages. -->
52+
<PlatformsRuntimeJsonFile Include="$(NuGetPackageRoot)microsoft.netcore.platforms/*/PortableRuntimeIdentifierGraph.json" />
53+
</ItemGroup>
54+
55+
<ItemGroup>
56+
<RepoProject Include="$(RepoProjectsDir)*.proj" />
57+
<RepoProjectDirectory Include="@(RepoProject->'$(SrcDir)%(Filename)')" />
58+
59+
<!-- Add some other potential top-level project directories for a more specific report. -->
60+
<RepoProjectDirectory Include="$(SourceBuiltSdksDir);$(TasksDir);$(ArtifactsObjDir)" />
61+
<!-- Finally, scan entire source-build, in case project.assets.json ends up in an unexpected place. -->
62+
<RepoProjectDirectory Include="$(RepoRoot)" />
63+
</ItemGroup>
64+
65+
<ItemGroup>
66+
<!-- This file is a resource tracked by Git, not generated by restore. Ignore false positive. -->
67+
<IgnoredProjectAssetsJsonFile Include="$(SrcDir)*nuget-client*/**/test/NuGet.Core.Tests/NuGet.Build.Tasks.Pack.Test/compiler/resources/project.assets.json"/>
68+
</ItemGroup>
69+
70+
<WritePackageUsageData
71+
RestoredPackageFiles="@(RestoredPackageFile)"
72+
TarballPrebuiltPackageFiles="@(TarballPrebuiltPackageFile)"
73+
SourceBuiltPackageFiles="@(SourceBuiltPackageFile)"
74+
ReferencePackageFiles="@(ReferencePackageFile)"
75+
PlatformsRuntimeJsonFiles="@(PlatformsRuntimeJsonFile)"
76+
TargetRid="$(TargetRid)"
77+
ProjectDirectories="@(RepoProjectDirectory)"
78+
RootDir="$(RepoRoot)"
79+
IgnoredProjectAssetsJsonFiles="@(IgnoredProjectAssetsJsonFile)"
80+
DataFile="$(PackageReportDataFile)"
81+
ProjectAssetsJsonArchiveFile="$(ProjectAssetsJsonArchiveFile)" />
82+
83+
<!-- Copy packages detected as prebuilts to the artifacts prebuilt folder -->
84+
<ItemGroup>
85+
<AllowedPackageFile Include="@(TarballPrebuiltPackageFile)" />
86+
<AllowedPackageFile Include="@(SourceBuiltPackageFile)" />
87+
<AllowedPackageFile Include="@(ReferencePackageFile)" />
88+
<AllowedPackageFile>
89+
<LCFilename>$([System.String]::Copy(%(Filename)).ToLowerInvariant())</LCFilename>
90+
</AllowedPackageFile>
91+
92+
<PrebuiltPackageFile Include="@(RestoredPackageFile)" >
93+
<LCFilename>$([System.String]::Copy(%(Filename)).ToLowerInvariant())</LCFilename>
94+
</PrebuiltPackageFile>
95+
<PrebuiltPackageFile Remove="@(AllowedPackageFile)" MatchOnMetadata="LCFilename" />
96+
</ItemGroup>
97+
98+
<Copy SourceFiles="@(PrebuiltPackageFile)" DestinationFolder="$(ResultingPrebuiltPackagesDir)" />
99+
100+
<MakeDir Directories="$(BaseIntermediateOutputPath)" />
101+
<Touch Files="$(BaseIntermediateOutputPath)WritePrebuiltUsageData.complete" AlwaysCreate="true">
102+
<Output TaskParameter="TouchedFiles" ItemName="FileWrites" />
103+
</Touch>
104+
</Target>
105+
106+
<!-- After building, generate a prebuilt usage report.
107+
Also, dev scenario: rewrite a prebuilt-report. This makes it easy to add data to an existing
108+
prebuilt report without performing another full build. This doesn't reevalutate which packages
109+
are prebuilts or search the projects: it uses the existing usage.xml file to generate report
110+
XMLs based on the info put in artifacts/. -->
111+
<UsingTask TaskName="Microsoft.DotNet.UnifiedBuild.Tasks.UsageReport.ValidateUsageAgainstBaseline" AssemblyFile="$(MicrosoftDotNetUnifiedBuildTasksAssembly)" TaskFactory="TaskHostFactory" />
112+
<UsingTask TaskName="Microsoft.DotNet.UnifiedBuild.Tasks.UsageReport.WriteUsageReports" AssemblyFile="$(MicrosoftDotNetUnifiedBuildTasksAssembly)" TaskFactory="TaskHostFactory" />
113+
<Target Name="ReportPrebuiltUsage"
114+
Inputs="$(MSBuildProjectFullPath)"
115+
Outputs="$(BaseIntermediateOutputPath)ReportPrebuiltUsage.complete"
116+
DependsOnTargets="WritePrebuiltUsageData"
117+
BeforeTargets="Build">
118+
<PropertyGroup>
119+
<FailOnPrebuiltBaselineError Condition="'$(FailOnPrebuiltBaselineError)' == ''">false</FailOnPrebuiltBaselineError>
120+
</PropertyGroup>
121+
122+
<ItemGroup>
123+
<PackageVersionPropsSavedSnapshotFile Include="$(PackageVersionsDir)PackageVersions.*.Snapshot.props" />
124+
</ItemGroup>
125+
<Copy SourceFiles="@(PackageVersionPropsSnapshotFiles)" DestinationFolder="$(ArtifactsLogDir)snapshots/" />
126+
127+
<WriteUsageReports DataFile="$(PackageReportDataFile)"
128+
PackageVersionPropsSnapshots="@(PackageVersionPropsSavedSnapshotFile)"
129+
ProdConBuildManifestFile="$(ProdConManifestFile)"
130+
PoisonedReportFile="$(PoisonedReportFile)"
131+
OutputDirectory="$(ArtifactsLogDir)" />
132+
133+
<PropertyGroup Condition="'$(ContinueOnPrebuiltBaselineError)' == ''">
134+
<ContinueOnPrebuiltBaselineError>false</ContinueOnPrebuiltBaselineError>
135+
<ContinueOnPrebuiltBaselineError Condition="'$(FailOnPrebuiltBaselineError)' != 'true'">true</ContinueOnPrebuiltBaselineError>
136+
</PropertyGroup>
137+
138+
<ValidateUsageAgainstBaseline
139+
DataFile="$(PackageReportDataFile)"
140+
BaselineDataFile="$(BaselineDataFile)"
141+
OutputBaselineFile="$(ArtifactsLogDir)generated-new-baseline.xml"
142+
OutputReportFile="$(ArtifactsLogDir)baseline-comparison.xml"
143+
AllowTestProjectUsage="$(AllowTestProjectUsage)"
144+
ContinueOnError="$(ContinueOnPrebuiltBaselineError)" />
145+
146+
<MakeDir Directories="$(BaseIntermediateOutputPath)" />
147+
<Touch Files="$(BaseIntermediateOutputPath)ReportPrebuiltUsage.complete" AlwaysCreate="true">
148+
<Output TaskParameter="TouchedFiles" ItemName="FileWrites" />
149+
</Touch>
22150
</Target>
23151

24152
<UsingTask TaskName="Microsoft.DotNet.UnifiedBuild.Tasks.WriteSBRPUsageReport" AssemblyFile="$(MicrosoftDotNetUnifiedBuildTasksAssembly)" TaskFactory="TaskHostFactory" />
@@ -110,17 +238,6 @@
110238
<RemoveDir Directories="$(IntermediateSdkSymbolsLayout)" />
111239
</Target>
112240

113-
<!--
114-
Dev scenario: rewrite a prebuilt-report. This makes it easy to add data to an existing
115-
prebuilt report without performing another full build. This doesn't reevalutate which packages
116-
are prebuilts or search the projects: it uses the existing usage.xml file to generate report
117-
XMLs based on the info put in artifacts/.
118-
-->
119-
<Target Name="RewritePrebuiltUsageReport">
120-
<MSBuild Projects="$(RepoProjectsDir)$(RootRepo).proj"
121-
Targets="ReportPrebuiltUsage" />
122-
</Target>
123-
124241
<UsingTask TaskName="Microsoft.DotNet.SourceBuild.Tasks.LeakDetection.CheckForPoison" AssemblyFile="$(MicrosoftDotNetSourceBuildTasksLeakDetectionAssembly)" TaskFactory="TaskHostFactory" Condition="'$(EnablePoison)' == 'true'" />
125242
<Target Name="ReportPoisonUsage"
126243
BeforeTargets="Build"
@@ -168,7 +285,7 @@
168285
Inputs="$(MSBuildProjectFullPath)"
169286
Outputs="$(BaseIntermediateOutputPath)GeneratePrebuiltBurndownData.complete" >
170287
<PropertyGroup>
171-
<PrebuiltBurndownDataFile>$(PackageReportDir)PrebuiltBurndownData.csv</PrebuiltBurndownDataFile>
288+
<PrebuiltBurndownDataFile>$(ArtifactsLogDir)PrebuiltBurndownData.csv</PrebuiltBurndownDataFile>
172289
</PropertyGroup>
173290

174291
<WriteUsageBurndownData RootDirectory="$(RepoRoot)"

Diff for: ‎eng/init-poison.proj

+1-1
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
Outputs="$(BaseIntermediateOutputPath)PoisonPrebuiltPackages.complete">
1616
<PropertyGroup>
1717
<SourceBuiltPoisonMarkerFile>.source-built.xml</SourceBuiltPoisonMarkerFile>
18-
<SourceBuiltPoisonReportDataFile>$(PackageReportDir)poison-source-built-catalog.xml</SourceBuiltPoisonReportDataFile>
18+
<SourceBuiltPoisonReportDataFile>$(ArtifactsLogDir)poison-source-built-catalog.xml</SourceBuiltPoisonReportDataFile>
1919
</PropertyGroup>
2020

2121
<ItemGroup>

0 commit comments

Comments
 (0)