Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Adds a local copy of the npm packages for the validator #3241

Open
wants to merge 8 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,11 @@ public class CoreFeatureConfiguration
/// </summary>
public int DefaultIncludeCountPerSearch { get; set; } = 100;

/// <summary>
/// Configuration for loading packages in for validation.
/// </summary>
public FhirPackageConfiguration PackageConfiguration { get; set; } = new();

/// <summary>
/// Gets or sets a value whether we need to run profile validation during resource creation.
/// </summary>
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
// -------------------------------------------------------------------------------------------------
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
// -------------------------------------------------------------------------------------------------

using System.Collections.Generic;

namespace Microsoft.Health.Fhir.Core.Configs;

public class FhirPackageConfiguration
{
/// <summary>
/// True if the names of the default FHIR packages should be included automatically.
/// hl7.fhir.[R4] and hl7.fhir.[R4].expansions.
/// </summary>
public bool IncludeDefaultPackages { get; set; }

/// <summary>
/// Specify the package server if different to http://packages.fhir.org.
/// </summary>
public string PackageSource { get; set; }

/// <summary>
/// Names of the packages to be included for validation.
/// </summary>
public ICollection<string> PackageNames { get; } = new HashSet<string>();
}
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
<Project Sdk="Microsoft.NET.Sdk">
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<RootNamespace>Microsoft.Health.Fhir.Api</RootNamespace>
</PropertyGroup>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,24 @@
<RootNamespace>Microsoft.Health.Fhir.Web</RootNamespace>
</PropertyGroup>

<ItemGroup>
<None Remove="hl7.fhir.r4.core-4.0.1.tgz" />
<None Remove="hl7.fhir.r4.elements-4.0.1.tgz" />
<None Remove="hl7.fhir.r4.expansions-4.0.1.tgz" />
</ItemGroup>

<ItemGroup>
<Content Include="hl7.fhir.r4.core-4.0.1.tgz">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="hl7.fhir.r4.elements-4.0.1.tgz">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="hl7.fhir.r4.expansions-4.0.1.tgz">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
Comment on lines +15 to +23
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we be checking these into OSS?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is a good question. Previously we did reference the specification.zip file, which was part of the nuget that we were using. This is the same content, but in this case we are including the package in our repo.

</ItemGroup>

<ItemGroup>
<PackageReference Include="IdentityServer4" />
<PackageReference Include="Microsoft.ApplicationInsights.AspNetCore" />
Expand Down
Binary file not shown.
Binary file not shown.
Binary file not shown.
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,24 @@
<RootNamespace>Microsoft.Health.Fhir.Web</RootNamespace>
</PropertyGroup>

<ItemGroup>
<None Remove="hl7.fhir.r4b.core-4.3.0.tgz" />
<None Remove="hl7.fhir.r4b.elements-4.3.0.tgz" />
<None Remove="hl7.fhir.r4b.expansions-4.3.0.tgz" />
</ItemGroup>

<ItemGroup>
<Content Include="hl7.fhir.r4b.core-4.3.0.tgz">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="hl7.fhir.r4b.elements-4.3.0.tgz">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="hl7.fhir.r4b.expansions-4.3.0.tgz">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
</ItemGroup>

<ItemGroup>
<PackageReference Include="IdentityServer4" />
<PackageReference Include="Microsoft.ApplicationInsights.AspNetCore" />
Expand Down
Binary file not shown.
Binary file not shown.
Binary file not shown.
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,20 @@
<RootNamespace>Microsoft.Health.Fhir.Web</RootNamespace>
</PropertyGroup>

<ItemGroup>
<None Remove="hl7.fhir.r5.core#5.0.0.tgz" />
<None Remove="hl7.fhir.r5.expansions#5.0.0.tgz" />
</ItemGroup>

<ItemGroup>
<Content Include="hl7.fhir.r5.core#5.0.0.tgz">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="hl7.fhir.r5.expansions#5.0.0.tgz">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
</ItemGroup>

<ItemGroup>
<PackageReference Include="IdentityServer4" />
<PackageReference Include="Microsoft.ApplicationInsights.AspNetCore" />
Expand Down
Binary file not shown.
Binary file not shown.
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,10 @@ public void Load(IServiceCollection services)
services.AddSingleton<ISupportedProfilesStore>(x => x.GetRequiredService<ServerProvideProfileValidation>());
services.AddSingleton<IProvideProfilesForValidation>(x => x.GetRequiredService<ServerProvideProfileValidation>());

services.AddSingleton<IProfileValidator, ProfileValidator>();
services.Add<ProfileValidator>()
.Singleton()
.AsSelf()
.AsImplementedInterfaces();
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
// -------------------------------------------------------------------------------------------------
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
// -------------------------------------------------------------------------------------------------

using Microsoft.Health.Fhir.Core.Features.Validation;
using Microsoft.Health.Fhir.Tests.Common;
using Microsoft.Health.Test.Utilities;
using Xunit;

namespace Microsoft.Health.Fhir.Core.UnitTests.Features.Validation;

[Trait(Traits.OwningTeam, OwningTeam.Fhir)]
[Trait(Traits.Category, Categories.Operations)]
public class ProfileValidatorTests
{
[Fact]
public void GivenAProfileValidator_WhenUsingReflectedVariables_TheyCanAllBeResolved()
{
(string Server, string CorePackageName, string ExpansionsPackageName) variables = ProfileValidator.GetFhirPackageVariables();

Assert.NotEmpty(variables.Server);
Assert.NotEmpty(variables.CorePackageName);
Assert.NotEmpty(variables.ExpansionsPackageName);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
<Compile Include="$(MSBuildThisFileDirectory)Features\Search\TypedElementSearchIndexerTests.cs" />
<Compile Include="$(MSBuildThisFileDirectory)Features\Search\USCoreTestHelper.cs" />
<Compile Include="$(MSBuildThisFileDirectory)Features\Security\Authorization\RoleBasedFhirAuthorizationServiceTests.cs" />
<Compile Include="$(MSBuildThisFileDirectory)Features\Validation\ProfileValidatorTests.cs" />
<Compile Include="$(MSBuildThisFileDirectory)Features\Validation\ResourceProfileValidatorTests.cs" />
<Compile Include="..\Microsoft.Health.Fhir.Shared.Core.UnitTests\Features\Persistence\ResourceWrapperFactoryTests.cs" />
<Compile Include="$(MSBuildThisFileDirectory)Features\Search\Converters\AddressToStringSearchValueConverterTests.cs" />
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,43 +4,139 @@
// -------------------------------------------------------------------------------------------------

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Threading;
using EnsureThat;
using Hl7.Fhir.ElementModel;
using Hl7.Fhir.Model;
using Hl7.Fhir.Specification.Source;
using Hl7.Fhir.Validation;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Microsoft.Health.Fhir.Core.Configs;
using Microsoft.Health.Fhir.Core.Models;
using Polly;
using Task = System.Threading.Tasks.Task;

namespace Microsoft.Health.Fhir.Core.Features.Validation
{
public class ProfileValidator : IProfileValidator
public class ProfileValidator : IProfileValidator, IHostedService
{
private readonly IResourceResolver _resolver;
private readonly ILogger<ProfileValidator> _logger;
private readonly IProvideProfilesForValidation _profilesResolver;
private readonly Lazy<IResourceResolver> _resolver;
private readonly CoreFeatureConfiguration _coreConfig;
private readonly ValidateOperationConfiguration _options;

public ProfileValidator(IProvideProfilesForValidation profilesResolver, IOptions<ValidateOperationConfiguration> options)
public ProfileValidator(
IProvideProfilesForValidation profilesResolver,
IOptions<CoreFeatureConfiguration> coreFeatureConfiguration,
IOptions<ValidateOperationConfiguration> options,
ILogger<ProfileValidator> logger)
{
EnsureArg.IsNotNull(profilesResolver, nameof(profilesResolver));
EnsureArg.IsNotNull(options?.Value, nameof(options));
_profilesResolver = EnsureArg.IsNotNull(profilesResolver, nameof(profilesResolver));
_options = EnsureArg.IsNotNull(options?.Value, nameof(options));
_coreConfig = EnsureArg.IsNotNull(coreFeatureConfiguration?.Value, nameof(options));
_logger = EnsureArg.IsNotNull(logger, nameof(logger));

_resolver = new Lazy<IResourceResolver>(
() => Policy.Handle<Exception>()
.WaitAndRetry(5, i => TimeSpan.FromSeconds(i * 2))
.Execute(Initialize),
LazyThreadSafetyMode.ExecutionAndPublication);
}

private IResourceResolver Initialize()
{
try
{
_resolver = new MultiResolver(new CachedResolver(ZipSource.CreateValidationSource(), options.Value.CacheDurationInSeconds), profilesResolver);
(string Server, string CorePackageName, string ExpansionsPackageName) packageVariables = GetFhirPackageVariables();

var packagesToInclude = new HashSet<string>();

if (_coreConfig.PackageConfiguration.IncludeDefaultPackages)
{
packagesToInclude.Add(packageVariables.CorePackageName);
packagesToInclude.Add(packageVariables.ExpansionsPackageName);

#if !R5
// elements package name is not included in the reflection metadata, but we should try and include it
var extensionsPackageName = packageVariables.ExpansionsPackageName.Replace("expansions", "elements", StringComparison.OrdinalIgnoreCase);
packagesToInclude.Add(extensionsPackageName);
#endif
}

foreach (var package in _coreConfig.PackageConfiguration.PackageNames)
{
packagesToInclude.Add(package);
}

FhirPackageSource validationSource;

if (!string.IsNullOrEmpty(_coreConfig.PackageConfiguration.PackageSource) &&
!_coreConfig.PackageConfiguration.PackageSource.StartsWith("http", StringComparison.OrdinalIgnoreCase))
{
// Allow local packages
var resolvedPath = Path.GetFullPath(_coreConfig.PackageConfiguration.PackageSource, Environment.CurrentDirectory);

var packageFileList = Directory.GetFiles(resolvedPath)
.Where(f => packagesToInclude.Any(p => f.Contains(p, StringComparison.OrdinalIgnoreCase)
&& f.EndsWith("tgz", StringComparison.OrdinalIgnoreCase))).ToArray();

if (packageFileList.Length != packagesToInclude.Count)
{
_logger.LogWarning(
"Attempted to load {DesiredPackageList} packages, but found {ActualPackageList} in file system.",
string.Join(',', packagesToInclude),
string.Join(',', packageFileList));
}

validationSource = new FhirPackageSource(packageFileList);
}
else
{
// Look for packages on an npm server
var server = string.IsNullOrEmpty(_coreConfig.PackageConfiguration.PackageSource) ? packageVariables.Server : _coreConfig.PackageConfiguration.PackageSource;
validationSource = new FhirPackageSource(server, packagesToInclude.ToArray());
}

// Ensure packages are downloaded to temp folder
var patient = validationSource.ListResourceUris(ResourceType.Patient).ToArray();

return new MultiResolver(_profilesResolver, new CachedResolver(validationSource, _options.CacheDurationInSeconds));
}
catch (Exception)
catch (Exception ex)
{
// Something went wrong during profile loading, what should we do?
_logger.LogError(ex, "Error initializing profile validator");
throw;
}
}

internal static (string Server, string CorePackageName, string ExpansionsPackageName) GetFhirPackageVariables()
{
Type type = typeof(FhirPackageSource);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Where is this coming from? I can't find reference to this FhirPackageSource or where its fields are defined.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These are compiled into the Firely libraries, it isn't a great dependency, but its a way to find the NPM urls. Perhaps we should define these ourselves?
There is a unit test to cover they exist: https://github.com/microsoft/fhir-server/pull/3241/files#diff-98805c9eb649d4044301a52d69bb13b1e09768a441ded93ce851277986c8103aR18

FieldInfo serverField = type.GetField("FHIR_PACKAGE_SERVER", BindingFlags.Static | BindingFlags.NonPublic);
FieldInfo coreField = type.GetField("FHIR_CORE_PACKAGE_NAME", BindingFlags.Static | BindingFlags.NonPublic);
FieldInfo expansionsField = type.GetField("FHIR_CORE_EXPANSIONS_PACKAGE_NAME", BindingFlags.Static | BindingFlags.NonPublic);

var corePackageName = coreField?.GetValue(null) as string;
var expansionsPackageName = expansionsField?.GetValue(null) as string;

corePackageName = corePackageName.Replace("xml", string.Empty, StringComparison.OrdinalIgnoreCase);
expansionsPackageName = expansionsPackageName.Replace("xml", string.Empty, StringComparison.OrdinalIgnoreCase);

return (serverField?.GetValue(null) as string, corePackageName, expansionsPackageName);
}

private Validator GetValidator()
{
var ctx = new ValidationSettings()
{
ResourceResolver = _resolver,
ResourceResolver = _resolver.Value,
GenerateSnapshot = true,
Trace = false,
ResolveExternalReferences = false,
Expand Down Expand Up @@ -77,5 +173,23 @@ public OperationOutcomeIssue[] TryValidate(ITypedElement resource, string profil

return outcomeIssues;
}

public Task StartAsync(CancellationToken cancellationToken)
{
Task.Run(
() =>
{
IResourceResolver resolver = _resolver.Value;
_logger.LogInformation($"Profile validator {resolver.GetType()} initialized");
},
cancellationToken).ConfigureAwait(false);

return Task.CompletedTask;
}

public Task StopAsync(CancellationToken cancellationToken)
{
return Task.CompletedTask;
}
}
}
5 changes: 5 additions & 0 deletions src/Microsoft.Health.Fhir.Shared.Web/appsettings.json
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,11 @@
"SupportsTransaction": true,
"SupportsSelectiveSearchParameters": false,
"IncludeTotalInBundle": "None",
"PackageConfiguration": {
"IncludeDefaultPackages": true,
"PackageSource": "https://pkgs.dev.azure.com/microsofthealthoss/FhirServer/_packaging/Public/npm/registry/",
"PackageNames": []
},
"ProfileValidationOnCreate": false,
"ProfileValidationOnUpdate": false,
"SupportsResourceChangeCapture": false,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,24 @@
<RootNamespace>Microsoft.Health.Fhir.Web</RootNamespace>
</PropertyGroup>

<ItemGroup>
<None Remove="hl7.fhir.r3.core-3.0.2.tgz" />
<None Remove="hl7.fhir.r3.elements-3.0.2.tgz" />
<None Remove="hl7.fhir.r3.expansions-3.0.2.tgz" />
</ItemGroup>

<ItemGroup>
<Content Include="hl7.fhir.r3.core-3.0.2.tgz">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="hl7.fhir.r3.elements-3.0.2.tgz">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="hl7.fhir.r3.expansions-3.0.2.tgz">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
</ItemGroup>

<ItemGroup>
<PackageReference Include="IdentityServer4" />
<PackageReference Include="Microsoft.ApplicationInsights.AspNetCore" />
Expand Down
Binary file not shown.
Binary file not shown.
Binary file not shown.
Loading