-
Notifications
You must be signed in to change notification settings - Fork 531
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Adds a local copy of the npm packages for the validator #3241
base: main
Are you sure you want to change the base?
Changes from all commits
066058b
9f720a7
67b5f92
255276b
9126fdc
ec52987
8ae853c
de641e4
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,27 @@ | ||
// ------------------------------------------------------------------------------------------------- | ||
// Copyright (c) Microsoft Corporation. All rights reserved. | ||
// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. | ||
// ------------------------------------------------------------------------------------------------- | ||
|
||
using System.Collections.Generic; | ||
|
||
namespace Microsoft.Health.Fhir.Core.Configs; | ||
|
||
public class FhirPackageConfiguration | ||
{ | ||
/// <summary> | ||
/// True if the names of the default FHIR packages should be included automatically. | ||
/// hl7.fhir.[R4] and hl7.fhir.[R4].expansions. | ||
/// </summary> | ||
public bool IncludeDefaultPackages { get; set; } | ||
|
||
/// <summary> | ||
/// Specify the package server if different to http://packages.fhir.org. | ||
/// </summary> | ||
public string PackageSource { get; set; } | ||
|
||
/// <summary> | ||
/// Names of the packages to be included for validation. | ||
/// </summary> | ||
public ICollection<string> PackageNames { get; } = new HashSet<string>(); | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,26 @@ | ||
// ------------------------------------------------------------------------------------------------- | ||
// Copyright (c) Microsoft Corporation. All rights reserved. | ||
// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. | ||
// ------------------------------------------------------------------------------------------------- | ||
|
||
using Microsoft.Health.Fhir.Core.Features.Validation; | ||
using Microsoft.Health.Fhir.Tests.Common; | ||
using Microsoft.Health.Test.Utilities; | ||
using Xunit; | ||
|
||
namespace Microsoft.Health.Fhir.Core.UnitTests.Features.Validation; | ||
|
||
[Trait(Traits.OwningTeam, OwningTeam.Fhir)] | ||
[Trait(Traits.Category, Categories.Operations)] | ||
public class ProfileValidatorTests | ||
{ | ||
[Fact] | ||
public void GivenAProfileValidator_WhenUsingReflectedVariables_TheyCanAllBeResolved() | ||
{ | ||
(string Server, string CorePackageName, string ExpansionsPackageName) variables = ProfileValidator.GetFhirPackageVariables(); | ||
|
||
Assert.NotEmpty(variables.Server); | ||
Assert.NotEmpty(variables.CorePackageName); | ||
Assert.NotEmpty(variables.ExpansionsPackageName); | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -4,43 +4,139 @@ | |
// ------------------------------------------------------------------------------------------------- | ||
|
||
using System; | ||
using System.Collections.Generic; | ||
using System.IO; | ||
using System.Linq; | ||
using System.Reflection; | ||
using System.Threading; | ||
using EnsureThat; | ||
using Hl7.Fhir.ElementModel; | ||
using Hl7.Fhir.Model; | ||
using Hl7.Fhir.Specification.Source; | ||
using Hl7.Fhir.Validation; | ||
using Microsoft.Extensions.Hosting; | ||
using Microsoft.Extensions.Logging; | ||
using Microsoft.Extensions.Options; | ||
using Microsoft.Health.Fhir.Core.Configs; | ||
using Microsoft.Health.Fhir.Core.Models; | ||
using Polly; | ||
using Task = System.Threading.Tasks.Task; | ||
|
||
namespace Microsoft.Health.Fhir.Core.Features.Validation | ||
{ | ||
public class ProfileValidator : IProfileValidator | ||
public class ProfileValidator : IProfileValidator, IHostedService | ||
{ | ||
private readonly IResourceResolver _resolver; | ||
private readonly ILogger<ProfileValidator> _logger; | ||
private readonly IProvideProfilesForValidation _profilesResolver; | ||
private readonly Lazy<IResourceResolver> _resolver; | ||
private readonly CoreFeatureConfiguration _coreConfig; | ||
private readonly ValidateOperationConfiguration _options; | ||
|
||
public ProfileValidator(IProvideProfilesForValidation profilesResolver, IOptions<ValidateOperationConfiguration> options) | ||
public ProfileValidator( | ||
IProvideProfilesForValidation profilesResolver, | ||
IOptions<CoreFeatureConfiguration> coreFeatureConfiguration, | ||
IOptions<ValidateOperationConfiguration> options, | ||
ILogger<ProfileValidator> logger) | ||
{ | ||
EnsureArg.IsNotNull(profilesResolver, nameof(profilesResolver)); | ||
EnsureArg.IsNotNull(options?.Value, nameof(options)); | ||
_profilesResolver = EnsureArg.IsNotNull(profilesResolver, nameof(profilesResolver)); | ||
_options = EnsureArg.IsNotNull(options?.Value, nameof(options)); | ||
_coreConfig = EnsureArg.IsNotNull(coreFeatureConfiguration?.Value, nameof(options)); | ||
_logger = EnsureArg.IsNotNull(logger, nameof(logger)); | ||
|
||
_resolver = new Lazy<IResourceResolver>( | ||
() => Policy.Handle<Exception>() | ||
.WaitAndRetry(5, i => TimeSpan.FromSeconds(i * 2)) | ||
.Execute(Initialize), | ||
LazyThreadSafetyMode.ExecutionAndPublication); | ||
} | ||
|
||
private IResourceResolver Initialize() | ||
{ | ||
try | ||
{ | ||
_resolver = new MultiResolver(new CachedResolver(ZipSource.CreateValidationSource(), options.Value.CacheDurationInSeconds), profilesResolver); | ||
(string Server, string CorePackageName, string ExpansionsPackageName) packageVariables = GetFhirPackageVariables(); | ||
|
||
var packagesToInclude = new HashSet<string>(); | ||
|
||
if (_coreConfig.PackageConfiguration.IncludeDefaultPackages) | ||
{ | ||
packagesToInclude.Add(packageVariables.CorePackageName); | ||
packagesToInclude.Add(packageVariables.ExpansionsPackageName); | ||
|
||
#if !R5 | ||
// elements package name is not included in the reflection metadata, but we should try and include it | ||
var extensionsPackageName = packageVariables.ExpansionsPackageName.Replace("expansions", "elements", StringComparison.OrdinalIgnoreCase); | ||
LTA-Thinking marked this conversation as resolved.
Show resolved
Hide resolved
|
||
packagesToInclude.Add(extensionsPackageName); | ||
#endif | ||
} | ||
|
||
foreach (var package in _coreConfig.PackageConfiguration.PackageNames) | ||
{ | ||
packagesToInclude.Add(package); | ||
} | ||
|
||
FhirPackageSource validationSource; | ||
|
||
if (!string.IsNullOrEmpty(_coreConfig.PackageConfiguration.PackageSource) && | ||
!_coreConfig.PackageConfiguration.PackageSource.StartsWith("http", StringComparison.OrdinalIgnoreCase)) | ||
{ | ||
// Allow local packages | ||
var resolvedPath = Path.GetFullPath(_coreConfig.PackageConfiguration.PackageSource, Environment.CurrentDirectory); | ||
|
||
var packageFileList = Directory.GetFiles(resolvedPath) | ||
.Where(f => packagesToInclude.Any(p => f.Contains(p, StringComparison.OrdinalIgnoreCase) | ||
&& f.EndsWith("tgz", StringComparison.OrdinalIgnoreCase))).ToArray(); | ||
|
||
if (packageFileList.Length != packagesToInclude.Count) | ||
{ | ||
_logger.LogWarning( | ||
"Attempted to load {DesiredPackageList} packages, but found {ActualPackageList} in file system.", | ||
string.Join(',', packagesToInclude), | ||
string.Join(',', packageFileList)); | ||
} | ||
|
||
validationSource = new FhirPackageSource(packageFileList); | ||
} | ||
else | ||
{ | ||
// Look for packages on an npm server | ||
var server = string.IsNullOrEmpty(_coreConfig.PackageConfiguration.PackageSource) ? packageVariables.Server : _coreConfig.PackageConfiguration.PackageSource; | ||
validationSource = new FhirPackageSource(server, packagesToInclude.ToArray()); | ||
} | ||
|
||
// Ensure packages are downloaded to temp folder | ||
var patient = validationSource.ListResourceUris(ResourceType.Patient).ToArray(); | ||
|
||
return new MultiResolver(_profilesResolver, new CachedResolver(validationSource, _options.CacheDurationInSeconds)); | ||
} | ||
catch (Exception) | ||
catch (Exception ex) | ||
{ | ||
// Something went wrong during profile loading, what should we do? | ||
_logger.LogError(ex, "Error initializing profile validator"); | ||
throw; | ||
} | ||
} | ||
|
||
internal static (string Server, string CorePackageName, string ExpansionsPackageName) GetFhirPackageVariables() | ||
{ | ||
Type type = typeof(FhirPackageSource); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Where is this coming from? I can't find reference to this FhirPackageSource or where its fields are defined. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. These are compiled into the Firely libraries, it isn't a great dependency, but its a way to find the NPM urls. Perhaps we should define these ourselves? |
||
FieldInfo serverField = type.GetField("FHIR_PACKAGE_SERVER", BindingFlags.Static | BindingFlags.NonPublic); | ||
FieldInfo coreField = type.GetField("FHIR_CORE_PACKAGE_NAME", BindingFlags.Static | BindingFlags.NonPublic); | ||
FieldInfo expansionsField = type.GetField("FHIR_CORE_EXPANSIONS_PACKAGE_NAME", BindingFlags.Static | BindingFlags.NonPublic); | ||
|
||
var corePackageName = coreField?.GetValue(null) as string; | ||
var expansionsPackageName = expansionsField?.GetValue(null) as string; | ||
|
||
corePackageName = corePackageName.Replace("xml", string.Empty, StringComparison.OrdinalIgnoreCase); | ||
expansionsPackageName = expansionsPackageName.Replace("xml", string.Empty, StringComparison.OrdinalIgnoreCase); | ||
|
||
return (serverField?.GetValue(null) as string, corePackageName, expansionsPackageName); | ||
} | ||
|
||
private Validator GetValidator() | ||
{ | ||
var ctx = new ValidationSettings() | ||
{ | ||
ResourceResolver = _resolver, | ||
ResourceResolver = _resolver.Value, | ||
GenerateSnapshot = true, | ||
Trace = false, | ||
ResolveExternalReferences = false, | ||
|
@@ -77,5 +173,23 @@ public OperationOutcomeIssue[] TryValidate(ITypedElement resource, string profil | |
|
||
return outcomeIssues; | ||
} | ||
|
||
public Task StartAsync(CancellationToken cancellationToken) | ||
{ | ||
Task.Run( | ||
() => | ||
{ | ||
IResourceResolver resolver = _resolver.Value; | ||
_logger.LogInformation($"Profile validator {resolver.GetType()} initialized"); | ||
}, | ||
cancellationToken).ConfigureAwait(false); | ||
|
||
return Task.CompletedTask; | ||
} | ||
|
||
public Task StopAsync(CancellationToken cancellationToken) | ||
{ | ||
return Task.CompletedTask; | ||
} | ||
} | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Should we be checking these into OSS?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This is a good question. Previously we did reference the specification.zip file, which was part of the nuget that we were using. This is the same content, but in this case we are including the package in our repo.