diff --git a/docs/arch/adr-2603-atomic-searchparameter-crud-and-cache-refresh-ownership.md b/docs/arch/adr-2603-atomic-searchparameter-crud-and-cache-refresh-ownership.md new file mode 100644 index 0000000000..c0301020ef --- /dev/null +++ b/docs/arch/adr-2603-atomic-searchparameter-crud-and-cache-refresh-ownership.md @@ -0,0 +1,60 @@ +# ADR 2603: Atomic SearchParameter CRUD Operations +Labels: [SQL](https://github.com/microsoft/fhir-server/labels/Area-SQL) | [Core](https://github.com/microsoft/fhir-server/labels/Area-Core) | [SearchParameter](https://github.com/microsoft/fhir-server/labels/Area-SearchParameter) + +## Context +SearchParameter create, update, and delete operations require two coordinated writes: the SearchParameter status row (`dbo.SearchParam` table) and the resource itself (`dbo.Resource` via `dbo.MergeResources`). Previously, these writes occurred in separate steps in the request pipeline, creating partial-failure windows where one could succeed while the other fails — producing orphaned or inconsistent state. + +Composing these writes into a single atomic SQL operation introduced a new problem for transaction bundles: `dbo.MergeSearchParams` acquires an exclusive lock on `dbo.SearchParam` per entry, and that lock is held by the outer bundle transaction. When the next entry's behavior pipeline calls `GetAllSearchParameterStatus` on a separate connection, it blocks on the same table, causing a timeout. This required a deferred-flush approach for transaction bundles. + +Additionally, SearchParameter CRUD behaviors previously performed direct in-memory cache mutations (`AddNewSearchParameters`, `DeleteSearchParameter`, etc.) during the request pipeline. This duplicated responsibility with the `SearchParameterCacheRefreshBackgroundService`, which already polls the database and applies cache updates across all instances. + +Key considerations: +- Eliminating partial-commit windows between status and resource persistence. +- Handling the lock contention on `dbo.SearchParam` introduced by composed writes in transaction bundles. +- Simplifying cache ownership by removing direct cache mutations from the CRUD request path. +- Preserving existing behavior for non-SearchParameter resources and Cosmos DB paths. + +## Decision +We implement three complementary changes: + +### 1. Composed writes for single operations (SQL) +For individual SearchParameter CRUD, behaviors queue pending status updates in request context (`SearchParameter.PendingStatusUpdates`) instead of persisting directly. `SqlServerFhirDataStore` detects pending statuses and calls `dbo.MergeSearchParams` (which internally calls `dbo.MergeResources`) so both writes execute in one stored-procedure-owned transaction. + +### 2. Deferred flush for transaction bundles +For transaction bundles, per-entry resource upserts call `dbo.MergeResources` only (no SearchParam table touch), avoiding the exclusive lock. `BundleHandler` accumulates pending statuses across all entries and flushes them in a single `dbo.MergeSearchParams` call at the end of the bundle, still within the outer transaction scope. + +```mermaid +graph TD; + A[SearchParameter Operation] -->|Single operation| B[Behavior queues pending status in request context]; + B --> C[SqlServerFhirDataStore calls MergeSearchParams]; + C --> D[MergeSearchParams: status + MergeResources in one transaction]; + A -->|Transaction bundle| E[Per-entry: Behavior queues status, UpsertAsync calls MergeResources only]; + E --> F[BundleHandler drains pending statuses after each entry]; + F --> G[After all entries: single MergeSearchParams flush within outer transaction]; + G --> H[Transaction commits atomically]; +``` + +### 3. Cache update removal from CRUD path +SearchParameter CRUD behaviors no longer perform direct in-memory cache mutations. All cache updates (`AddNewSearchParameters`, `DeleteSearchParameter`, `UpdateSearchParameterStatus`) are now solely owned by the `SearchParameterCacheRefreshBackgroundService`, which periodically polls the database via `GetAndApplySearchParameterUpdates`. This simplifies the CRUD path and ensures consistent cache convergence across distributed instances. + +### Scope +- **SQL Server**: Full atomic guarantees for create, update, and delete of SearchParameter resources, both single and in transaction bundles. +- **Cosmos DB**: Pending statuses are flushed after resource upsert (improved sequencing, not a single transactional unit). +- **Unchanged**: Non-SearchParameter CRUD, existing SearchParameter status lifecycle states, cache convergence model. + +## Status +Pending acceptance + +## Consequences +- **Positive Impacts:** + - Eliminates orphaned status/resource records from partial commits. + - Clarifies ownership: behaviors queue intent, data stores persist atomically, background service owns cache. + - Deferred-flush approach avoids lock contention introduced by composed writes in transaction bundles. + - Removing cache mutations from CRUD simplifies the request path and eliminates a class of cache-divergence bugs. + +- **Potential Drawbacks:** + - Increased complexity in request context, data store, and bundle handler coordination. + - SQL schema migration required (`MergeSearchParams` expanded to accept resource TVPs). + - Eventual consistency window: cache may lag behind database until the next background refresh cycle. + - Cosmos DB path remains best-effort sequencing rather than true atomic commit. + diff --git a/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Operations/BulkDelete/BulkDeleteProcessingJobTests.cs b/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Operations/BulkDelete/BulkDeleteProcessingJobTests.cs index 6c1f174e16..08e455c150 100644 --- a/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Operations/BulkDelete/BulkDeleteProcessingJobTests.cs +++ b/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Operations/BulkDelete/BulkDeleteProcessingJobTests.cs @@ -15,7 +15,9 @@ using Microsoft.Health.Fhir.Core.Features.Operations.BulkDelete; using Microsoft.Health.Fhir.Core.Features.Persistence; using Microsoft.Health.Fhir.Core.Features.Search; +using Microsoft.Health.Fhir.Core.Features.Search.Parameters; using Microsoft.Health.Fhir.Core.Messages.Delete; +using Microsoft.Health.Fhir.Core.Models; using Microsoft.Health.Fhir.Core.UnitTests.Extensions; using Microsoft.Health.Fhir.Tests.Common; using Microsoft.Health.JobManagement; @@ -24,6 +26,8 @@ using NSubstitute; using Xunit; +using FhirJobConflictException = global::Microsoft.Health.Fhir.Core.Features.Operations.JobConflictException; + namespace Microsoft.Health.Fhir.Core.UnitTests.Features.Operations.BulkDelete { [Trait(Traits.OwningTeam, OwningTeam.Fhir)] @@ -32,6 +36,7 @@ public class BulkDeleteProcessingJobTests { private IDeletionService _deleter; private BulkDeleteProcessingJob _processingJob; + private ISearchParameterOperations _searchParameterOperations; private ISearchService _searchService; private IQueueClient _queueClient; @@ -42,7 +47,8 @@ public BulkDeleteProcessingJobTests() .Returns(Task.FromResult(new SearchResult(5, new List>()))); _queueClient = Substitute.For(); _deleter = Substitute.For(); - _processingJob = new BulkDeleteProcessingJob(_deleter.CreateMockScopeFactory(), Substitute.For>(), Substitute.For(), _searchService.CreateMockScopeFactory(), _queueClient); + _searchParameterOperations = Substitute.For(); + _processingJob = new BulkDeleteProcessingJob(_deleter.CreateMockScopeFactory(), Substitute.For>(), Substitute.For(), _searchParameterOperations, _searchService.CreateMockScopeFactory(), _queueClient); } [Fact] @@ -103,5 +109,23 @@ public async Task GivenProcessingJob_WhenJobIsRunWithMultipleResourceTypes_ThenF var actualDefinition = JsonConvert.DeserializeObject(definitions[0]); Assert.Equal(2, actualDefinition.Type.SplitByOrSeparator().Count()); } + + [Fact] + public async Task GivenProcessingJobForSearchParameter_WhenReindexStartsBeforeExecution_ThenConflictIsThrown() + { + var definition = new BulkDeleteDefinition(JobType.BulkDeleteProcessing, DeleteOperation.HardDelete, KnownResourceTypes.SearchParameter, new List>(), new List(), "https:\\test.com", "https:\\test.com", "test"); + var jobInfo = new JobInfo + { + Id = 1, + Definition = JsonConvert.SerializeObject(definition), + }; + + _searchParameterOperations + .When(x => x.EnsureNoActiveReindexJobAsync(Arg.Any())) + .Do(_ => throw new FhirJobConflictException("reindex running")); + + await Assert.ThrowsAsync(() => _processingJob.ExecuteAsync(jobInfo, CancellationToken.None)); + await _deleter.DidNotReceiveWithAnyArgs().DeleteMultipleAsync(default, default, default); + } } } diff --git a/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Search/Registry/SearchParameterStatusManagerTests.cs b/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Search/Registry/SearchParameterStatusManagerTests.cs index 53688b3f01..8463ef776d 100644 --- a/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Search/Registry/SearchParameterStatusManagerTests.cs +++ b/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Search/Registry/SearchParameterStatusManagerTests.cs @@ -205,5 +205,54 @@ public async Task GivenASPStatusManager_WhenInitializingAndResolverThrowsExcepti Assert.False(list[2].IsSupported); Assert.False(list[2].IsPartiallySupported); } + + [Fact] + public async Task GivenASPStatusManager_WhenUpdatingStatus_ThenInMemoryCacheIsNotMutatedAndMediatorIsNotPublished() + { + // Arrange - Initialize so search parameters have known in-memory state + await _manager.EnsureInitializedAsync(CancellationToken.None); + + var list = _searchParameterDefinitionManager.GetSearchParameters("Account").ToList(); + + // Capture initial in-memory state for the Enabled parameter (index 0: ResourceId) + bool initialIsSearchable = list[0].IsSearchable; + bool initialIsSupported = list[0].IsSupported; + bool initialIsPartiallySupported = list[0].IsPartiallySupported; + SortParameterStatus initialSortStatus = list[0].SortStatus; + + // Clear any mediator calls from initialization + _mediator.ClearReceivedCalls(); + + // Act - Call UpdateSearchParameterStatusAsync to change status to Supported + await _manager.UpdateSearchParameterStatusAsync( + new[] { ResourceId }, + SearchParameterStatus.Supported, + CancellationToken.None); + + // Assert - DB write occurred + await _searchParameterStatusDataStore + .Received(1) + .UpsertStatuses( + Arg.Is>(statuses => + statuses.Count == 1 && + statuses[0].Uri.OriginalString == ResourceId && + statuses[0].Status == SearchParameterStatus.Supported), + Arg.Any()); + + // Assert - In-memory SearchParameterInfo was NOT modified + // Re-fetch from the definition manager to make the assertion intent explicit + var refreshedList = _searchParameterDefinitionManager.GetSearchParameters("Account").ToList(); + Assert.Equal(initialIsSearchable, refreshedList[0].IsSearchable); + Assert.Equal(initialIsSupported, refreshedList[0].IsSupported); + Assert.Equal(initialIsPartiallySupported, refreshedList[0].IsPartiallySupported); + Assert.Equal(initialSortStatus, refreshedList[0].SortStatus); + + // Assert - Mediator was NOT called (no SearchParametersUpdatedNotification) + await _mediator + .DidNotReceive() + .Publish( + Arg.Any(), + Arg.Any()); + } } } diff --git a/src/Microsoft.Health.Fhir.Core/Features/Definition/SearchParameterDefinitionBuilder.cs b/src/Microsoft.Health.Fhir.Core/Features/Definition/SearchParameterDefinitionBuilder.cs index 54766cdd7c..8b9054850b 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Definition/SearchParameterDefinitionBuilder.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Definition/SearchParameterDefinitionBuilder.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -63,7 +63,8 @@ internal static void Build( searchParameters, uriDictionary, modelInfoProvider, - isSystemDefined).ToLookup( + isSystemDefined, + logger).ToLookup( entry => entry.ResourceType, entry => entry.SearchParameter); @@ -121,7 +122,8 @@ private static SearchParameterInfo GetOrCreateSearchParameterInfo(SearchParamete IReadOnlyCollection searchParamCollection, ConcurrentDictionary uriDictionary, IModelInfoProvider modelInfoProvider, - bool isSystemDefined = false) + bool isSystemDefined, + ILogger logger) { var issues = new List(); var searchParameters = searchParamCollection.Select((x, entryIndex) => @@ -151,8 +153,27 @@ private static SearchParameterInfo GetOrCreateSearchParameterInfo(SearchParamete { SearchParameterInfo searchParameterInfo = GetOrCreateSearchParameterInfo(searchParameter, uriDictionary); - // Mark spec-defined search parameters as system-defined - searchParameterInfo.IsSystemDefined = isSystemDefined; + // Mark spec-defined search parameters as system-defined. + // Once marked, this should remain true across subsequent Build calls. + bool wasSystemDefined = searchParameterInfo.IsSystemDefined; + searchParameterInfo.IsSystemDefined |= isSystemDefined; + + if (!wasSystemDefined && searchParameterInfo.IsSystemDefined) + { + logger.LogDebug( + "SearchParameter IsSystemDefined enabled: Url={Url}, Code={Code}, BuildIsSystemDefined={BuildIsSystemDefined}", + searchParameterInfo.Url?.OriginalString, + searchParameterInfo.Code, + isSystemDefined); + } + else if (wasSystemDefined && !isSystemDefined) + { + logger.LogWarning( + "SearchParameter IsSystemDefined downgrade ignored: Url={Url}, Code={Code}, BuildIsSystemDefined={BuildIsSystemDefined}", + searchParameterInfo.Url?.OriginalString, + searchParameterInfo.Code, + isSystemDefined); + } if (searchParameterInfo.Code == "_profile" && searchParameterInfo.Type == SearchParamType.Reference) { @@ -174,6 +195,8 @@ private static SearchParameterInfo GetOrCreateSearchParameterInfo(SearchParamete EnsureNoIssues(); + SearchParameterInfo.ResourceTypeSearchParameter.IsSystemDefined = true; + var validatedSearchParameters = new List<(string ResourceType, SearchParameterInfo SearchParameter)> { // _type is currently missing from the search params definition bundle, so we inject it in here. diff --git a/src/Microsoft.Health.Fhir.Core/Features/Operations/BulkDelete/BulkDeleteProcessingJob.cs b/src/Microsoft.Health.Fhir.Core/Features/Operations/BulkDelete/BulkDeleteProcessingJob.cs index 03597a2ce1..a835662801 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Operations/BulkDelete/BulkDeleteProcessingJob.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Operations/BulkDelete/BulkDeleteProcessingJob.cs @@ -19,7 +19,9 @@ using Microsoft.Health.Fhir.Core.Features.Operations.BulkDelete.Messages; using Microsoft.Health.Fhir.Core.Features.Persistence; using Microsoft.Health.Fhir.Core.Features.Search; +using Microsoft.Health.Fhir.Core.Features.Search.Parameters; using Microsoft.Health.Fhir.Core.Messages.Delete; +using Microsoft.Health.Fhir.Core.Models; using Microsoft.Health.JobManagement; using Newtonsoft.Json; @@ -31,6 +33,7 @@ public class BulkDeleteProcessingJob : IJob private readonly Func> _deleterFactory; private readonly RequestContextAccessor _contextAccessor; private readonly IMediator _mediator; + private readonly ISearchParameterOperations _searchParameterOperations; private readonly Func> _searchService; private readonly IQueueClient _queueClient; @@ -38,12 +41,14 @@ public BulkDeleteProcessingJob( Func> deleterFactory, RequestContextAccessor contextAccessor, IMediator mediator, + ISearchParameterOperations searchParameterOperations, Func> searchService, IQueueClient queueClient) { _deleterFactory = EnsureArg.IsNotNull(deleterFactory, nameof(deleterFactory)); _contextAccessor = EnsureArg.IsNotNull(contextAccessor, nameof(contextAccessor)); _mediator = EnsureArg.IsNotNull(mediator, nameof(mediator)); + _searchParameterOperations = EnsureArg.IsNotNull(searchParameterOperations, nameof(searchParameterOperations)); _searchService = EnsureArg.IsNotNull(searchService, nameof(searchService)); _queueClient = EnsureArg.IsNotNull(queueClient, nameof(queueClient)); } @@ -78,6 +83,13 @@ public async Task ExecuteAsync(JobInfo jobInfo, CancellationToken cancel Exception exception = null; List types = definition.Type.SplitByOrSeparator().ToList(); + if (types.Count > 0 + && string.Equals(types[0], KnownResourceTypes.SearchParameter, StringComparison.OrdinalIgnoreCase) + && !(definition.ExcludedResourceTypes?.Any(x => string.Equals(x, KnownResourceTypes.SearchParameter, StringComparison.OrdinalIgnoreCase)) ?? false)) + { + await _searchParameterOperations.EnsureNoActiveReindexJobAsync(cancellationToken); + } + try { resourcesDeleted = await deleter.Value.DeleteMultipleAsync( diff --git a/src/Microsoft.Health.Fhir.Core/Features/Operations/Reindex/CreateReindexRequestHandler.cs b/src/Microsoft.Health.Fhir.Core/Features/Operations/Reindex/CreateReindexRequestHandler.cs index 304725646b..706adc882a 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Operations/Reindex/CreateReindexRequestHandler.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Operations/Reindex/CreateReindexRequestHandler.cs @@ -69,11 +69,6 @@ public async Task Handle(CreateReindexRequest request, Ca return new CreateReindexResponse(existingJob); } - // We need to pull in latest search parameter updates from the data store before creating a reindex job. - // There could be a potential delay of before - // search parameter updates on one instance propagates to other instances. - await _searchParameterOperations.GetAndApplySearchParameterUpdates(cancellationToken); - // What this handles is the scenario where a user is effectively forcing a reindex to run by passing // in a parameter of targetSearchParameterTypes. From those we can identify the base resource types. var searchParameterResourceTypes = new HashSet(); diff --git a/src/Microsoft.Health.Fhir.Core/Features/Operations/Reindex/ReindexOrchestratorJob.cs b/src/Microsoft.Health.Fhir.Core/Features/Operations/Reindex/ReindexOrchestratorJob.cs index eb80deebdf..7162064d78 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Operations/Reindex/ReindexOrchestratorJob.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Operations/Reindex/ReindexOrchestratorJob.cs @@ -43,8 +43,8 @@ public sealed class ReindexOrchestratorJob : IJob private readonly IModelInfoProvider _modelInfoProvider; private readonly ISearchParameterOperations _searchParameterOperations; private readonly bool _isSurrogateIdRangingSupported; - private readonly CoreFeatureConfiguration _coreFeatureConfiguration; private readonly OperationsConfiguration _operationsConfiguration; + private readonly int _searchParameterCacheRefreshIntervalSeconds; private CancellationToken _cancellationToken; private IQueueClient _queueClient; @@ -77,6 +77,12 @@ public sealed class ReindexOrchestratorJob : IJob /// private static readonly AsyncPolicy _searchParameterStatusRetries = Policy.WrapAsync(_requestRateRetries, _timeoutRetries); + /// + /// Retry policy for reindex query execution. + /// Handles transient SQL timeouts and Cosmos DB request rate limiting. + /// + private static readonly AsyncPolicy _reindexQueryRetries = Policy.WrapAsync(_requestRateRetries, _timeoutRetries); + private HashSet _processedJobIds = new HashSet(); private HashSet _processedSearchParameters = new HashSet(); private List _jobsToProcess; @@ -113,8 +119,8 @@ public ReindexOrchestratorJob( _modelInfoProvider = modelInfoProvider; _searchParameterStatusManager = searchParameterStatusManager; _searchParameterOperations = searchParameterOperations; - _coreFeatureConfiguration = coreFeatureConfiguration.Value; _operationsConfiguration = operationsConfiguration.Value; + _searchParameterCacheRefreshIntervalSeconds = Math.Max(1, coreFeatureConfiguration.Value.SearchParameterCacheRefreshIntervalSeconds); // Determine support for surrogate ID ranging once // This is to ensure Gen1 Reindex still works as expected but we still maintain perf on job inseration to SQL @@ -183,18 +189,40 @@ public async Task ExecuteAsync(JobInfo jobInfo, CancellationToken cancel return JsonConvert.SerializeObject(_currentResult); } - private async Task WaitForRefresh() - { - await Task.Delay(_operationsConfiguration.Reindex.CacheRefreshWaitMultiplier * _coreFeatureConfiguration.SearchParameterCacheRefreshIntervalSeconds * 1000, _cancellationToken); - } - private async Task RefreshSearchParameterCache(bool isReindexStart) { - // before starting anything wait for natural cache refresh. this will also make sure that all processing pods have latest search param definitions. + // Wait for the background cache refresh service to complete N successful refresh cycles. + // This ensures all instances (including processing pods) have the latest search parameter definitions. var suffix = isReindexStart ? "Start" : "End"; _logger.LogJobInformation(_jobInfo, $"Reindex orchestrator job started cache refresh at the {suffix}."); await TryLogEvent($"ReindexOrchestratorJob={_jobInfo.Id}.ExecuteAsync.{suffix}", "Warn", "Started", null, _cancellationToken); - await WaitForRefresh(); // wait for M * cache refresh intervals + + // First, wait for the local background refresh service to complete N cycles. + // This ensures _searchParamLastUpdated is up-to-date on THIS instance before + // we use it as the convergence target for cross-instance checks. + await _searchParameterOperations.WaitForRefreshCyclesAsync(_operationsConfiguration.Reindex.CacheRefreshWaitMultiplier, _cancellationToken); + + if (_isSurrogateIdRangingSupported) + { + // SQL Server: After local refresh, verify ALL instances have converged to + // the same SearchParamLastUpdated via the EventLog table. This prevents the + // orchestrator from creating reindex ranges while other instances still have + // stale search parameter caches and would write resources with wrong hashes. + // Use the same lookback as active host detection so we do not miss qualifying + // refresh events that occurred shortly before this instance entered the wait. + var activeHostsSince = DateTime.UtcNow.AddSeconds(-20 * _searchParameterCacheRefreshIntervalSeconds); + var syncStartDate = activeHostsSince; + await _searchParameterOperations.WaitForAllInstancesCacheConsistencyAsync(syncStartDate, activeHostsSince, _cancellationToken); + } + else + { + // Cosmos DB: There is no EventLog-based convergence tracking, so wait a fixed + // delay to allow all instances to refresh their search parameter caches from + // the shared Cosmos container. + var delayMs = _operationsConfiguration.Reindex.CacheRefreshWaitMultiplier * _searchParameterCacheRefreshIntervalSeconds * 1000; + _logger.LogJobInformation(_jobInfo, "Cosmos DB detected — waiting {DelayMs}ms for cache propagation across instances.", delayMs); + await Task.Delay(delayMs, _cancellationToken); + } // Update the reindex job record with the latest hash map var currentDate = _searchParameterOperations.SearchParamLastUpdated.HasValue ? _searchParameterOperations.SearchParamLastUpdated.Value : DateTimeOffset.MinValue; @@ -727,7 +755,8 @@ private async Task GetResourceCountForQueryAsync(ReindexJobQuerySt { try { - return await searchService.Value.SearchForReindexAsync(queryParametersList, searchParameterHash, countOnly: countOnly, cancellationToken, true); + return await _reindexQueryRetries.ExecuteAsync( + async () => await searchService.Value.SearchForReindexAsync(queryParametersList, searchParameterHash, countOnly: countOnly, cancellationToken, true)); } catch (Exception ex) { @@ -756,6 +785,8 @@ private async Task UpdateSearchParameterStatus(List completedJobs, List // Check if all the resource types which are base types of the search parameter // were reindexed by this job. If so, then we should mark the search parameters // as fully reindexed + var disabledParamUris = new List(); + var deletedParamUris = new List(); var fullyIndexedParamUris = new List(); var searchParamStatusCollection = await _searchParameterStatusManager.GetAllSearchParameterStatus(cancellationToken); @@ -772,16 +803,10 @@ private async Task UpdateSearchParameterStatus(List completedJobs, List switch (spStatus) { case SearchParameterStatus.PendingDisable: - _logger.LogJobInformation(_jobInfo, "Reindex job updating the status of the fully indexed search parameter, parameter: '{ParamUri}' to Disabled.", searchParameterUrl); - await _searchParameterStatusRetries.ExecuteAsync( - async () => await _searchParameterStatusManager.UpdateSearchParameterStatusAsync(new List() { searchParameterUrl }, SearchParameterStatus.Disabled, cancellationToken)); - _processedSearchParameters.Add(searchParameterUrl); + disabledParamUris.Add(searchParameterUrl); break; case SearchParameterStatus.PendingDelete: - _logger.LogJobInformation(_jobInfo, "Reindex job updating the status of the fully indexed search parameter, parameter: '{ParamUri}' to Deleted.", searchParameterUrl); - await _searchParameterStatusRetries.ExecuteAsync( - async () => await _searchParameterStatusManager.UpdateSearchParameterStatusAsync(new List() { searchParameterUrl }, SearchParameterStatus.Deleted, cancellationToken)); - _processedSearchParameters.Add(searchParameterUrl); + deletedParamUris.Add(searchParameterUrl); break; case SearchParameterStatus.Supported: case SearchParameterStatus.Enabled: @@ -790,6 +815,22 @@ await _searchParameterStatusRetries.ExecuteAsync( } } + if (disabledParamUris.Count > 0) + { + _logger.LogJobInformation(_jobInfo, "Reindex job updating the status of the fully indexed search parameter, parameters: '{ParamUris}' to Disabled.", string.Join("', '", disabledParamUris)); + await _searchParameterStatusRetries.ExecuteAsync( + async () => await _searchParameterStatusManager.UpdateSearchParameterStatusAsync(disabledParamUris, SearchParameterStatus.Disabled, cancellationToken)); + _processedSearchParameters.UnionWith(disabledParamUris); + } + + if (deletedParamUris.Count > 0) + { + _logger.LogJobInformation(_jobInfo, "Reindex job updating the status of the fully indexed search parameter, parameters: '{ParamUris}' to Deleted.", string.Join("', '", deletedParamUris)); + await _searchParameterStatusRetries.ExecuteAsync( + async () => await _searchParameterStatusManager.UpdateSearchParameterStatusAsync(deletedParamUris, SearchParameterStatus.Deleted, cancellationToken)); + _processedSearchParameters.UnionWith(deletedParamUris); + } + if (fullyIndexedParamUris.Count > 0) { _logger.LogJobInformation(_jobInfo, "Reindex job updating the status of the fully indexed search parameter, parameters: '{ParamUris} to Enabled.'", string.Join("', '", fullyIndexedParamUris)); diff --git a/src/Microsoft.Health.Fhir.Core/Features/Operations/Reindex/ReindexProcessingJob.cs b/src/Microsoft.Health.Fhir.Core/Features/Operations/Reindex/ReindexProcessingJob.cs index d6faf25957..9febcb89b1 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Operations/Reindex/ReindexProcessingJob.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Operations/Reindex/ReindexProcessingJob.cs @@ -132,8 +132,9 @@ private async Task CheckDiscrepancies(CancellationToken cancellationToken) var msg = $"ResourceType={resourceType} SearchParameterHash: Requested={requestedSearchParameterHash} {(isBad ? "!=" : "=")} Current={searchParameterHash}"; if (isBad) { - _logger.LogJobWarning(_jobInfo, msg); + _logger.LogJobError(_jobInfo, msg); await TryLogEvent($"ReindexProcessingJob={_jobInfo.Id}.GetResourcesToReindexAsync", "Error", msg, null, cancellationToken); // elevate in SQL to log w/o extra settings + throw new ReindexJobException(msg); } else { @@ -150,11 +151,11 @@ private async Task CheckDiscrepancies(CancellationToken cancellationToken) isBad = _reindexProcessingJobDefinition.SearchParamLastUpdated > currentDate; msg = $"SearchParamLastUpdated: Requested={requested} {(isBad ? ">" : "<=")} Current={current}"; //// If timestamp from definition (requested by orchestrator) is more recent, then cache on processing VM is stale. - //// Cannot just refresh here because we might be missing resources updated via API. if (isBad) { - _logger.LogJobWarning(_jobInfo, msg); + _logger.LogJobError(_jobInfo, msg); await TryLogEvent($"ReindexProcessingJob={_jobInfo.Id}.ExecuteAsync", "Error", msg, null, cancellationToken); // elevate in SQL to log w/o extra settings + throw new ReindexJobException(msg); } else // normal { diff --git a/src/Microsoft.Health.Fhir.Core/Features/Persistence/ResourceWrapperOperation.cs b/src/Microsoft.Health.Fhir.Core/Features/Persistence/ResourceWrapperOperation.cs index f2622c77dd..eb361489f8 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Persistence/ResourceWrapperOperation.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Persistence/ResourceWrapperOperation.cs @@ -3,7 +3,9 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- +using System.Collections.Generic; using EnsureThat; +using Microsoft.Health.Fhir.Core.Features.Search.Registry; using Microsoft.Health.Fhir.Core.Models; namespace Microsoft.Health.Fhir.Core.Features.Persistence @@ -46,6 +48,8 @@ public ResourceWrapperOperation( public BundleResourceContext BundleResourceContext { get; } + public IReadOnlyList PendingSearchParameterStatuses { get; internal set; } + #pragma warning disable CA1024 // Use properties where appropriate public DataStoreOperationIdentifier GetIdentifier() { diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/CreateOrUpdateSearchParameterBehavior.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/CreateOrUpdateSearchParameterBehavior.cs index e14819b408..981437305e 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/CreateOrUpdateSearchParameterBehavior.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/CreateOrUpdateSearchParameterBehavior.cs @@ -4,12 +4,19 @@ // ------------------------------------------------------------------------------------------------- using System; +using System.Collections.Generic; +using System.Linq; using System.Threading; using System.Threading.Tasks; using EnsureThat; +using Hl7.Fhir.ElementModel; using MediatR; +using Microsoft.Health.Core.Features.Context; using Microsoft.Health.Fhir.Core.Exceptions; +using Microsoft.Health.Fhir.Core.Extensions; +using Microsoft.Health.Fhir.Core.Features.Context; using Microsoft.Health.Fhir.Core.Features.Persistence; +using Microsoft.Health.Fhir.Core.Features.Search.Registry; using Microsoft.Health.Fhir.Core.Messages.Create; using Microsoft.Health.Fhir.Core.Messages.Upsert; using Microsoft.Health.Fhir.Core.Models; @@ -19,25 +26,41 @@ namespace Microsoft.Health.Fhir.Core.Features.Search.Parameters public class CreateOrUpdateSearchParameterBehavior : IPipelineBehavior, IPipelineBehavior { - private ISearchParameterOperations _searchParameterOperations; - private IFhirDataStore _fhirDataStore; + private readonly ISearchParameterOperations _searchParameterOperations; + private readonly IFhirDataStore _fhirDataStore; + private readonly ISearchParameterStatusManager _searchParameterStatusManager; + private readonly RequestContextAccessor _requestContextAccessor; + private readonly IModelInfoProvider _modelInfoProvider; - public CreateOrUpdateSearchParameterBehavior(ISearchParameterOperations searchParameterOperations, IFhirDataStore fhirDataStore) + public CreateOrUpdateSearchParameterBehavior( + ISearchParameterOperations searchParameterOperations, + IFhirDataStore fhirDataStore, + ISearchParameterStatusManager searchParameterStatusManager, + RequestContextAccessor requestContextAccessor, + IModelInfoProvider modelInfoProvider) { EnsureArg.IsNotNull(searchParameterOperations, nameof(searchParameterOperations)); EnsureArg.IsNotNull(fhirDataStore, nameof(fhirDataStore)); + EnsureArg.IsNotNull(searchParameterStatusManager, nameof(searchParameterStatusManager)); + EnsureArg.IsNotNull(requestContextAccessor, nameof(requestContextAccessor)); + EnsureArg.IsNotNull(modelInfoProvider, nameof(modelInfoProvider)); _searchParameterOperations = searchParameterOperations; _fhirDataStore = fhirDataStore; + _searchParameterStatusManager = searchParameterStatusManager; + _requestContextAccessor = requestContextAccessor; + _modelInfoProvider = modelInfoProvider; } public async Task Handle(CreateResourceRequest request, RequestHandlerDelegate next, CancellationToken cancellationToken) { if (request.Resource.InstanceType.Equals(KnownResourceTypes.SearchParameter, StringComparison.Ordinal)) { - // Before committing the SearchParameter resource to the data store, add it to the SearchParameterDefinitionManager - // and parse the fhirPath, as well as validate the parameter type - await _searchParameterOperations.AddSearchParameterAsync(request.Resource.Instance, cancellationToken); + // Before committing the SearchParameter resource to the data store, validate the parameter type + await _searchParameterOperations.ValidateSearchParameterAsync(request.Resource.Instance, cancellationToken); + + var url = request.Resource.Instance.GetStringScalar("url"); + await QueueStatusAsync(url, SearchParameterStatus.Supported, cancellationToken); } // Allow the resource to be updated with the normal handler @@ -67,19 +90,70 @@ public async Task Handle(UpsertResourceRequest request, if (prevSearchParamResource != null && prevSearchParamResource.IsDeleted == false) { - // Update the SearchParameterDefinitionManager with the new SearchParameter in order to validate any changes - // to the fhirpath or the datatype - await _searchParameterOperations.UpdateSearchParameterAsync(request.Resource.Instance, prevSearchParamResource.RawResource, cancellationToken); + // Validate any changes to the fhirpath or the datatype + await _searchParameterOperations.ValidateSearchParameterAsync(request.Resource.Instance, cancellationToken); + + var previousUrl = _modelInfoProvider.ToTypedElement(prevSearchParamResource.RawResource).GetStringScalar("url"); + var newUrl = request.Resource.Instance.GetStringScalar("url"); + + if (!string.IsNullOrWhiteSpace(previousUrl) && !previousUrl.Equals(newUrl, StringComparison.Ordinal)) + { + await QueueStatusAsync(previousUrl, SearchParameterStatus.Deleted, cancellationToken); + } + + await QueueStatusAsync(newUrl, SearchParameterStatus.Supported, cancellationToken); } else { // No previous version exists or it was deleted, so add it as a new SearchParameter - await _searchParameterOperations.AddSearchParameterAsync(request.Resource.Instance, cancellationToken); + await _searchParameterOperations.ValidateSearchParameterAsync(request.Resource.Instance, cancellationToken); + + var url = request.Resource.Instance.GetStringScalar("url"); + await QueueStatusAsync(url, SearchParameterStatus.Supported, cancellationToken); } } // Now allow the resource to updated per the normal behavior return await next(cancellationToken); } + + private async Task QueueStatusAsync(string url, SearchParameterStatus status, CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(url)) + { + return; + } + + var context = _requestContextAccessor.RequestContext; + if (context == null) + { + return; + } + + if (!context.Properties.TryGetValue(SearchParameterRequestContextPropertyNames.PendingStatusUpdates, out var value) || + value is not List pendingStatuses) + { + pendingStatuses = new List(); + context.Properties[SearchParameterRequestContextPropertyNames.PendingStatusUpdates] = pendingStatuses; + } + + var currentStatuses = await _searchParameterStatusManager.GetAllSearchParameterStatus(cancellationToken); + var existing = currentStatuses.FirstOrDefault(s => string.Equals(s.Uri?.OriginalString, url, StringComparison.Ordinal)); + + var update = new ResourceSearchParameterStatus + { + Uri = new Uri(url), + Status = status, + LastUpdated = existing?.LastUpdated ?? DateTimeOffset.UtcNow, + IsPartiallySupported = existing?.IsPartiallySupported ?? false, + SortStatus = existing?.SortStatus ?? SortParameterStatus.Disabled, + }; + + lock (pendingStatuses) + { + pendingStatuses.RemoveAll(s => string.Equals(s.Uri?.OriginalString, url, StringComparison.Ordinal)); + pendingStatuses.Add(update); + } + } } } diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/DeleteSearchParameterBehavior.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/DeleteSearchParameterBehavior.cs index f3b6c303bd..7be8c387dc 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/DeleteSearchParameterBehavior.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/DeleteSearchParameterBehavior.cs @@ -4,16 +4,20 @@ // ------------------------------------------------------------------------------------------------- using System; +using System.Collections.Generic; using System.Linq; using System.Threading; using System.Threading.Tasks; using EnsureThat; using Hl7.Fhir.ElementModel; using MediatR; +using Microsoft.Health.Core.Features.Context; using Microsoft.Health.Fhir.Core.Exceptions; using Microsoft.Health.Fhir.Core.Extensions; +using Microsoft.Health.Fhir.Core.Features.Context; using Microsoft.Health.Fhir.Core.Features.Definition; using Microsoft.Health.Fhir.Core.Features.Persistence; +using Microsoft.Health.Fhir.Core.Features.Search.Registry; using Microsoft.Health.Fhir.Core.Features.Validation; using Microsoft.Health.Fhir.Core.Messages.Delete; using Microsoft.Health.Fhir.Core.Models; @@ -23,22 +27,34 @@ namespace Microsoft.Health.Fhir.Core.Features.Search.Parameters public class DeleteSearchParameterBehavior : IPipelineBehavior where TDeleteResourceRequest : DeleteResourceRequest, IRequest { - private ISearchParameterOperations _searchParameterOperations; - private IFhirDataStore _fhirDataStore; - private ISearchParameterDefinitionManager _searchParameterDefinitionManager; + private readonly ISearchParameterOperations _searchParameterOperations; + private readonly IFhirDataStore _fhirDataStore; + private readonly ISearchParameterDefinitionManager _searchParameterDefinitionManager; + private readonly ISearchParameterStatusManager _searchParameterStatusManager; + private readonly RequestContextAccessor _requestContextAccessor; + private readonly IModelInfoProvider _modelInfoProvider; public DeleteSearchParameterBehavior( ISearchParameterOperations searchParameterOperations, IFhirDataStore fhirDataStore, - ISearchParameterDefinitionManager searchParameterDefinitionManager) + ISearchParameterDefinitionManager searchParameterDefinitionManager, + ISearchParameterStatusManager searchParameterStatusManager, + RequestContextAccessor requestContextAccessor, + IModelInfoProvider modelInfoProvider) { EnsureArg.IsNotNull(searchParameterOperations, nameof(searchParameterOperations)); EnsureArg.IsNotNull(fhirDataStore, nameof(fhirDataStore)); EnsureArg.IsNotNull(searchParameterDefinitionManager, nameof(searchParameterDefinitionManager)); + EnsureArg.IsNotNull(searchParameterStatusManager, nameof(searchParameterStatusManager)); + EnsureArg.IsNotNull(requestContextAccessor, nameof(requestContextAccessor)); + EnsureArg.IsNotNull(modelInfoProvider, nameof(modelInfoProvider)); _searchParameterOperations = searchParameterOperations; _fhirDataStore = fhirDataStore; _searchParameterDefinitionManager = searchParameterDefinitionManager; + _searchParameterStatusManager = searchParameterStatusManager; + _requestContextAccessor = requestContextAccessor; + _modelInfoProvider = modelInfoProvider; } public async Task Handle(TDeleteResourceRequest request, RequestHandlerDelegate next, CancellationToken cancellationToken) @@ -70,13 +86,52 @@ public async Task Handle(TDeleteResourceRequest request // If the search parameter exists and is not already deleted, delete it if (!searchParamResource.IsDeleted) { - // First update the in-memory SearchParameterDefinitionManager, and remove the status metadata from the data store - // then remove the SearchParameter resource from the data store - await _searchParameterOperations.DeleteSearchParameterAsync(searchParamResource.RawResource, cancellationToken); + var typed = _modelInfoProvider.ToTypedElement(searchParamResource.RawResource); + var url = typed.GetStringScalar("url"); + await QueuePendingDeleteStatusAsync(url, cancellationToken); } } return await next(cancellationToken); } + + private async Task QueuePendingDeleteStatusAsync(string url, CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(url)) + { + return; + } + + var context = _requestContextAccessor.RequestContext; + if (context == null) + { + return; + } + + if (!context.Properties.TryGetValue(SearchParameterRequestContextPropertyNames.PendingStatusUpdates, out var value) || + value is not List pendingStatuses) + { + pendingStatuses = new List(); + context.Properties[SearchParameterRequestContextPropertyNames.PendingStatusUpdates] = pendingStatuses; + } + + var currentStatuses = await _searchParameterStatusManager.GetAllSearchParameterStatus(cancellationToken); + var existing = currentStatuses.FirstOrDefault(s => string.Equals(s.Uri?.OriginalString, url, StringComparison.Ordinal)); + + var update = new ResourceSearchParameterStatus + { + Uri = new Uri(url), + Status = SearchParameterStatus.PendingDelete, + LastUpdated = existing?.LastUpdated ?? DateTimeOffset.UtcNow, + IsPartiallySupported = existing?.IsPartiallySupported ?? false, + SortStatus = existing?.SortStatus ?? SortParameterStatus.Disabled, + }; + + lock (pendingStatuses) + { + pendingStatuses.RemoveAll(s => string.Equals(s.Uri?.OriginalString, url, StringComparison.Ordinal)); + pendingStatuses.Add(update); + } + } } } diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/ISearchParameterOperations.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/ISearchParameterOperations.cs index d2ce3f275a..7978224e24 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/ISearchParameterOperations.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/ISearchParameterOperations.cs @@ -4,10 +4,12 @@ // ------------------------------------------------------------------------------------------------- using System; +using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using Hl7.Fhir.ElementModel; using Microsoft.Health.Fhir.Core.Features.Persistence; +using Microsoft.Health.Fhir.Core.Features.Search.Registry; namespace Microsoft.Health.Fhir.Core.Features.Search.Parameters { @@ -15,11 +17,13 @@ public interface ISearchParameterOperations { DateTimeOffset? SearchParamLastUpdated { get; } - Task AddSearchParameterAsync(ITypedElement searchParam, CancellationToken cancellationToken); - Task DeleteSearchParameterAsync(RawResource searchParamResource, CancellationToken cancellationToken, bool ignoreSearchParameterNotSupportedException = false); - Task UpdateSearchParameterAsync(ITypedElement searchParam, RawResource previousSearchParam, CancellationToken cancellationToken); + Task ValidateSearchParameterAsync(ITypedElement searchParam, CancellationToken cancellationToken); + + Task UpdateSearchParameterStatusAsync(IReadOnlyCollection searchParameterUris, SearchParameterStatus status, CancellationToken cancellationToken, bool ignoreSearchParameterNotSupportedException = false); + + Task EnsureNoActiveReindexJobAsync(CancellationToken cancellationToken); /// /// This method should be called periodically to get any updates to SearchParameters @@ -32,5 +36,25 @@ public interface ISearchParameterOperations Task GetAndApplySearchParameterUpdates(CancellationToken cancellationToken, bool forceFullRefresh = false); string GetSearchParameterHash(string resourceType); + + /// + /// Waits for the specified number of successful cache refresh cycles to complete. + /// Each cycle corresponds to a successful execution of the background cache refresh service. + /// + /// The number of successful refresh cycles to wait for. If zero or negative, returns immediately. + /// Cancellation token. + /// A task that completes when the requested number of refresh cycles have occurred. + Task WaitForRefreshCyclesAsync(int cycleCount, CancellationToken cancellationToken); + + /// + /// Waits until all active server instances have converged their search parameter caches + /// to the current instance's SearchParamLastUpdated timestamp. For SQL, this verifies via + /// the EventLog table. For Cosmos/File-based, this returns immediately. + /// + /// Only cache refresh sync records on or after this time are considered for convergence. + /// Only active-host evidence on or after this time is considered. + /// Cancellation token. + /// A task that completes when all instances have consistent caches. + Task WaitForAllInstancesCacheConsistencyAsync(DateTime syncStartDate, DateTime activeHostsSince, CancellationToken cancellationToken); } } diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/SearchParameterOperations.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/SearchParameterOperations.cs index caf88919d2..f8e9eb462b 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/SearchParameterOperations.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/SearchParameterOperations.cs @@ -14,9 +14,12 @@ using Microsoft.Health.Extensions.DependencyInjection; using Microsoft.Health.Fhir.Core.Exceptions; using Microsoft.Health.Fhir.Core.Extensions; +using Microsoft.Health.Fhir.Core.Features; using Microsoft.Health.Fhir.Core.Features.Definition; using Microsoft.Health.Fhir.Core.Features.Definition.BundleWrappers; +using Microsoft.Health.Fhir.Core.Features.Operations; using Microsoft.Health.Fhir.Core.Features.Persistence; +using Microsoft.Health.Fhir.Core.Features.Search; using Microsoft.Health.Fhir.Core.Features.Search.Registry; using Microsoft.Health.Fhir.Core.Models; @@ -29,9 +32,12 @@ public class SearchParameterOperations : ISearchParameterOperations private readonly IModelInfoProvider _modelInfoProvider; private readonly ISearchParameterSupportResolver _searchParameterSupportResolver; private readonly IDataStoreSearchParameterValidator _dataStoreSearchParameterValidator; + private readonly Func> _fhirOperationDataStoreFactory; private readonly Func> _searchServiceFactory; private readonly ILogger _logger; private DateTimeOffset? _searchParamLastUpdated; + private int _activeConsistencyWaiters; + private volatile TaskCompletionSource _refreshSignal = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); public SearchParameterOperations( SearchParameterStatusManager searchParameterStatusManager, @@ -39,6 +45,7 @@ public SearchParameterOperations( IModelInfoProvider modelInfoProvider, ISearchParameterSupportResolver searchParameterSupportResolver, IDataStoreSearchParameterValidator dataStoreSearchParameterValidator, + Func> fhirOperationDataStoreFactory, Func> searchServiceFactory, ILogger logger) { @@ -47,6 +54,7 @@ public SearchParameterOperations( EnsureArg.IsNotNull(modelInfoProvider, nameof(modelInfoProvider)); EnsureArg.IsNotNull(searchParameterSupportResolver, nameof(searchParameterSupportResolver)); EnsureArg.IsNotNull(dataStoreSearchParameterValidator, nameof(dataStoreSearchParameterValidator)); + EnsureArg.IsNotNull(fhirOperationDataStoreFactory, nameof(fhirOperationDataStoreFactory)); EnsureArg.IsNotNull(searchServiceFactory, nameof(searchServiceFactory)); EnsureArg.IsNotNull(logger, nameof(logger)); @@ -55,6 +63,7 @@ public SearchParameterOperations( _modelInfoProvider = modelInfoProvider; _searchParameterSupportResolver = searchParameterSupportResolver; _dataStoreSearchParameterValidator = dataStoreSearchParameterValidator; + _fhirOperationDataStoreFactory = fhirOperationDataStoreFactory; _searchServiceFactory = searchServiceFactory; _logger = logger; } @@ -75,7 +84,147 @@ public string GetSearchParameterHash(string resourceType) } } - public async Task AddSearchParameterAsync(ITypedElement searchParam, CancellationToken cancellationToken) + /// + public async Task WaitForRefreshCyclesAsync(int cycleCount, CancellationToken cancellationToken) + { + if (cycleCount <= 0) + { + return; + } + + // Safety net: if the background service is not publishing notifications (e.g. crashed), + // fail fast rather than blocking indefinitely. Under normal conditions with a 20-second + // refresh interval, even 3 cycles would complete in ~60 seconds. + using var timeoutCts = new CancellationTokenSource(TimeSpan.FromMinutes(5)); + using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeoutCts.Token); + + for (int i = 0; i < cycleCount; i++) + { + linkedCts.Token.ThrowIfCancellationRequested(); + + // Capture the current signal before awaiting + var currentSignal = _refreshSignal; + using var registration = linkedCts.Token.Register(() => currentSignal.TrySetCanceled()); + + try + { + await currentSignal.Task; + } + catch (TaskCanceledException) when (timeoutCts.IsCancellationRequested && !cancellationToken.IsCancellationRequested) + { + throw new TimeoutException( + $"SearchParameterCacheRefreshBackgroundService did not complete {cycleCount} refresh cycle(s) within 5 minutes. The server may be in an unhealthy state."); + } + } + } + + /// + public async Task WaitForAllInstancesCacheConsistencyAsync(DateTime syncStartDate, DateTime activeHostsSince, CancellationToken cancellationToken) + { + if (!_searchParamLastUpdated.HasValue) + { + _logger.LogInformation("SearchParamLastUpdated is null — skipping cross-instance cache consistency check."); + return; + } + + var targetTimestamp = _searchParamLastUpdated.Value.ToString("yyyy-MM-dd HH:mm:ss.fffffff"); + var waitStart = DateTime.UtcNow; + + Interlocked.Increment(ref _activeConsistencyWaiters); + + try + { + await TryLogConsistencyWaitEventAsync( + "Warn", + $"Target={targetTimestamp} PollIntervalSeconds=30 TimeoutMinutes=10 SyncStartDate={syncStartDate:O} ActiveHostsSince={activeHostsSince:O}", + null, + cancellationToken); + + // Poll with a timeout — same 10-minute safety net as WaitForRefreshCyclesAsync + using var timeoutCts = new CancellationTokenSource(TimeSpan.FromMinutes(10)); + using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeoutCts.Token); + + while (!linkedCts.Token.IsCancellationRequested) + { + var result = await _searchParameterStatusManager.CheckCacheConsistencyAsync(targetTimestamp, syncStartDate, activeHostsSince, linkedCts.Token); + + if (result.IsConsistent) + { + var elapsedMilliseconds = (long)(DateTime.UtcNow - waitStart).TotalMilliseconds; + + _logger.LogInformation( + "All {TotalActiveHosts} active host(s) have converged to SearchParamLastUpdated={Target}.", + result.TotalActiveHosts, + targetTimestamp); + + await TryLogConsistencyWaitEventAsync( + "Warn", + $"Target={targetTimestamp} TotalActiveHosts={result.TotalActiveHosts} ConvergedHosts={result.ConvergedHosts} ElapsedMs={elapsedMilliseconds}", + waitStart, + cancellationToken); + + return; + } + + _logger.LogInformation( + "Cache consistency check: {ConvergedHosts}/{TotalActiveHosts} hosts converged to SearchParamLastUpdated={Target}. Waiting...", + result.ConvergedHosts, + result.TotalActiveHosts, + targetTimestamp); + + try + { + await Task.Delay(TimeSpan.FromSeconds(30), linkedCts.Token); + } + catch (OperationCanceledException) when (timeoutCts.IsCancellationRequested && !cancellationToken.IsCancellationRequested) + { + var elapsedMilliseconds = (long)(DateTime.UtcNow - waitStart).TotalMilliseconds; + var timeoutMessage = $"Target={targetTimestamp} TotalActiveHosts={result.TotalActiveHosts} ConvergedHosts={result.ConvergedHosts} ElapsedMs={elapsedMilliseconds}"; + + await TryLogConsistencyWaitEventAsync( + "Error", + timeoutMessage, + waitStart, + CancellationToken.None); + + throw new TimeoutException( + $"Not all instances converged to SearchParamLastUpdated={targetTimestamp} within 10 minutes. The server may be in an unhealthy state."); + } + } + + cancellationToken.ThrowIfCancellationRequested(); + } + finally + { + Interlocked.Decrement(ref _activeConsistencyWaiters); + } + } + + private async Task TryLogConsistencyWaitEventAsync(string status, string text, DateTime? startDate, CancellationToken cancellationToken) + { + try + { + using IScoped searchService = _searchServiceFactory(); + await searchService.Value.TryLogEvent(nameof(WaitForAllInstancesCacheConsistencyAsync), status, text, startDate, cancellationToken); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to log WaitForAllInstancesCacheConsistencyAsync event."); + } + } + + public async Task EnsureNoActiveReindexJobAsync(CancellationToken cancellationToken) + { + using IScoped fhirOperationDataStore = _fhirOperationDataStoreFactory(); + (bool found, string id) activeReindexJob = await fhirOperationDataStore.Value.CheckActiveReindexJobsAsync(cancellationToken); + + if (activeReindexJob.found) + { + throw new JobConflictException(Core.Resources.ChangesToSearchParametersNotAllowedWhileReindexing); + } + } + + public async Task ValidateSearchParameterAsync(ITypedElement searchParam, CancellationToken cancellationToken) { var searchParameterWrapper = new SearchParameterWrapper(searchParam); var searchParameterUrl = searchParameterWrapper.Url; @@ -114,11 +263,6 @@ await SearchParameterConcurrencyManager.ExecuteWithLockAsync( { throw new SearchParameterNotSupportedException(errorMessage); } - - _logger.LogInformation("Adding the search parameter '{Url}'", searchParameterWrapper.Url); - _searchParameterDefinitionManager.AddNewSearchParameters(new List { searchParam }); - - await _searchParameterStatusManager.AddSearchParameterStatusAsync(new List { searchParameterWrapper.Url }, cancellationToken); } catch (FhirException fex) { @@ -147,7 +291,8 @@ await SearchParameterConcurrencyManager.ExecuteWithLockAsync( } /// - /// Marks the Search Parameter as PendingDelete. + /// Marks the Search Parameter as PendingDelete. This is only used by DeletionService.cs and will be removed when refactoring is done + /// to allow deletion service to properly handle Hard deletions for Search Parameters (e.g. allow reindex prior to removing resource from DB). /// /// Search Parameter to update to Pending Delete status. /// Cancellation Token @@ -163,19 +308,10 @@ await SearchParameterConcurrencyManager.ExecuteWithLockAsync( { try { - // We need to make sure we have the latest search parameters before trying to delete - // existing search parameter. This is to avoid trying to update a search parameter that - // was recently added and that hasn't propogated to all fhir-server instances. - await GetAndApplySearchParameterUpdates(cancellationToken); + await EnsureNoActiveReindexJobAsync(cancellationToken); - // First we delete the status metadata from the data store as this function depends on - // the in memory definition manager. Once complete we remove the SearchParameter from - // the definition manager. _logger.LogInformation("Deleting the search parameter '{Url}'", searchParameterUrl); - await _searchParameterStatusManager.UpdateSearchParameterStatusAsync(new List() { searchParameterUrl }, SearchParameterStatus.PendingDelete, cancellationToken, ignoreSearchParameterNotSupportedException); - - // Update the status of the search parameter in the definition manager once the status is updated in the store. - _searchParameterDefinitionManager.UpdateSearchParameterStatus(searchParameterUrl, SearchParameterStatus.PendingDelete); + await _searchParameterStatusManager.UpdateSearchParameterStatusAsync(new[] { searchParameterUrl }, SearchParameterStatus.PendingDelete, cancellationToken); } catch (FhirException fex) { @@ -203,83 +339,10 @@ await SearchParameterConcurrencyManager.ExecuteWithLockAsync( cancellationToken); } - public async Task UpdateSearchParameterAsync(ITypedElement searchParam, RawResource previousSearchParam, CancellationToken cancellationToken) + public async Task UpdateSearchParameterStatusAsync(IReadOnlyCollection searchParameterUris, SearchParameterStatus status, CancellationToken cancellationToken, bool ignoreSearchParameterNotSupportedException = false) { - var prevSearchParam = _modelInfoProvider.ToTypedElement(previousSearchParam); - var prevSearchParamUrl = prevSearchParam.GetStringScalar("url"); - - await SearchParameterConcurrencyManager.ExecuteWithLockAsync( - prevSearchParamUrl, - async () => - { - try - { - // We need to make sure we have the latest search parameters before trying to update - // existing search parameter. This is to avoid trying to update a search parameter that - // was recently added and that hasn't propogated to all fhir-server instances. - await GetAndApplySearchParameterUpdates(cancellationToken); - - var searchParameterWrapper = new SearchParameterWrapper(searchParam); - var searchParameterInfo = new SearchParameterInfo(searchParameterWrapper); - (bool Supported, bool IsPartiallySupported) supportedResult = _searchParameterSupportResolver.IsSearchParameterSupported(searchParameterInfo); - - if (!supportedResult.Supported) - { - throw new SearchParameterNotSupportedException(searchParameterInfo.Url); - } - - // check data store specific support for SearchParameter - if (!_dataStoreSearchParameterValidator.ValidateSearchParameter(searchParameterInfo, out var errorMessage)) - { - throw new SearchParameterNotSupportedException(errorMessage); - } - - // As any part of the SearchParameter may have been changed, including the URL - // the most reliable method of updating the SearchParameter is to delete the previous - // data and insert the updated version - - if (!searchParameterWrapper.Url.Equals(prevSearchParamUrl, StringComparison.Ordinal)) - { - _logger.LogInformation("Deleting the search parameter '{Url}' (update step 1/2)", prevSearchParamUrl); - await _searchParameterStatusManager.DeleteSearchParameterStatusAsync(prevSearchParamUrl, cancellationToken); - try - { - _searchParameterDefinitionManager.DeleteSearchParameter(prevSearchParam); - } - catch (ResourceNotFoundException) - { - // do nothing, there may not be a search parameter to remove - } - } - - _logger.LogInformation("Adding the search parameter '{Url}' (update step 2/2)", searchParameterWrapper.Url); - _searchParameterDefinitionManager.AddNewSearchParameters(new List() { searchParam }); - await _searchParameterStatusManager.AddSearchParameterStatusAsync(new List() { searchParameterWrapper.Url }, cancellationToken); - } - catch (FhirException fex) - { - _logger.LogError(fex, "Error updating search parameter."); - fex.Issues.Add(new OperationOutcomeIssue( - OperationOutcomeConstants.IssueSeverity.Error, - OperationOutcomeConstants.IssueType.Exception, - Core.Resources.CustomSearchUpdateError)); - - throw; - } - catch (Exception ex) when (!(ex is FhirException)) - { - _logger.LogError(ex, "Unexpected error updating search parameter."); - var customSearchException = new ConfigureCustomSearchException(Core.Resources.CustomSearchUpdateError); - customSearchException.Issues.Add(new OperationOutcomeIssue( - OperationOutcomeConstants.IssueSeverity.Error, - OperationOutcomeConstants.IssueType.Exception, - ex.Message)); - - throw customSearchException; - } - }, - _logger, - cancellationToken); + await EnsureNoActiveReindexJobAsync(cancellationToken); + await _searchParameterStatusManager.UpdateSearchParameterStatusAsync(searchParameterUris, status, cancellationToken, ignoreSearchParameterNotSupportedException); } /// @@ -372,11 +435,47 @@ public async Task GetAndApplySearchParameterUpdates(CancellationToken cancellati var inCache = ParametersAreInCache(statusesToFetch, cancellationToken); - // if cache is updated directly and not from the database not all will have corresponding resources. Do not advance timestamp as results are not conclusive. - if (results.LastUpdated.HasValue && inCache && allHaveResources) // this should be the ony place in the code to assign last updated + DateTimeOffset? searchParamLastUpdatedToLog = null; + + // If cache is updated directly and not from the database not all will have corresponding resources. + // Do not advance or log the timestamp unless the cache contents are conclusive for this cycle. + if (inCache && allHaveResources) { - _searchParamLastUpdated = results.LastUpdated.Value; + if (results.LastUpdated.HasValue) // this should be the only place in the code to assign last updated + { + _searchParamLastUpdated = results.LastUpdated.Value; + } + + searchParamLastUpdatedToLog = _searchParamLastUpdated; } + + if (searchParamLastUpdatedToLog.HasValue + && (results.LastUpdated.HasValue || Volatile.Read(ref _activeConsistencyWaiters) > 0)) + { + // Log to EventLog for cross-instance convergence tracking (SQL only; Cosmos/File are no-ops). + // Emit the current cache timestamp for no-op refresh cycles only while a consistency waiter is active. + try + { + var lastUpdatedText = searchParamLastUpdatedToLog.Value.ToString("yyyy-MM-dd HH:mm:ss.fffffff"); + using IScoped searchService = _searchServiceFactory(); + await searchService.Value.TryLogEvent( + "SearchParameterCacheRefresh", + "End", + $"SearchParamLastUpdated={lastUpdatedText}", + null, + cancellationToken); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to log SearchParameterCacheRefresh event. Cross-instance convergence checks may be affected."); + } + } + + // Signal waiters that a refresh cycle has completed. + // This fires every cycle (even when no changes are found) because + // WaitForRefreshCyclesAsync counts completed cycles, not cycles with changes. + var previous = Interlocked.Exchange(ref _refreshSignal, new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously)); + previous.TrySetResult(true); } // This should handle racing condition between saving new parameter on one VM and refreshing cache on the other, @@ -425,45 +524,64 @@ private async Task> GetSearchParametersByUrls( } const int chunkSize = 100; - var searchParametersByUrl = new Dictionary(); + var searchParametersByUrl = new Dictionary(StringComparer.Ordinal); + var unresolvedUrls = new HashSet(urls, StringComparer.Ordinal); - // Process URLs in chunks to avoid SQL query limitations - for (int i = 0; i < urls.Count; i += chunkSize) - { - var urlChunk = urls.Skip(i).Take(chunkSize).ToList(); + using IScoped search = _searchServiceFactory.Invoke(); + + string continuationToken = null; - using IScoped search = _searchServiceFactory.Invoke(); + do + { + cancellationToken.ThrowIfCancellationRequested(); - // Build a query like: url=url1,url2,url3 - var urlQueryValue = string.Join(",", urlChunk); var queryParams = new List> { - new Tuple("url", urlQueryValue), - new Tuple("_count", chunkSize.ToString()), // we only need a maximum of chunkSize results back + Tuple.Create(KnownQueryParameterNames.Count, chunkSize.ToString()), }; - var result = await search.Value.SearchAsync(KnownResourceTypes.SearchParameter, queryParams, cancellationToken); + if (!string.IsNullOrEmpty(continuationToken)) + { + queryParams.Add( + Tuple.Create( + KnownQueryParameterNames.ContinuationToken, + ContinuationTokenEncoder.Encode(continuationToken))); + } - if (result != null) + var result = await search.Value.SearchAsync(KnownResourceTypes.SearchParameter, queryParams, cancellationToken); + if (result?.Results != null) { - foreach (var searchResultEntry in result.Results) + foreach (var entry in result.Results) { - var typedElement = searchResultEntry.Resource.RawResource.ToITypedElement(_modelInfoProvider); - var url = typedElement.GetStringScalar("url"); + var typedElement = entry.Resource?.RawResource?.ToITypedElement(_modelInfoProvider); + if (typedElement == null) + { + continue; + } - if (!string.IsNullOrEmpty(url)) + var url = typedElement.GetStringScalar("url"); + if (!string.IsNullOrEmpty(url) && unresolvedUrls.Remove(url)) { - if (!searchParametersByUrl.ContainsKey(url)) - { - searchParametersByUrl[url] = typedElement; - } - else + searchParametersByUrl[url] = typedElement; + + if (unresolvedUrls.Count == 0) { - _logger.LogWarning("More than one SearchParameter found with url {Url}. Using the first one found.", url); + return searchParametersByUrl; } } } } + + continuationToken = result?.ContinuationToken; + } + while (!string.IsNullOrEmpty(continuationToken)); + + if (unresolvedUrls.Count > 0) + { + _logger.LogWarning( + "Could not resolve {Count} SearchParameter URL(s). Samples: {Urls}", + unresolvedUrls.Count, + string.Join(", ", unresolvedUrls.Take(10))); } return searchParametersByUrl; diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/SearchParameterRequestContextPropertyNames.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/SearchParameterRequestContextPropertyNames.cs new file mode 100644 index 0000000000..6c725a699a --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/SearchParameterRequestContextPropertyNames.cs @@ -0,0 +1,12 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +namespace Microsoft.Health.Fhir.Core.Features.Search.Parameters +{ + public static class SearchParameterRequestContextPropertyNames + { + public const string PendingStatusUpdates = "SearchParameter.PendingStatusUpdates"; + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/CacheConsistencyResult.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/CacheConsistencyResult.cs new file mode 100644 index 0000000000..216e93db9d --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/CacheConsistencyResult.cs @@ -0,0 +1,29 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +namespace Microsoft.Health.Fhir.Core.Features.Search.Registry +{ + /// + /// Result of checking whether all active instances have converged their search parameter caches + /// to a target version. + /// + public class CacheConsistencyResult + { + /// + /// Gets or sets a value indicating whether all active instances have consistent caches. + /// + public bool IsConsistent { get; set; } + + /// + /// Gets or sets the total number of active hosts discovered. + /// + public int TotalActiveHosts { get; set; } + + /// + /// Gets or sets the number of hosts that have converged to the target version. + /// + public int ConvergedHosts { get; set; } + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/FilebasedSearchParameterStatusDataStore.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/FilebasedSearchParameterStatusDataStore.cs index 47cd01f251..6aeb4cb248 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/FilebasedSearchParameterStatusDataStore.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/FilebasedSearchParameterStatusDataStore.cs @@ -98,6 +98,12 @@ public void SyncStatuses(IReadOnlyCollection stat // Do nothing. This is only required for SQL. } + public Task CheckCacheConsistencyAsync(string targetSearchParamLastUpdated, DateTime syncStartDate, DateTime activeHostsSince, CancellationToken cancellationToken) + { + // File-based registry is single-instance only. Always consistent. + return Task.FromResult(new CacheConsistencyResult { IsConsistent = true, TotalActiveHosts = 1, ConvergedHosts = 1 }); + } + public async Task GetMaxLastUpdatedAsync(CancellationToken cancellationToken) { // Get all statuses to find the max LastUpdated diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/ISearchParameterStatusDataStore.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/ISearchParameterStatusDataStore.cs index 7e1e111c0a..dbe92be6d2 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/ISearchParameterStatusDataStore.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/ISearchParameterStatusDataStore.cs @@ -19,5 +19,16 @@ public interface ISearchParameterStatusDataStore void SyncStatuses(IReadOnlyCollection statuses); Task TryLogEvent(string process, string status, string text, DateTime? startDate, CancellationToken cancellationToken); + + /// + /// Checks whether all active instances have converged their search parameter caches + /// to at least the specified target timestamp. + /// + /// The target SearchParamLastUpdated timestamp to check for. + /// Only cache refresh sync records on or after this time are considered for convergence. + /// Only active-host evidence on or after this time is considered. + /// Cancellation token. + /// A indicating convergence status. + Task CheckCacheConsistencyAsync(string targetSearchParamLastUpdated, DateTime syncStartDate, DateTime activeHostsSince, CancellationToken cancellationToken); } } diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/ISearchParameterStatusManager.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/ISearchParameterStatusManager.cs index b287c33f66..cb7ced63e0 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/ISearchParameterStatusManager.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/ISearchParameterStatusManager.cs @@ -24,5 +24,7 @@ public interface ISearchParameterStatusManager Task Handle(SearchParameterDefinitionManagerInitialized notification, CancellationToken cancellationToken); Task UpdateSearchParameterStatusAsync(IReadOnlyCollection searchParameterUris, SearchParameterStatus status, CancellationToken cancellationToken, bool ignoreSearchParameterNotSupportedException = false); + + Task CheckCacheConsistencyAsync(string targetSearchParamLastUpdated, DateTime syncStartDate, DateTime activeHostsSince, CancellationToken cancellationToken); } } diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/SearchParameterCacheRefreshBackgroundService.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/SearchParameterCacheRefreshBackgroundService.cs index 10c0474dad..e1f1b4ac9e 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/SearchParameterCacheRefreshBackgroundService.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/SearchParameterCacheRefreshBackgroundService.cs @@ -137,8 +137,17 @@ private async void OnRefreshTimer(object state) } finally { - // Always release the semaphore to allow the next refresh operation - _refreshSemaphore.Release(); + // Guard against ObjectDisposedException: because OnRefreshTimer is async void, + // Timer.Dispose(WaitHandle) only waits for the synchronous portion of the callback. + // The async continuation can resume after the semaphore has been disposed during shutdown. + try + { + _refreshSemaphore.Release(); + } + catch (ObjectDisposedException) + { + // Expected during host shutdown when Dispose() races with an in-flight async callback. + } } } diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/SearchParameterStatusManager.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/SearchParameterStatusManager.cs index 69c4ad19d1..51c1cf1255 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/SearchParameterStatusManager.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/SearchParameterStatusManager.cs @@ -144,7 +144,6 @@ public async Task UpdateSearchParameterStatusAsync(IReadOnlyCollection s } var searchParameterStatusList = new List(); - var updated = new List(); var parameters = (await _searchParameterStatusDataStore.GetSearchParameterStatuses(cancellationToken)) .ToDictionary(x => x.Uri.OriginalString, StringComparer.Ordinal); @@ -152,52 +151,25 @@ public async Task UpdateSearchParameterStatusAsync(IReadOnlyCollection s { _logger.LogInformation("Setting the search parameter status of '{Uri}' to '{NewStatus}'", uri, status.ToString()); - try - { - SearchParameterInfo paramInfo = _searchParameterDefinitionManager.GetSearchParameter(uri); - updated.Add(paramInfo); - paramInfo.IsSearchable = status == SearchParameterStatus.Enabled; - paramInfo.IsSupported = status == SearchParameterStatus.Supported || status == SearchParameterStatus.Enabled; - - if (parameters.TryGetValue(uri, out var existingStatus)) - { - existingStatus.Status = status; - - if (paramInfo.IsSearchable && existingStatus.SortStatus == SortParameterStatus.Supported) - { - existingStatus.SortStatus = SortParameterStatus.Enabled; - paramInfo.SortStatus = SortParameterStatus.Enabled; - } + // Validate that the search parameter exists in the definition manager + _searchParameterDefinitionManager.GetSearchParameter(uri); - searchParameterStatusList.Add(existingStatus); - } - else - { - searchParameterStatusList.Add(new ResourceSearchParameterStatus - { - Status = status, - Uri = new Uri(uri), - }); - } + if (parameters.TryGetValue(uri, out var existingStatus)) + { + existingStatus.Status = status; + searchParameterStatusList.Add(existingStatus); } - catch (SearchParameterNotSupportedException ex) + else { - _logger.LogError(ex, "The search parameter '{Uri}' not supported.", uri); - - // Note: SearchParameterNotSupportedException can be thrown by SearchParameterDefinitionManager.GetSearchParameter - // when the given url is not found in its cache that can happen when the cache becomes out of sync with the store. - // Use this flag to ignore the exception and continue the update process for the rest of search parameters. - // (e.g. $bulk-delete ensuring deletion of as many search parameters as possible.) - if (!ignoreSearchParameterNotSupportedException) + searchParameterStatusList.Add(new ResourceSearchParameterStatus { - throw; - } + Status = status, + Uri = new Uri(uri), + }); } } await _searchParameterStatusDataStore.UpsertStatuses(searchParameterStatusList, cancellationToken); - - await _mediator.Publish(new SearchParametersUpdatedNotification(updated), cancellationToken); } public async Task AddSearchParameterStatusAsync(IReadOnlyCollection searchParamUris, CancellationToken cancellationToken) @@ -293,6 +265,11 @@ private static TempStatus EvaluateSearchParamStatus(ResourceSearchParameterStatu return tempStatus; } + public async Task CheckCacheConsistencyAsync(string targetSearchParamLastUpdated, DateTime syncStartDate, DateTime activeHostsSince, CancellationToken cancellationToken) + { + return await _searchParameterStatusDataStore.CheckCacheConsistencyAsync(targetSearchParamLastUpdated, syncStartDate, activeHostsSince, cancellationToken); + } + private struct TempStatus { public bool IsSearchable; diff --git a/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/CosmosFhirDataStoreTests.cs b/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/CosmosFhirDataStoreTests.cs index df0bb3d619..536aee17f9 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/CosmosFhirDataStoreTests.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/CosmosFhirDataStoreTests.cs @@ -30,6 +30,7 @@ using Microsoft.Health.Fhir.Core.Features.Persistence; using Microsoft.Health.Fhir.Core.Features.Persistence.Orchestration; using Microsoft.Health.Fhir.Core.Features.Search; +using Microsoft.Health.Fhir.Core.Features.Search.Registry; using Microsoft.Health.Fhir.Core.Features.Search.SearchValues; using Microsoft.Health.Fhir.Core.Models; using Microsoft.Health.Fhir.Core.UnitTests.Extensions; @@ -81,7 +82,9 @@ public CosmosFhirDataStoreTests() Options.Create(new CoreFeatureConfiguration()), _bundleOrchestrator, new Lazy(Substitute.For()), - ModelInfoProvider.Instance); + ModelInfoProvider.Instance, + Substitute.For(), + requestContextAccessor); } [Fact] diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosFhirDataStore.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosFhirDataStore.cs index d60951de65..7cffade697 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosFhirDataStore.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosFhirDataStore.cs @@ -22,15 +22,19 @@ using Microsoft.Extensions.Options; using Microsoft.Health.Abstractions.Exceptions; using Microsoft.Health.Core; +using Microsoft.Health.Core.Features.Context; using Microsoft.Health.Extensions.DependencyInjection; using Microsoft.Health.Fhir.Core.Configs; using Microsoft.Health.Fhir.Core.Exceptions; using Microsoft.Health.Fhir.Core.Extensions; using Microsoft.Health.Fhir.Core.Features; using Microsoft.Health.Fhir.Core.Features.Conformance; +using Microsoft.Health.Fhir.Core.Features.Context; using Microsoft.Health.Fhir.Core.Features.Definition; using Microsoft.Health.Fhir.Core.Features.Persistence; using Microsoft.Health.Fhir.Core.Features.Persistence.Orchestration; +using Microsoft.Health.Fhir.Core.Features.Search.Parameters; +using Microsoft.Health.Fhir.Core.Features.Search.Registry; using Microsoft.Health.Fhir.Core.Models; using Microsoft.Health.Fhir.CosmosDb.Core.Configs; using Microsoft.Health.Fhir.CosmosDb.Features.Queries; @@ -69,6 +73,8 @@ public sealed class CosmosFhirDataStore : IFhirDataStore, IProvideCapability private readonly CoreFeatureConfiguration _coreFeatures; private readonly IBundleOrchestrator _bundleOrchestrator; private readonly IModelInfoProvider _modelInfoProvider; + private readonly ISearchParameterStatusDataStore _searchParameterStatusDataStore; + private readonly RequestContextAccessor _requestContextAccessor; /// /// Initializes a new instance of the class. @@ -86,6 +92,8 @@ public sealed class CosmosFhirDataStore : IFhirDataStore, IProvideCapability /// Bundle orchestrator /// The supported search parameters /// The model info provider to determine the FHIR version when handling resource conflicts. + /// The search parameter status data store used to persist queued search parameter status updates. + /// Accessor for request-scoped context properties, including pending search parameter status updates. public CosmosFhirDataStore( IScoped containerScope, CosmosDataStoreConfiguration cosmosDataStoreConfiguration, @@ -96,7 +104,9 @@ public CosmosFhirDataStore( IOptions coreFeatures, IBundleOrchestrator bundleOrchestrator, Lazy supportedSearchParameters, - IModelInfoProvider modelInfoProvider) + IModelInfoProvider modelInfoProvider, + ISearchParameterStatusDataStore searchParameterStatusDataStore, + RequestContextAccessor requestContextAccessor) { EnsureArg.IsNotNull(containerScope, nameof(containerScope)); EnsureArg.IsNotNull(cosmosDataStoreConfiguration, nameof(cosmosDataStoreConfiguration)); @@ -107,6 +117,8 @@ public CosmosFhirDataStore( EnsureArg.IsNotNull(coreFeatures, nameof(coreFeatures)); EnsureArg.IsNotNull(bundleOrchestrator, nameof(bundleOrchestrator)); EnsureArg.IsNotNull(supportedSearchParameters, nameof(supportedSearchParameters)); + EnsureArg.IsNotNull(searchParameterStatusDataStore, nameof(searchParameterStatusDataStore)); + EnsureArg.IsNotNull(requestContextAccessor, nameof(requestContextAccessor)); _containerScope = containerScope; _cosmosDataStoreConfiguration = cosmosDataStoreConfiguration; @@ -117,6 +129,8 @@ public CosmosFhirDataStore( _coreFeatures = coreFeatures.Value; _bundleOrchestrator = bundleOrchestrator; _modelInfoProvider = modelInfoProvider; + _searchParameterStatusDataStore = searchParameterStatusDataStore; + _requestContextAccessor = requestContextAccessor; } public async Task TryLogEvent(string process, string status, string text, DateTime? startDate, CancellationToken cancellationToken) @@ -245,19 +259,32 @@ public async Task UpsertAsync(ResourceWrapperOperation resource, IBundleOrchestratorOperation operation = _bundleOrchestrator.GetOperation(resource.BundleResourceContext.BundleOperationId); // Internally Bundle Operation calls "MergeAsync". - return await operation.AppendResourceAsync(resource, this, cancellationToken).ConfigureAwait(false); + UpsertOutcome result = await operation.AppendResourceAsync(resource, this, cancellationToken).ConfigureAwait(false); + if (string.Equals(resource.Wrapper.ResourceTypeName, KnownResourceTypes.SearchParameter, StringComparison.Ordinal)) + { + await PersistPendingSearchParameterStatusUpdatesAsync(cancellationToken); + } + + return result; } else { try { - return await InternalUpsertAsync( + var upsertOutcome = await InternalUpsertAsync( resource.Wrapper, resource.WeakETag, resource.AllowCreate, resource.KeepHistory, cancellationToken, resource.RequireETagOnUpdate); + + if (string.Equals(resource.Wrapper.ResourceTypeName, KnownResourceTypes.SearchParameter, StringComparison.Ordinal)) + { + await PersistPendingSearchParameterStatusUpdatesAsync(cancellationToken); + } + + return upsertOutcome; } catch (FhirException fhirException) { @@ -475,6 +502,57 @@ await retryPolicy.ExecuteAsync( } } + private async Task PersistPendingSearchParameterStatusUpdatesAsync(CancellationToken cancellationToken) + { + var context = _requestContextAccessor.RequestContext; + if (context?.Properties == null) + { + return; + } + + if (!context.Properties.TryGetValue(SearchParameterRequestContextPropertyNames.PendingStatusUpdates, out var value) || + value is not List pendingStatuses || + pendingStatuses.Count == 0) + { + return; + } + + List snapshot; + lock (pendingStatuses) + { + if (pendingStatuses.Count == 0) + { + return; + } + + snapshot = pendingStatuses + .Where(status => status?.Uri != null) + .GroupBy(status => status.Uri.OriginalString, StringComparer.Ordinal) + .Select(group => group.Last()) + .ToList(); + } + + if (snapshot.Count == 0) + { + return; + } + + await _searchParameterStatusDataStore.UpsertStatuses(snapshot, cancellationToken); + + lock (pendingStatuses) + { + foreach (var item in snapshot) + { + pendingStatuses.Remove(item); + } + + if (pendingStatuses.Count == 0) + { + context.Properties.Remove(SearchParameterRequestContextPropertyNames.PendingStatusUpdates); + } + } + } + public async Task GetAsync(ResourceKey key, CancellationToken cancellationToken) { EnsureArg.IsNotNull(key, nameof(key)); diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Registry/CosmosDbSearchParameterStatusDataStore.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Registry/CosmosDbSearchParameterStatusDataStore.cs index 7aee487ffa..ef7a3e7667 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Registry/CosmosDbSearchParameterStatusDataStore.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Registry/CosmosDbSearchParameterStatusDataStore.cs @@ -114,6 +114,13 @@ public void SyncStatuses(IReadOnlyCollection stat // Do nothing. This is only required for SQL. } + public Task CheckCacheConsistencyAsync(string targetSearchParamLastUpdated, DateTime syncStartDate, DateTime activeHostsSince, CancellationToken cancellationToken) + { + // Cosmos DB does not use EventLog-based convergence tracking. + // Return immediately as consistent since each instance refreshes from the same Cosmos container. + return Task.FromResult(new CacheConsistencyResult { IsConsistent = true, TotalActiveHosts = 1, ConvergedHosts = 1 }); + } + public void Dispose() { _statusListSemaphore?.Dispose(); diff --git a/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Controllers/BulkDeleteControllerTests.cs b/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Controllers/BulkDeleteControllerTests.cs index 3ba3187b32..904c6eac0e 100644 --- a/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Controllers/BulkDeleteControllerTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Controllers/BulkDeleteControllerTests.cs @@ -23,12 +23,15 @@ using Microsoft.Health.Fhir.Core.Features; using Microsoft.Health.Fhir.Core.Features.Operations.BulkDelete.Messages; using Microsoft.Health.Fhir.Core.Features.Routing; +using Microsoft.Health.Fhir.Core.Features.Search.Parameters; using Microsoft.Health.Fhir.Core.Messages.Delete; using Microsoft.Health.Fhir.Core.Models; using Microsoft.Health.Fhir.Tests.Common; using Microsoft.Health.Test.Utilities; using NSubstitute; using Xunit; + +using FhirJobConflictException = global::Microsoft.Health.Fhir.Core.Features.Operations.JobConflictException; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Api.UnitTests.Controllers @@ -42,10 +45,12 @@ public class BulkDeleteControllerTests private readonly BulkDeleteController _controller; private readonly HttpRequest _httpRequest; private readonly IMediator _mediator; + private readonly ISearchParameterOperations _searchParameterOperations; public BulkDeleteControllerTests() { _mediator = Substitute.For(); + _searchParameterOperations = Substitute.For(); _mediator.Send( Arg.Any(), Arg.Any()) @@ -76,6 +81,7 @@ public BulkDeleteControllerTests() _controller = new BulkDeleteController( _mediator, + _searchParameterOperations, urlResolver); _controller.ControllerContext = new ControllerContext( new ActionContext( @@ -186,6 +192,30 @@ await Run( purgeHistory)); } + [Fact] + public async Task GivenBulkDeleteForSearchParameter_WhenReindexIsRunning_ThenConflictIsThrown() + { + _searchParameterOperations + .When(x => x.EnsureNoActiveReindexJobAsync(Arg.Any())) + .Do(_ => throw new FhirJobConflictException("reindex running")); + + await Assert.ThrowsAsync(() => _controller.BulkDeleteByResourceType(KnownResourceTypes.SearchParameter, new HardDeleteModel(), false, false)); + await _mediator.DidNotReceiveWithAnyArgs().Send(default, default); + } + + [Fact] + public async Task GivenBulkDeleteWithSearchParameterExcluded_WhenReindexIsRunning_ThenRequestStillSucceeds() + { + _searchParameterOperations + .When(x => x.EnsureNoActiveReindexJobAsync(Arg.Any())) + .Do(_ => throw new FhirJobConflictException("reindex running")); + + var response = await _controller.BulkDelete(new HardDeleteModel(), false, false, KnownResourceTypes.SearchParameter); + + Assert.IsType(response); + await _searchParameterOperations.DidNotReceive().EnsureNoActiveReindexJobAsync(Arg.Any()); + } + [Theory] [InlineData(0, HttpStatusCode.Accepted)] [InlineData(1, HttpStatusCode.OK)] diff --git a/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Features/Resources/Bundle/BundleHandlerEdgeCaseTests.cs b/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Features/Resources/Bundle/BundleHandlerEdgeCaseTests.cs index d58e91e88e..b16d90ffe3 100644 --- a/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Features/Resources/Bundle/BundleHandlerEdgeCaseTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Features/Resources/Bundle/BundleHandlerEdgeCaseTests.cs @@ -28,6 +28,7 @@ using Microsoft.Health.Fhir.Core.Features.Persistence.Orchestration; using Microsoft.Health.Fhir.Core.Features.Resources; using Microsoft.Health.Fhir.Core.Features.Search; +using Microsoft.Health.Fhir.Core.Features.Search.Registry; using Microsoft.Health.Fhir.Core.Features.Security.Authorization; using Microsoft.Health.Fhir.Core.Features.Validation; using Microsoft.Health.Fhir.Core.UnitTests.Features.Context; @@ -163,6 +164,7 @@ private IFhirRequestContext CreateRequestContextForBundleHandlerProcessing(Bundl mediator, router, profilesResolver, + Substitute.For(), NullLogger.Instance); return fhirRequestContextAccessor.RequestContext; diff --git a/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Features/Resources/Bundle/BundleHandlerTests.cs b/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Features/Resources/Bundle/BundleHandlerTests.cs index dd84494a52..f6c1f42c20 100644 --- a/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Features/Resources/Bundle/BundleHandlerTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Features/Resources/Bundle/BundleHandlerTests.cs @@ -35,6 +35,7 @@ using Microsoft.Health.Fhir.Core.Features.Resources; using Microsoft.Health.Fhir.Core.Features.Resources.Bundle; using Microsoft.Health.Fhir.Core.Features.Search; +using Microsoft.Health.Fhir.Core.Features.Search.Registry; using Microsoft.Health.Fhir.Core.Features.Security.Authorization; using Microsoft.Health.Fhir.Core.Features.Validation; using Microsoft.Health.Fhir.Core.Messages.Bundle; @@ -139,6 +140,7 @@ public BundleHandlerTests() _mediator, _router, _profilesResolver, + Substitute.For(), NullLogger.Instance); } diff --git a/src/Microsoft.Health.Fhir.Shared.Api/Controllers/BulkDeleteController.cs b/src/Microsoft.Health.Fhir.Shared.Api/Controllers/BulkDeleteController.cs index 8433e9f924..9b0188d85b 100644 --- a/src/Microsoft.Health.Fhir.Shared.Api/Controllers/BulkDeleteController.cs +++ b/src/Microsoft.Health.Fhir.Shared.Api/Controllers/BulkDeleteController.cs @@ -24,7 +24,9 @@ using Microsoft.Health.Fhir.Core.Features.Operations.BulkDelete; using Microsoft.Health.Fhir.Core.Features.Operations.BulkDelete.Messages; using Microsoft.Health.Fhir.Core.Features.Routing; +using Microsoft.Health.Fhir.Core.Features.Search.Parameters; using Microsoft.Health.Fhir.Core.Messages.Delete; +using Microsoft.Health.Fhir.Core.Models; using Microsoft.Health.Fhir.ValueSets; namespace Microsoft.Health.Fhir.Api.Controllers @@ -35,6 +37,7 @@ namespace Microsoft.Health.Fhir.Api.Controllers public class BulkDeleteController : Controller { private readonly IMediator _mediator; + private readonly ISearchParameterOperations _searchParameterOperations; private readonly IUrlResolver _urlResolver; private readonly HashSet _excludedParameters = new(new PropertyEqualityComparer(StringComparison.OrdinalIgnoreCase, s => s)) @@ -48,9 +51,11 @@ public class BulkDeleteController : Controller public BulkDeleteController( IMediator mediator, + ISearchParameterOperations searchParameterOperations, IUrlResolver urlResolver) { _mediator = EnsureArg.IsNotNull(mediator, nameof(mediator)); + _searchParameterOperations = EnsureArg.IsNotNull(searchParameterOperations, nameof(searchParameterOperations)); _urlResolver = EnsureArg.IsNotNull(urlResolver, nameof(urlResolver)); } @@ -147,11 +152,26 @@ private async Task SendDeleteRequest(string typeParameter, bool h excludedResourceTypesList = excludedResourceTypes.Split(',').ToList(); } + if (CanAffectSearchParameters(typeParameter, excludedResourceTypesList)) + { + await _searchParameterOperations.EnsureNoActiveReindexJobAsync(HttpContext.RequestAborted); + } + CreateBulkDeleteResponse result = await _mediator.BulkDeleteAsync(deleteOperation, typeParameter, searchParameters, softDeleteCleanup, excludedResourceTypesList, removeReferences, HttpContext.RequestAborted); var response = JobResult.Accepted(); response.SetContentLocationHeader(_urlResolver, OperationsConstants.BulkDelete, result.Id.ToString()); return response; } + + private static bool CanAffectSearchParameters(string resourceType, IList excludedResourceTypes) + { + if (excludedResourceTypes?.Any(x => string.Equals(x, KnownResourceTypes.SearchParameter, StringComparison.OrdinalIgnoreCase)) == true) + { + return false; + } + + return string.Equals(resourceType, KnownResourceTypes.SearchParameter, StringComparison.OrdinalIgnoreCase); + } } } diff --git a/src/Microsoft.Health.Fhir.Shared.Api/Features/Resources/Bundle/BundleHandler.cs b/src/Microsoft.Health.Fhir.Shared.Api/Features/Resources/Bundle/BundleHandler.cs index 3b10e179d4..06745778a3 100644 --- a/src/Microsoft.Health.Fhir.Shared.Api/Features/Resources/Bundle/BundleHandler.cs +++ b/src/Microsoft.Health.Fhir.Shared.Api/Features/Resources/Bundle/BundleHandler.cs @@ -48,6 +48,8 @@ using Microsoft.Health.Fhir.Core.Features.Resources; using Microsoft.Health.Fhir.Core.Features.Resources.Bundle; using Microsoft.Health.Fhir.Core.Features.Routing; +using Microsoft.Health.Fhir.Core.Features.Search.Parameters; +using Microsoft.Health.Fhir.Core.Features.Search.Registry; using Microsoft.Health.Fhir.Core.Features.Security; using Microsoft.Health.Fhir.Core.Features.Validation; using Microsoft.Health.Fhir.Core.Messages.Bundle; @@ -88,6 +90,7 @@ public partial class BundleHandler : IRequestHandler _accumulatedPendingStatuses = new(); + /// /// Headers to propagate from the inner actions to the outer HTTP request. /// @@ -137,6 +143,7 @@ public BundleHandler( IMediator mediator, IRouter router, IProvideProfilesForValidation profilesResolver, + ISearchParameterStatusDataStore searchParameterStatusDataStore, ILogger logger) : this() { @@ -156,6 +163,7 @@ public BundleHandler( _mediator = EnsureArg.IsNotNull(mediator, nameof(mediator)); _router = EnsureArg.IsNotNull(router, nameof(router)); _profilesResolver = EnsureArg.IsNotNull(profilesResolver, nameof(profilesResolver)); + _searchParameterStatusDataStore = EnsureArg.IsNotNull(searchParameterStatusDataStore, nameof(searchParameterStatusDataStore)); _logger = EnsureArg.IsNotNull(logger, nameof(logger)); // Not all versions support the same enum values, so do the dictionary creation in the version specific partial. @@ -483,6 +491,7 @@ private async Task ExecuteTransactionForAllRequestsAsync(Hl7.Fhir.Model.Bundle r using (var transaction = _transactionHandler.BeginTransaction()) { await ProcessAllResourcesInABundleAsRequestsAsync(responseBundle, BundleProcessingLogic.Sequential, cancellationToken); + await FlushPendingSearchParameterStatusesAsync(cancellationToken); transaction.Complete(); } } @@ -503,6 +512,44 @@ private async Task ExecuteTransactionForAllRequestsAsync(Hl7.Fhir.Model.Bundle r } } + /// + /// After each bundle entry's handler invocation, drains any pending search parameter statuses + /// from the request context into . + /// This ensures statuses survive across context replacements in SetupContexts. + /// + private void DrainPendingSearchParameterStatuses() + { + var context = _fhirRequestContextAccessor.RequestContext; + if (context?.Properties == null || + !context.Properties.TryGetValue(SearchParameterRequestContextPropertyNames.PendingStatusUpdates, out object value) || + value is not List statuses || + statuses.Count == 0) + { + return; + } + + _accumulatedPendingStatuses.AddRange(statuses); + context.Properties.Remove(SearchParameterRequestContextPropertyNames.PendingStatusUpdates); + } + + private async Task FlushPendingSearchParameterStatusesAsync(CancellationToken cancellationToken) + { + // Also drain any remaining statuses from the last entry's context. + DrainPendingSearchParameterStatuses(); + + if (_accumulatedPendingStatuses.Count == 0) + { + return; + } + + _logger.LogInformation( + "FlushPendingSearchParameterStatuses: Flushing {Count} accumulated statuses.", + _accumulatedPendingStatuses.Count); + + await _searchParameterStatusDataStore.UpsertStatuses(_accumulatedPendingStatuses, cancellationToken); + _accumulatedPendingStatuses.Clear(); + } + private async Task FillRequestLists(List bundleEntries, CancellationToken cancellationToken) { if (_bundleConfiguration.EntryLimit != default && bundleEntries.Count > _bundleConfiguration.EntryLimit) @@ -673,6 +720,8 @@ private async Task ExecuteRequestsWithSingleHttpVerbInSequenceAs await resourceContext.Context.Handler.Invoke(httpContext); + DrainPendingSearchParameterStatuses(); + // we will retry a 429 one time per request in the bundle if (httpContext.Response.StatusCode == (int)HttpStatusCode.TooManyRequests) { diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Definition/SearchParameterDefinitionBuilderTests.cs b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Definition/SearchParameterDefinitionBuilderTests.cs index 990b915c75..50f1cc04ad 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Definition/SearchParameterDefinitionBuilderTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Definition/SearchParameterDefinitionBuilderTests.cs @@ -219,6 +219,66 @@ public void GivenExistingSearchParameterWithSameUrl_WhenBuilt_ThenStaticResource Assert.Same(SearchParameterInfo.ResourceTypeSearchParameter, resolved); } + [Fact] + public void GivenValidSearchParameterDefinitionFile_WhenBuilt_ThenInjectedResourceTypeParameterIsSystemDefined() + { + var bundle = SearchParameterDefinitionBuilder.ReadEmbeddedSearchParameters( + _validEntriesFile, + ModelInfoProvider.Instance, + $"{typeof(Definitions).Namespace}.DefinitionFiles", + typeof(EmbeddedResourceManager).Assembly); + + SearchParameterDefinitionBuilder.Build( + bundle.Entries.Select(e => e.Resource).ToList(), + _uriDictionary, + _resourceTypeDictionary, + ModelInfoProvider.Instance, + _searchParameterComparer, + NullLogger.Instance); + + Assert.True(_uriDictionary.TryGetValue(SearchParameterNames.ResourceTypeUri.OriginalString, out var resolved)); + Assert.True(resolved.IsSystemDefined); + } + + [Fact] + public void GivenSystemDefinedResourceTypeParameter_WhenBuiltAgainAsNonSystem_ThenSystemDefinedRemainsTrue() + { + SearchParameterDefinitionBuilder.Build( + new List(), + _uriDictionary, + _resourceTypeDictionary, + ModelInfoProvider.Instance, + _searchParameterComparer, + NullLogger.Instance, + isSystemDefined: true); + + var resourceTypeSearchParameter = new SearchParameter + { + Url = SearchParameterNames.ResourceTypeUri.OriginalString, + Code = SearchParameterNames.ResourceType, + Name = SearchParameterNames.ResourceType, + Type = SearchParamType.Token, + Expression = "Resource.type().name", +#if R4 || R4B || Stu3 + Base = new List { ResourceType.Resource }, +#else + Base = new List { VersionIndependentResourceTypesAll.Resource }, +#endif + }; + + SearchParameterDefinitionBuilder.Build( + new List { resourceTypeSearchParameter.ToTypedElement() }, + _uriDictionary, + _resourceTypeDictionary, + ModelInfoProvider.Instance, + _searchParameterComparer, + NullLogger.Instance, + isSystemDefined: false); + + Assert.True(_uriDictionary.TryGetValue(SearchParameterNames.ResourceTypeUri.OriginalString, out var resolved)); + Assert.True(resolved.IsSystemDefined); + } + [Theory] [MemberData(nameof(GetSearchParameterConflictsData))] public void GivenSearchParametersWithConflicts_WhenBuilt_ThenResourceTypeDictionaryShouldContainSuperSetSearchParameters( diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Operations/Reindex/ReindexOrchestratorJobTests.cs b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Operations/Reindex/ReindexOrchestratorJobTests.cs index b1830b0c36..15647d721f 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Operations/Reindex/ReindexOrchestratorJobTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Operations/Reindex/ReindexOrchestratorJobTests.cs @@ -264,6 +264,14 @@ public async Task ExecuteAsync_WhenCancellationRequested_ReturnsJobCancelledResu var cancellationTokenSource = new CancellationTokenSource(); cancellationTokenSource.CancelAfter(10); // Cancel after short delay + // Make WaitForRefreshCyclesAsync block until cancellation, simulating a real wait + _searchParameterOperations.WaitForRefreshCyclesAsync(Arg.Any(), Arg.Any()) + .Returns(callInfo => + { + var ct = callInfo.ArgAt(1); + return Task.Delay(Timeout.Infinite, ct); + }); + var jobInfo = await CreateReindexJobRecord(); var orchestrator = CreateReindexOrchestratorJob(waitMultiplier: 1); @@ -574,7 +582,7 @@ public async Task GetResourceCountForQueryAsync_WithValidQuery_ReturnsSearchResu .Returns(paramHashMap); _searchParameterOperations.GetSearchParameterHash(Arg.Any()) - .Returns("hash"); + .Returns(callInfo => paramHashMap.TryGetValue(callInfo.ArgAt(0), out var h) ? h : null); // Create a search result with 100 resources var searchResult = CreateSearchResult( @@ -2038,5 +2046,43 @@ public async Task CheckForCompletionAsync_WithCaseVariantUrls_MaintainsBothVaria // Assert Assert.NotNull(jobResult); } + + [Fact] + public async Task RefreshSearchParameterCache_WaitsForConfiguredNumberOfCacheRefreshCycles() + { + // Arrange + int configuredMultiplier = 3; + + var emptyStatus = new List(); + _searchParameterStatusManager.GetAllSearchParameterStatus(Arg.Any()) + .Returns(emptyStatus); + + var jobInfo = await CreateReindexJobRecord(); + var orchestrator = CreateReindexOrchestratorJob(waitMultiplier: configuredMultiplier); + + // Act + var result = await orchestrator.ExecuteAsync(jobInfo, _cancellationToken); + + // Assert - WaitForRefreshCyclesAsync should have been called twice (Start and End) with the configured multiplier + await _searchParameterOperations.Received(2).WaitForRefreshCyclesAsync(configuredMultiplier, Arg.Any()); + } + + [Fact] + public async Task RefreshSearchParameterCache_WithZeroMultiplier_DoesNotWait() + { + // Arrange + var emptyStatus = new List(); + _searchParameterStatusManager.GetAllSearchParameterStatus(Arg.Any()) + .Returns(emptyStatus); + + var jobInfo = await CreateReindexJobRecord(); + var orchestrator = CreateReindexOrchestratorJob(waitMultiplier: 0); + + // Act + var result = await orchestrator.ExecuteAsync(jobInfo, _cancellationToken); + + // Assert - WaitForRefreshCyclesAsync should have been called with 0 (returns immediately) + await _searchParameterOperations.Received(2).WaitForRefreshCyclesAsync(0, Arg.Any()); + } } } diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Operations/Reindex/ReindexProcessingJobTests.cs b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Operations/Reindex/ReindexProcessingJobTests.cs index 1e3804b07b..23711e1fa2 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Operations/Reindex/ReindexProcessingJobTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Operations/Reindex/ReindexProcessingJobTests.cs @@ -957,6 +957,8 @@ public async Task ExecuteAsync_WithOutOfMemoryException_ReducesBatchSizeAndRetri Status = JobStatus.Running, }; + _searchParameterOperations.GetSearchParameterHash(Arg.Any()).Returns(job.SearchParameterHash); + var successfulEntries = Enumerable.Range(1, 5) .Select(i => CreateSearchResultEntry(i.ToString(), expectedResourceType)) .ToList(); @@ -1311,5 +1313,201 @@ public async Task ExecuteAsync_WithRepeatedOutOfMemoryException_ReturnsProcessin Assert.NotNull(jobResult.Error); Assert.Contains(expectedErrorSubstring, jobResult.Error); } + + [Fact] + public async Task CheckDiscrepancies_WhenHashMismatch_ThrowsReindexJobException() + { + // Arrange + var expectedResourceType = "Account"; + var requestedHash = "orchestratorHash"; + var staleHash = "staleHash"; + + var job = new ReindexProcessingJobDefinition() + { + MaximumNumberOfResourcesPerQuery = 100, + MaximumNumberOfResourcesPerWrite = 100, + ResourceType = expectedResourceType, + ResourceCount = new SearchResultReindex() + { + Count = _mockedSearchCount, + EndResourceSurrogateId = 0, + StartResourceSurrogateId = 0, + }, + SearchParameterHash = requestedHash, + SearchParameterUrls = new List() { "http://hl7.org/fhir/SearchParam/Account-status" }, + TypeId = (int)JobType.ReindexProcessing, + }; + + _searchParameterOperations.GetSearchParameterHash(Arg.Any()).Returns(staleHash); + + var jobInfo = new JobInfo() + { + Id = 1, + Definition = JsonConvert.SerializeObject(job), + QueueType = (byte)QueueType.Reindex, + GroupId = 1, + CreateDate = DateTime.UtcNow, + Status = JobStatus.Running, + }; + + // Act & Assert - Job should fail immediately on mismatch + var exception = await Assert.ThrowsAsync( + async () => await _reindexProcessingJobTaskFactory().ExecuteAsync(jobInfo, _cancellationToken)); + + Assert.Contains($"ResourceType={expectedResourceType} SearchParameterHash: Requested={requestedHash} != Current={staleHash}", exception.Message); + await _searchParameterOperations.DidNotReceive().WaitForRefreshCyclesAsync(Arg.Any(), Arg.Any()); + } + + [Fact] + public async Task CheckDiscrepancies_WhenHashMismatch_DoesNotWaitForRefresh() + { + // Arrange + var expectedResourceType = "Account"; + var requestedHash = "orchestratorHash"; + var staleHash = "staleHash"; + + var job = new ReindexProcessingJobDefinition() + { + MaximumNumberOfResourcesPerQuery = 100, + MaximumNumberOfResourcesPerWrite = 100, + ResourceType = expectedResourceType, + ResourceCount = new SearchResultReindex() + { + Count = _mockedSearchCount, + EndResourceSurrogateId = 0, + StartResourceSurrogateId = 0, + }, + SearchParameterHash = requestedHash, + SearchParameterUrls = new List() { "http://hl7.org/fhir/SearchParam/Account-status" }, + TypeId = (int)JobType.ReindexProcessing, + }; + + _searchParameterOperations.GetSearchParameterHash(Arg.Any()) + .Returns(staleHash); + + var jobInfo = new JobInfo() + { + Id = 1, + Definition = JsonConvert.SerializeObject(job), + QueueType = (byte)QueueType.Reindex, + GroupId = 1, + CreateDate = DateTime.UtcNow, + Status = JobStatus.Running, + }; + + // Act & Assert - Job should fail without trying to self-heal + var exception = await Assert.ThrowsAsync( + async () => await _reindexProcessingJobTaskFactory().ExecuteAsync(jobInfo, _cancellationToken)); + + Assert.Contains($"ResourceType={expectedResourceType} SearchParameterHash: Requested={requestedHash} != Current={staleHash}", exception.Message); + await _searchParameterOperations.DidNotReceive().WaitForRefreshCyclesAsync(Arg.Any(), Arg.Any()); + } + + [Fact] + public async Task CheckDiscrepancies_WhenHashMatches_DoesNotWaitForRefresh() + { + // Arrange + var expectedResourceType = "Account"; + var matchingHash = "matchingHash"; + + var job = new ReindexProcessingJobDefinition() + { + MaximumNumberOfResourcesPerQuery = 100, + MaximumNumberOfResourcesPerWrite = 100, + ResourceType = expectedResourceType, + ResourceCount = new SearchResultReindex() + { + Count = _mockedSearchCount, + EndResourceSurrogateId = 0, + StartResourceSurrogateId = 0, + }, + SearchParameterHash = matchingHash, + SearchParameterUrls = new List() { "http://hl7.org/fhir/SearchParam/Account-status" }, + TypeId = (int)JobType.ReindexProcessing, + }; + + _searchParameterOperations.GetSearchParameterHash(Arg.Any()).Returns(matchingHash); + + var jobInfo = new JobInfo() + { + Id = 1, + Definition = JsonConvert.SerializeObject(job), + QueueType = (byte)QueueType.Reindex, + GroupId = 1, + CreateDate = DateTime.UtcNow, + Status = JobStatus.Running, + }; + + var searchResultEntries = Enumerable.Range(1, _mockedSearchCount) + .Select(i => CreateSearchResultEntry(i.ToString(), expectedResourceType)) + .ToList(); + + _searchService.SearchForReindexAsync( + Arg.Any>>(), + Arg.Any(), + false, + Arg.Any(), + true) + .Returns(new SearchResult( + searchResultEntries, + null, + null, + new List>())); + + // Act + var result = JsonConvert.DeserializeObject( + await _reindexProcessingJobTaskFactory().ExecuteAsync(jobInfo, _cancellationToken)); + + // Assert - Job succeeded and WaitForRefreshCyclesAsync was NOT called + Assert.Equal(_mockedSearchCount, result.SucceededResourceCount); + await _searchParameterOperations.DidNotReceive().WaitForRefreshCyclesAsync(Arg.Any(), Arg.Any()); + } + + [Fact] + public async Task CheckDiscrepancies_WhenSearchParamLastUpdatedIsStale_ThrowsReindexJobException() + { + // Arrange + var expectedResourceType = "Account"; + var matchingHash = "matchingHash"; + var requestedLastUpdated = new DateTimeOffset(2026, 1, 1, 0, 0, 1, TimeSpan.Zero); + var currentLastUpdated = new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero); + + var job = new ReindexProcessingJobDefinition() + { + MaximumNumberOfResourcesPerQuery = 100, + MaximumNumberOfResourcesPerWrite = 100, + ResourceType = expectedResourceType, + ResourceCount = new SearchResultReindex() + { + Count = _mockedSearchCount, + EndResourceSurrogateId = 0, + StartResourceSurrogateId = 0, + }, + SearchParameterHash = matchingHash, + SearchParamLastUpdated = requestedLastUpdated, + SearchParameterUrls = new List() { "http://hl7.org/fhir/SearchParam/Account-status" }, + TypeId = (int)JobType.ReindexProcessing, + }; + + _searchParameterOperations.GetSearchParameterHash(Arg.Any()).Returns(matchingHash); + _searchParameterOperations.SearchParamLastUpdated.Returns(currentLastUpdated); + + var jobInfo = new JobInfo() + { + Id = 1, + Definition = JsonConvert.SerializeObject(job), + QueueType = (byte)QueueType.Reindex, + GroupId = 1, + CreateDate = DateTime.UtcNow, + Status = JobStatus.Running, + }; + + // Act & Assert + var exception = await Assert.ThrowsAsync( + async () => await _reindexProcessingJobTaskFactory().ExecuteAsync(jobInfo, _cancellationToken)); + + Assert.Contains("SearchParamLastUpdated: Requested=2026-01-01 00:00:01.000 > Current=2026-01-01 00:00:00.000", exception.Message); + await _searchParameterOperations.DidNotReceive().WaitForRefreshCyclesAsync(Arg.Any(), Arg.Any()); + } } } diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Operations/SearchParameterState/SearchParameterStateUpdateHandlerTests.cs b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Operations/SearchParameterState/SearchParameterStateUpdateHandlerTests.cs index 5bcb8da28e..e21caf1fdc 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Operations/SearchParameterState/SearchParameterStateUpdateHandlerTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Operations/SearchParameterState/SearchParameterStateUpdateHandlerTests.cs @@ -31,10 +31,11 @@ using Microsoft.Health.Fhir.Core.Models; using Microsoft.Health.Fhir.Core.UnitTests.Extensions; using Microsoft.Health.Fhir.Tests.Common; -using Microsoft.Health.JobManagement; using Microsoft.Health.Test.Utilities; using NSubstitute; using Xunit; + +using FhirJobConflictException = global::Microsoft.Health.Fhir.Core.Features.Operations.JobConflictException; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Shared.Core.UnitTests.Features.Operations.SearchParameterState @@ -64,10 +65,8 @@ public class SearchParameterStateUpdateHandlerTests private ISearchService _searchService = Substitute.For(); private readonly ILogger _logger = Substitute.For>(); private readonly ILogger _logger2 = Substitute.For>(); - private readonly IQueueClient _queueClient = Substitute.For(); private readonly IAuditLogger _auditLogger = Substitute.For(); - private readonly Func> _fhirOperationDataStoreFactory; - private readonly IFhirOperationDataStore _fhirOperationDataStore = Substitute.For(); + private readonly ISearchParameterOperations _searchParameterOperations = Substitute.For(); private readonly ISearchParameterComparer _searchParameterComparer = Substitute.For>(); public SearchParameterStateUpdateHandlerTests() @@ -101,28 +100,20 @@ public SearchParameterStateUpdateHandlerTests() fhirDataStoreProvider, NullLogger.Instance); - _fhirOperationDataStore.CheckActiveReindexJobsAsync(CancellationToken.None).Returns((false, string.Empty)); - - // Create a proper IScope mock - _fhirOperationDataStoreFactory = () => - { - var scope = Substitute.For>(); - scope.Value.Returns(_fhirOperationDataStore); - return scope; - }; - _searchParameterStatusManager = new SearchParameterStatusManager(_searchParameterStatusDataStore, _searchParameterDefinitionManager, _searchParameterSupportResolver, _mediator, _logger); _searchParameterStateUpdateHandler = new SearchParameterStateUpdateHandler( _authorizationService, _searchParameterStatusManager, + _searchParameterOperations, _logger2, - _queueClient, - _auditLogger, - _fhirOperationDataStoreFactory); // Pass the required parameter here + _auditLogger); _cancellationToken = CancellationToken.None; _authorizationService.CheckAccess(DataActions.SearchParameter, _cancellationToken).Returns(DataActions.SearchParameter); + _searchParameterOperations.EnsureNoActiveReindexJobAsync(Arg.Any()).Returns(Task.CompletedTask); + _searchParameterOperations.UpdateSearchParameterStatusAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any()).Returns(Task.CompletedTask); + var searchParamDefinitionStore = new List { new SearchParameterInfo( @@ -249,6 +240,7 @@ public async Task GivenARequestToUpdateSearchParameterStatus_WhenTheStatusIsEnab var statusPart = resourceResponse.Parameter[0].Part.Where(p => p.Name == SearchParameterStateProperties.Status).First(); Assert.True(urlPart.Value.ToString() == ResourceId); Assert.True(statusPart.Value.ToString() == SearchParameterStatus.Supported.ToString()); + await _searchParameterOperations.Received(1).UpdateSearchParameterStatusAsync(Arg.Is>(x => x.Count == 1 && x.First() == ResourceId), SearchParameterStatus.Supported, Arg.Any(), false); } [Fact] @@ -312,6 +304,7 @@ public async Task GivenARequestToUpdateSearchParameterStatus_WhenStatusIsDisable var statusPart = resourceResponse.Parameter[0].Part.Where(p => p.Name == SearchParameterStateProperties.Status).First(); Assert.True(urlPart.Value.ToString() == ResourceId); Assert.True(statusPart.Value.ToString() == SearchParameterStatus.PendingDisable.ToString()); + await _searchParameterOperations.Received(1).UpdateSearchParameterStatusAsync(Arg.Is>(x => x.Count == 1 && x.First() == ResourceId), SearchParameterStatus.PendingDisable, Arg.Any(), false); } [Fact] @@ -320,7 +313,7 @@ public async Task GivenARequestToUpdateSearchParameterStatus_WhenRequestIsValied await _searchParameterDefinitionManager.EnsureInitializedAsync(CancellationToken.None); var loggers = CreateTestAuditLogger(); - var searchParameterStateUpdateHandler = new SearchParameterStateUpdateHandler(_authorizationService, _searchParameterStatusManager, _logger2, _queueClient, loggers.auditLogger, _fhirOperationDataStoreFactory); + var searchParameterStateUpdateHandler = new SearchParameterStateUpdateHandler(_authorizationService, _searchParameterStatusManager, _searchParameterOperations, _logger2, loggers.auditLogger); List> updates = new List>() { new Tuple(new Uri(ResourceId), SearchParameterStatus.Disabled), @@ -342,6 +335,16 @@ public async Task GivenARequestToUpdateSearchParameterStatus_WhenRequestIsValied Assert.Contains("Status=PendingDisable", loggers.logger.LogRecords[0].State.ToString()); } + [Fact] + public async Task GivenReindexRunningBeforeStatusUpdate_WhenHandlingRequest_ThenJobConflictIsThrown() + { + _searchParameterOperations + .When(x => x.EnsureNoActiveReindexJobAsync(Arg.Any())) + .Do(_ => throw new FhirJobConflictException("reindex running")); + + await Assert.ThrowsAsync(() => _searchParameterStateUpdateHandler.Handle(new SearchParameterStateUpdateRequest(new List>()), default)); + } + private (IAuditLogger auditLogger, TestLogger logger) CreateTestAuditLogger() { IOptions optionsConfig = Substitute.For>(); diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterBehaviorTests.cs b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterBehaviorTests.cs index 6e9bc2fa4b..ce29c555ca 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterBehaviorTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterBehaviorTests.cs @@ -3,16 +3,20 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- +using System.Collections.Generic; using System.Net.Http; using System.Threading; using Hl7.Fhir.ElementModel; using Hl7.Fhir.Model; using Hl7.Fhir.Serialization; +using Microsoft.Health.Core.Features.Context; using Microsoft.Health.Fhir.Core.Exceptions; using Microsoft.Health.Fhir.Core.Extensions; +using Microsoft.Health.Fhir.Core.Features.Context; using Microsoft.Health.Fhir.Core.Features.Definition; using Microsoft.Health.Fhir.Core.Features.Persistence; using Microsoft.Health.Fhir.Core.Features.Search.Parameters; +using Microsoft.Health.Fhir.Core.Features.Search.Registry; using Microsoft.Health.Fhir.Core.Messages.Create; using Microsoft.Health.Fhir.Core.Messages.Delete; using Microsoft.Health.Fhir.Core.Messages.Upsert; @@ -35,6 +39,9 @@ public class SearchParameterBehaviorTests private readonly IResourceWrapperFactory _resourceWrapperFactory; private readonly IFhirDataStore _fhirDataStore; private readonly ISearchParameterDefinitionManager _searchParameterDefinitionManager = Substitute.For(); + private readonly ISearchParameterStatusManager _searchParameterStatusManager = Substitute.For(); + private readonly RequestContextAccessor _requestContextAccessor = Substitute.For>(); + private readonly IModelInfoProvider _modelInfoProvider = ModelInfoProvider.Instance; public SearchParameterBehaviorTests() { @@ -48,13 +55,9 @@ public SearchParameterBehaviorTests() Arg.Any()) .Returns(callInfo => { - // Implementation for Create method var resource = callInfo.ArgAt(0); var keepMeta = callInfo.ArgAt(1); - var keepVersion = callInfo.ArgAt(2); - - // Return a mock RawResource - return new RawResource("mock data", FhirResourceFormat.Json, keepMeta); + return new RawResource(new FhirJsonSerializer().SerializeToString(resource.ToPoco()), FhirResourceFormat.Json, keepMeta); }); _resourceWrapperFactory = Substitute.For(); @@ -75,15 +78,14 @@ public async Task GivenACreateResourceRequest_WhenCreatingAResourceOtherThanSear var response = new UpsertResourceResponse(new SaveOutcome(new RawResourceElement(wrapper), SaveOutcomeType.Created)); - var behavior = new CreateOrUpdateSearchParameterBehavior(_searchParameterOperations, _fhirDataStore); + var behavior = new CreateOrUpdateSearchParameterBehavior(_searchParameterOperations, _fhirDataStore, _searchParameterStatusManager, _requestContextAccessor, _modelInfoProvider); await behavior.Handle(request, async (ct) => await Task.Run(() => response), CancellationToken.None); - // Ensure for non-SearchParameter, that we do not call Add SearchParameter - await _searchParameterOperations.DidNotReceive().AddSearchParameterAsync(Arg.Any(), Arg.Any()); + await _searchParameterOperations.DidNotReceive().ValidateSearchParameterAsync(Arg.Any(), Arg.Any()); } [Fact] - public async Task GivenACreateResourceRequest_WhenCreatingASearchParameterResource_ThenAddNewSearchParameterShouldBeCalled() + public async Task GivenACreateResourceRequest_WhenCreatingASearchParameterResource_ThenValidateSearchParameterIsCalled() { var searchParameter = new SearchParameter() { Id = "Id" }; var resource = searchParameter.ToTypedElement().ToResourceElement(); @@ -93,10 +95,10 @@ public async Task GivenACreateResourceRequest_WhenCreatingASearchParameterResour var response = new UpsertResourceResponse(new SaveOutcome(new RawResourceElement(wrapper), SaveOutcomeType.Created)); - var behavior = new CreateOrUpdateSearchParameterBehavior(_searchParameterOperations, _fhirDataStore); + var behavior = new CreateOrUpdateSearchParameterBehavior(_searchParameterOperations, _fhirDataStore, _searchParameterStatusManager, _requestContextAccessor, _modelInfoProvider); await behavior.Handle(request, async (ct) => await Task.Run(() => response), CancellationToken.None); - await _searchParameterOperations.Received().AddSearchParameterAsync(Arg.Any(), Arg.Any()); + await _searchParameterOperations.Received().ValidateSearchParameterAsync(Arg.Any(), Arg.Any()); } [Fact] @@ -112,17 +114,17 @@ public async Task GivenADeleteResourceRequest_WhenDeletingAResourceOtherThanSear var response = new DeleteResourceResponse(key); - var behavior = new DeleteSearchParameterBehavior(_searchParameterOperations, _fhirDataStore, _searchParameterDefinitionManager); + var behavior = new DeleteSearchParameterBehavior(_searchParameterOperations, _fhirDataStore, _searchParameterDefinitionManager, _searchParameterStatusManager, _requestContextAccessor, _modelInfoProvider); await behavior.Handle(request, async (ct) => await Task.Run(() => response), CancellationToken.None); - // Ensure for non-SearchParameter, that we do not call Add SearchParameter - await _searchParameterOperations.DidNotReceive().DeleteSearchParameterAsync(Arg.Any(), Arg.Any()); + await _searchParameterOperations.DidNotReceive().ValidateSearchParameterAsync(Arg.Any(), Arg.Any()); + await _searchParameterStatusManager.DidNotReceive().GetAllSearchParameterStatus(Arg.Any()); } [Fact] - public async Task GivenADeleteResourceRequest_WhenDeletingASearchParameterResource_TheDeleteSearchParameterShouldBeCalled() + public async Task GivenADeleteResourceRequest_WhenDeletingASearchParameterResource_ThenPendingDeleteStatusIsQueued() { - var searchParameter = new SearchParameter() { Id = "Id" }; + var searchParameter = new SearchParameter() { Id = "Id", Url = "http://example.com/Id" }; var resource = searchParameter.ToTypedElement().ToResourceElement(); var key = new ResourceKey("SearchParameter", "Id"); @@ -130,19 +132,31 @@ public async Task GivenADeleteResourceRequest_WhenDeletingASearchParameterResour var wrapper = CreateResourceWrapper(resource, false); _fhirDataStore.GetAsync(key, Arg.Any()).Returns(wrapper); + _searchParameterStatusManager.GetAllSearchParameterStatus(Arg.Any()) + .Returns(new List()); + + var contextProperties = new Dictionary(); + var fhirContext = Substitute.For(); + fhirContext.Properties.Returns(contextProperties); + _requestContextAccessor.RequestContext.Returns(fhirContext); var response = new DeleteResourceResponse(key); - var behavior = new DeleteSearchParameterBehavior(_searchParameterOperations, _fhirDataStore, _searchParameterDefinitionManager); + var behavior = new DeleteSearchParameterBehavior(_searchParameterOperations, _fhirDataStore, _searchParameterDefinitionManager, _searchParameterStatusManager, _requestContextAccessor, _modelInfoProvider); await behavior.Handle(request, async (ct) => await Task.Run(() => response), CancellationToken.None); - await _searchParameterOperations.Received().DeleteSearchParameterAsync(Arg.Any(), Arg.Any()); + await _searchParameterStatusManager.Received().GetAllSearchParameterStatus(Arg.Any()); + Assert.True(contextProperties.ContainsKey(SearchParameterRequestContextPropertyNames.PendingStatusUpdates)); + var pendingStatuses = contextProperties[SearchParameterRequestContextPropertyNames.PendingStatusUpdates] as List; + Assert.Single(pendingStatuses); + Assert.Equal(SearchParameterStatus.PendingDelete, pendingStatuses[0].Status); + Assert.Equal("http://example.com/Id", pendingStatuses[0].Uri.OriginalString); } [Fact] - public async Task GivenADeleteResourceRequest_WhenDeletingAnAlreadyDeletedSearchParameterResource_TheDeleteSearchParameterShouldNotBeCalled() + public async Task GivenADeleteResourceRequest_WhenDeletingAnAlreadyDeletedSearchParameterResource_ThenNoStatusUpdateIsQueued() { - var searchParameter = new SearchParameter() { Id = "Id" }; + var searchParameter = new SearchParameter() { Id = "Id", Url = "http://example.com/Id" }; var resource = searchParameter.ToTypedElement().ToResourceElement(); var key = new ResourceKey("SearchParameter", "Id"); @@ -153,14 +167,30 @@ public async Task GivenADeleteResourceRequest_WhenDeletingAnAlreadyDeletedSearch var response = new DeleteResourceResponse(key); - var behavior = new DeleteSearchParameterBehavior(_searchParameterOperations, _fhirDataStore, _searchParameterDefinitionManager); - await behavior.Handle(request, async (ct) => await Task.Run(() => response), CancellationToken.None); + var behavior = new DeleteSearchParameterBehavior(_searchParameterOperations, _fhirDataStore, _searchParameterDefinitionManager, _searchParameterStatusManager, _requestContextAccessor, _modelInfoProvider); + await behavior.Handle(request, async (ct) => await Task.Run(() => response), CancellationToken.None); - await _searchParameterOperations.DidNotReceive().DeleteSearchParameterAsync(Arg.Any(), Arg.Any()); + await _searchParameterStatusManager.DidNotReceive().GetAllSearchParameterStatus(Arg.Any()); } [Fact] - public async Task GivenADeleteResourceRequest_WhenDeletingASystemDefinedSearchParameterResource_ThenDeleteSearchParameterShouldNotBeCalled() + public async Task GivenADeleteResourceRequest_WhenSearchParameterNotFoundInDataStore_ThenResourceNotFoundExceptionIsThrown() + { + var key = new ResourceKey("SearchParameter", "missing-id"); + var request = new DeleteResourceRequest(key, DeleteOperation.SoftDelete); + + _fhirDataStore.GetAsync(key, Arg.Any()).Returns((ResourceWrapper)null); + + var response = new DeleteResourceResponse(key); + + var behavior = new DeleteSearchParameterBehavior(_searchParameterOperations, _fhirDataStore, _searchParameterDefinitionManager, _searchParameterStatusManager, _requestContextAccessor, _modelInfoProvider); + + await Assert.ThrowsAsync(async () => + await behavior.Handle(request, async (ct) => await Task.Run(() => response), CancellationToken.None)); + } + + [Fact] + public async Task GivenADeleteResourceRequest_WhenDeletingASystemDefinedSearchParameterResource_ThenMethodNotAllowedExceptionIsThrown() { var searchParameter = new SearchParameter() { Id = "system-param", Url = "http://hl7.org/fhir/SearchParameter/system-param" }; var resource = searchParameter.ToTypedElement().ToResourceElement(); @@ -168,7 +198,6 @@ public async Task GivenADeleteResourceRequest_WhenDeletingASystemDefinedSearchPa var key = new ResourceKey("SearchParameter", "system-param"); var request = new DeleteResourceRequest(key, DeleteOperation.SoftDelete); - // Set up a system-defined search parameter in the definition manager var searchParameterInfo = new SearchParameterInfo("system-param", "system-param", Microsoft.Health.Fhir.ValueSets.SearchParamType.String, new System.Uri("http://hl7.org/fhir/SearchParameter/system-param")) { IsSystemDefined = true, @@ -178,18 +207,14 @@ public async Task GivenADeleteResourceRequest_WhenDeletingASystemDefinedSearchPa var response = new DeleteResourceResponse(key); - var behavior = new DeleteSearchParameterBehavior(_searchParameterOperations, _fhirDataStore, _searchParameterDefinitionManager); + var behavior = new DeleteSearchParameterBehavior(_searchParameterOperations, _fhirDataStore, _searchParameterDefinitionManager, _searchParameterStatusManager, _requestContextAccessor, _modelInfoProvider); - // Should throw MethodNotAllowedException for system-defined parameter await Assert.ThrowsAsync(async () => await behavior.Handle(request, async (ct) => await Task.Run(() => response), CancellationToken.None)); - - // Verify DeleteSearchParameterAsync was never called - await _searchParameterOperations.DidNotReceive().DeleteSearchParameterAsync(Arg.Any(), Arg.Any()); } [Fact] - public async Task GivenAnUpsertResourceRequest_WhenSearchParameterDoesNotExist_ThenAddSearchParameterShouldBeCalled() + public async Task GivenAnUpsertResourceRequest_WhenSearchParameterDoesNotExist_ThenValidateSearchParameterIsCalled() { var searchParameter = new SearchParameter() { Id = "NewId", Url = "http://example.com/new-param" }; var resource = searchParameter.ToTypedElement().ToResourceElement(); @@ -198,21 +223,18 @@ public async Task GivenAnUpsertResourceRequest_WhenSearchParameterDoesNotExist_T var request = new UpsertResourceRequest(resource, bundleResourceContext: null); var wrapper = CreateResourceWrapper(resource, false); - // Simulate ResourceNotFoundException when trying to get the previous version _fhirDataStore.GetAsync(key, Arg.Any()).Returns(x => throw new ResourceNotFoundException("Resource not found")); var response = new UpsertResourceResponse(new SaveOutcome(new RawResourceElement(wrapper), SaveOutcomeType.Created)); - var behavior = new CreateOrUpdateSearchParameterBehavior(_searchParameterOperations, _fhirDataStore); + var behavior = new CreateOrUpdateSearchParameterBehavior(_searchParameterOperations, _fhirDataStore, _searchParameterStatusManager, _requestContextAccessor, _modelInfoProvider); await behavior.Handle(request, async (ct) => await Task.Run(() => response), CancellationToken.None); - // Should call AddSearchParameterAsync since the resource doesn't exist - await _searchParameterOperations.Received().AddSearchParameterAsync(Arg.Any(), Arg.Any()); - await _searchParameterOperations.DidNotReceive().UpdateSearchParameterAsync(Arg.Any(), Arg.Any(), Arg.Any()); + await _searchParameterOperations.Received().ValidateSearchParameterAsync(Arg.Any(), Arg.Any()); } [Fact] - public async Task GivenAnUpsertResourceRequest_WhenSearchParameterExists_ThenUpdateSearchParameterShouldBeCalled() + public async Task GivenAnUpsertResourceRequest_WhenSearchParameterExists_ThenValidateSearchParameterIsCalled() { var oldSearchParameter = new SearchParameter() { Id = "ExistingId", Url = "http://example.com/existing-param", Version = "1" }; var newSearchParameter = new SearchParameter() { Id = "ExistingId", Url = "http://example.com/existing-param", Version = "2" }; @@ -225,17 +247,49 @@ public async Task GivenAnUpsertResourceRequest_WhenSearchParameterExists_ThenUpd var oldWrapper = CreateResourceWrapper(oldResource, false); var newWrapper = CreateResourceWrapper(newResource, false); - // Return existing resource when GetAsync is called _fhirDataStore.GetAsync(key, Arg.Any()).Returns(oldWrapper); var response = new UpsertResourceResponse(new SaveOutcome(new RawResourceElement(newWrapper), SaveOutcomeType.Updated)); - var behavior = new CreateOrUpdateSearchParameterBehavior(_searchParameterOperations, _fhirDataStore); + var behavior = new CreateOrUpdateSearchParameterBehavior(_searchParameterOperations, _fhirDataStore, _searchParameterStatusManager, _requestContextAccessor, _modelInfoProvider); + await behavior.Handle(request, async (ct) => await Task.Run(() => response), CancellationToken.None); + + await _searchParameterOperations.Received().ValidateSearchParameterAsync(Arg.Any(), Arg.Any()); + } + + [Fact] + public async Task GivenAnUpsertResourceRequest_WhenSearchParameterUrlChanges_ThenOldUrlIsQueuedAsDeletedAndNewUrlIsQueued() + { + var oldSearchParameter = new SearchParameter() { Id = "ExistingId", Url = "http://example.com/old-url", Version = "1" }; + var newSearchParameter = new SearchParameter() { Id = "ExistingId", Url = "http://example.com/new-url", Version = "2" }; + + var oldResource = oldSearchParameter.ToTypedElement().ToResourceElement(); + var newResource = newSearchParameter.ToTypedElement().ToResourceElement(); + + var key = new ResourceKey("SearchParameter", "ExistingId"); + var request = new UpsertResourceRequest(newResource, bundleResourceContext: null); + var oldWrapper = CreateResourceWrapper(oldResource, false); + var newWrapper = CreateResourceWrapper(newResource, false); + + _fhirDataStore.GetAsync(key, Arg.Any()).Returns(oldWrapper); + _searchParameterStatusManager.GetAllSearchParameterStatus(Arg.Any()) + .Returns(new List()); + + var contextProperties = new Dictionary(); + var fhirContext = Substitute.For(); + fhirContext.Properties.Returns(contextProperties); + _requestContextAccessor.RequestContext.Returns(fhirContext); + + var response = new UpsertResourceResponse(new SaveOutcome(new RawResourceElement(newWrapper), SaveOutcomeType.Updated)); + + var behavior = new CreateOrUpdateSearchParameterBehavior(_searchParameterOperations, _fhirDataStore, _searchParameterStatusManager, _requestContextAccessor, _modelInfoProvider); await behavior.Handle(request, async (ct) => await Task.Run(() => response), CancellationToken.None); - // Should call UpdateSearchParameterAsync since the resource exists - await _searchParameterOperations.DidNotReceive().AddSearchParameterAsync(Arg.Any(), Arg.Any()); - await _searchParameterOperations.Received().UpdateSearchParameterAsync(Arg.Any(), Arg.Any(), Arg.Any()); + var pendingStatuses = contextProperties[SearchParameterRequestContextPropertyNames.PendingStatusUpdates] as List; + Assert.NotNull(pendingStatuses); + Assert.Equal(2, pendingStatuses.Count); + Assert.Contains(pendingStatuses, s => s.Uri.OriginalString == "http://example.com/old-url" && s.Status == SearchParameterStatus.Deleted); + Assert.Contains(pendingStatuses, s => s.Uri.OriginalString == "http://example.com/new-url" && s.Status == SearchParameterStatus.Supported); } private ResourceWrapper CreateResourceWrapper(ResourceElement resource, bool isDeleted) diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterDefinitionManagerTests.cs b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterDefinitionManagerTests.cs index dd6bd9a702..090419761a 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterDefinitionManagerTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterDefinitionManagerTests.cs @@ -148,6 +148,7 @@ public SearchParameterDefinitionManagerTests() var searchParameterDataStoreValidator = Substitute.For(); searchParameterDataStoreValidator.ValidateSearchParameter(Arg.Any(), out Arg.Any()).Returns(true, null); + var fhirOperationDataStore = Substitute.For(); var searchService = Substitute.For(); _searchParameterOperations = new SearchParameterOperations( @@ -156,6 +157,7 @@ public SearchParameterDefinitionManagerTests() ModelInfoProvider.Instance, _searchParameterSupportResolver, searchParameterDataStoreValidator, + () => fhirOperationDataStore.CreateMockScope(), () => searchService.CreateMockScope(), NullLogger.Instance); } @@ -378,7 +380,7 @@ public void GivenASearchParameterDefinitionManager_WhenGettingNonexistentSearchP } [Fact] - public async Task GivenASearchParameterDefinitionManager_WhenGettingSearchParameterHashForExistingResourceType_ThenHashIsReturned() + public void GivenASearchParameterDefinitionManager_WhenGettingSearchParameterHashForExistingResourceType_ThenHashIsReturned() { // Initialize a search parameter var searchParam = new SearchParameter() @@ -395,11 +397,7 @@ public async Task GivenASearchParameterDefinitionManager_WhenGettingSearchParame Code = "test", }; - _searchParameterSupportResolver - .IsSearchParameterSupported(Arg.Is(p => p.Name == "test")) - .Returns((true, false)); - - await _searchParameterOperations.AddSearchParameterAsync(searchParam.ToTypedElement(), CancellationToken.None); + _searchParameterDefinitionManager.AddNewSearchParameters(new List { searchParam.ToTypedElement() }); var searchParamHash = _searchParameterDefinitionManager.GetSearchParameterHashForResourceType("Patient"); Assert.NotNull(searchParamHash); @@ -452,10 +450,11 @@ string CreateDefinition(string reference) .IsSearchParameterSupported(Arg.Is(p => p.Name == "test" && p.Component.All(c => c.ResolvedSearchParameter != null))) .Returns((true, false)); - await _searchParameterOperations.AddSearchParameterAsync(searchParam.ToTypedElement(), CancellationToken.None); - - var addedParam = _searchParameterDefinitionManager.GetSearchParameter("http://test/Patient-test"); - Assert.NotNull(addedParam); + // ValidateSearchParameterAsync only validates; it no longer updates the definition manager cache. + // Component resolution is verified implicitly: the support resolver mock above returns (true, false) + // only when all components have ResolvedSearchParameter set, so an unresolved component would cause + // ValidateSearchParameterAsync to throw SearchParameterNotSupportedException. + await _searchParameterOperations.ValidateSearchParameterAsync(searchParam.ToTypedElement(), CancellationToken.None); } [Fact] @@ -476,7 +475,7 @@ public async Task GivenASPDefinitionManager_WhenInitialed_ThenSearchParametersHa } [Fact] - public async Task GivenASearchParameterDefinitionManager_WhenAddingNewParameter_ThenParameterIsAdded() + public void GivenASearchParameterDefinitionManager_WhenAddingNewParameter_ThenParameterIsAdded() { var patientParams = _searchParameterDefinitionManager.GetSearchParameters("Patient"); var patientParamCount = patientParams.Count(); @@ -496,11 +495,7 @@ public async Task GivenASearchParameterDefinitionManager_WhenAddingNewParameter_ Code = "test", }; - _searchParameterSupportResolver - .IsSearchParameterSupported(Arg.Is(p => p.Name == "test")) - .Returns((true, false)); - - await _searchParameterOperations.AddSearchParameterAsync(searchParam.ToTypedElement(), CancellationToken.None); + _searchParameterDefinitionManager.AddNewSearchParameters(new List { searchParam.ToTypedElement() }); var patientParamsWithNew = _searchParameterDefinitionManager.GetSearchParameters("Patient"); Assert.Equal(patientParamCount + 1, patientParamsWithNew.Count()); diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterValidatorTests.cs b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterValidatorTests.cs index 400998abc3..c1912af0cd 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterValidatorTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterValidatorTests.cs @@ -10,7 +10,6 @@ using Microsoft.Health.Core.Features.Security.Authorization; using Microsoft.Health.Fhir.Core.Exceptions; using Microsoft.Health.Fhir.Core.Features.Definition; -using Microsoft.Health.Fhir.Core.Features.Operations; using Microsoft.Health.Fhir.Core.Features.Search.Parameters; using Microsoft.Health.Fhir.Core.Features.Search.Registry; using Microsoft.Health.Fhir.Core.Features.Security; @@ -31,7 +30,6 @@ namespace Microsoft.Health.Fhir.Core.UnitTests.Features.Search [Trait(Traits.Category, Categories.Search)] public class SearchParameterValidatorTests { - private readonly IFhirOperationDataStore _fhirOperationDataStore = Substitute.For(); private readonly IAuthorizationService _authorizationService = new DisabledFhirAuthorizationService(); private readonly ISearchParameterDefinitionManager _searchParameterDefinitionManager = Substitute.For(); private readonly IModelInfoProvider _modelInfoProvider = MockModelInfoProviderBuilder.Create(FhirSpecification.R4).AddKnownTypes("Patient").Build(); @@ -70,14 +68,14 @@ public SearchParameterValidatorTests() x[3] = searchParameterInfo; return true; }); - _fhirOperationDataStore.CheckActiveReindexJobsAsync(CancellationToken.None).Returns((false, string.Empty)); + _searchParameterOperations.EnsureNoActiveReindexJobAsync(CancellationToken.None).Returns(Task.CompletedTask); } [Theory] [MemberData(nameof(InvalidSearchParamData))] public async Task GivenInvalidSearchParam_WhenValidatingSearchParam_ThenResourceNotValidExceptionThrown(SearchParameter searchParam, string method) { - var validator = new SearchParameterValidator(() => _fhirOperationDataStore.CreateMockScope(), _authorizationService, _searchParameterDefinitionManager, _modelInfoProvider, _searchParameterOperations, _searchParameterComparer, NullLogger.Instance); + var validator = CreateValidator(); await Assert.ThrowsAsync(() => validator.ValidateSearchParameterInput(searchParam, method, CancellationToken.None)); } @@ -85,7 +83,7 @@ public async Task GivenInvalidSearchParam_WhenValidatingSearchParam_ThenResource [MemberData(nameof(ValidSearchParamData))] public async Task GivenValidSearchParam_WhenValidatingSearchParam_ThenNoExceptionThrown(SearchParameter searchParam, string method) { - var validator = new SearchParameterValidator(() => _fhirOperationDataStore.CreateMockScope(), _authorizationService, _searchParameterDefinitionManager, _modelInfoProvider, _searchParameterOperations, _searchParameterComparer, NullLogger.Instance); + var validator = CreateValidator(); await validator.ValidateSearchParameterInput(searchParam, method, CancellationToken.None); } @@ -94,7 +92,7 @@ public async Task GivenUnauthorizedUser_WhenValidatingSearchParam_ThenExceptionT { var authorizationService = Substitute.For>(); authorizationService.CheckAccess(DataActions.Reindex, Arg.Any()).Returns(DataActions.Write); - var validator = new SearchParameterValidator(() => _fhirOperationDataStore.CreateMockScope(), authorizationService, _searchParameterDefinitionManager, _modelInfoProvider, _searchParameterOperations, _searchParameterComparer, NullLogger.Instance); + var validator = new SearchParameterValidator(authorizationService, _searchParameterDefinitionManager, _modelInfoProvider, _searchParameterOperations, _searchParameterComparer, NullLogger.Instance); await Assert.ThrowsAsync(() => validator.ValidateSearchParameterInput(new SearchParameter(), "POST", CancellationToken.None)); } @@ -103,7 +101,7 @@ public async Task GivenUnauthorizedUser_WhenValidatingSearchParam_ThenExceptionT [MemberData(nameof(DuplicateCodeAtBaseResourceData))] public async Task GivenInvalidSearchParamWithDuplicateCode_WhenValidatingSearchParam_ThenResourceNotValidExceptionThrown(SearchParameter searchParam, string method) { - var validator = new SearchParameterValidator(() => _fhirOperationDataStore.CreateMockScope(), _authorizationService, _searchParameterDefinitionManager, _modelInfoProvider, _searchParameterOperations, _searchParameterComparer, NullLogger.Instance); + var validator = CreateValidator(); await Assert.ThrowsAsync(() => validator.ValidateSearchParameterInput(searchParam, method, CancellationToken.None)); } @@ -122,7 +120,7 @@ public async Task GivenValidSearchParamWithDuplicateUrl_WhenValidatingSearchPara return true; }); - var validator = new SearchParameterValidator(() => _fhirOperationDataStore.CreateMockScope(), _authorizationService, _searchParameterDefinitionManager, _modelInfoProvider, _searchParameterOperations, _searchParameterComparer, NullLogger.Instance); + var validator = CreateValidator(); if (searchParameterStatus == SearchParameterStatus.PendingDelete) { // Expecting no exception being thrown. @@ -162,7 +160,6 @@ public async Task GivenSearchParameter_WhenValidatingProperties_ThenConflictingP baseResourceTypes: new List { "DocumentReference" }); var validator = new SearchParameterValidator( - () => _fhirOperationDataStore.CreateMockScope(), _authorizationService, _searchParameterDefinitionManager, _modelInfoProvider, @@ -283,7 +280,7 @@ public async Task GivenSystemDefinedSearchParameter_WhenUpdating_ThenMethodNotAl return true; }); - var validator = new SearchParameterValidator(() => _fhirOperationDataStore.CreateMockScope(), _authorizationService, _searchParameterDefinitionManager, _modelInfoProvider, _searchParameterOperations, _searchParameterComparer, NullLogger.Instance); + var validator = CreateValidator(); // Act & Assert await Assert.ThrowsAsync(() => validator.ValidateSearchParameterInput(searchParam, "PUT", CancellationToken.None)); @@ -314,10 +311,31 @@ public async Task GivenCustomSearchParameter_WhenUpdating_ThenNoExceptionThrown( return true; }); - var validator = new SearchParameterValidator(() => _fhirOperationDataStore.CreateMockScope(), _authorizationService, _searchParameterDefinitionManager, _modelInfoProvider, _searchParameterOperations, _searchParameterComparer, NullLogger.Instance); + var validator = CreateValidator(); // Act - should not throw await validator.ValidateSearchParameterInput(searchParam, "PUT", CancellationToken.None); } + + [Fact] + public async Task GivenActiveReindexJob_WhenValidatingSearchParam_ThenJobConflictExceptionThrown() + { + _searchParameterOperations + .When(x => x.EnsureNoActiveReindexJobAsync(Arg.Any())) + .Do(_ => throw new System.Exception("reindex running")); + + var exception = await Assert.ThrowsAsync(() => CreateValidator().ValidateSearchParameterInput(new SearchParameter { Url = "http://unique" }, "POST", CancellationToken.None)); + + Assert.Equal("reindex running", exception.Message); + } + + private SearchParameterValidator CreateValidator() => + new SearchParameterValidator( + _authorizationService, + _searchParameterDefinitionManager, + _modelInfoProvider, + _searchParameterOperations, + _searchParameterComparer, + NullLogger.Instance); } } diff --git a/src/Microsoft.Health.Fhir.Shared.Core/Features/Operations/Reindex/ReindexSingleResourceRequestHandler.cs b/src/Microsoft.Health.Fhir.Shared.Core/Features/Operations/Reindex/ReindexSingleResourceRequestHandler.cs index 0043ec0827..410d3ce72e 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core/Features/Operations/Reindex/ReindexSingleResourceRequestHandler.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core/Features/Operations/Reindex/ReindexSingleResourceRequestHandler.cs @@ -74,8 +74,6 @@ public async Task Handle(ReindexSingleResourceReq throw new ResourceNotFoundException(string.Format(Core.Resources.ResourceNotFoundById, request.ResourceType, request.ResourceId)); } - await _searchParameterOperations.GetAndApplySearchParameterUpdates(cancellationToken); - // We need to extract the "new" search indices since the assumption is that // a new search parameter has been added to the fhir server. ResourceElement resourceElement = _resourceDeserializer.Deserialize(storedResource); diff --git a/src/Microsoft.Health.Fhir.Shared.Core/Features/Operations/SearchParameterState/SearchParameterStateUpdateHandler.cs b/src/Microsoft.Health.Fhir.Shared.Core/Features/Operations/SearchParameterState/SearchParameterStateUpdateHandler.cs index 70f7336d51..b9e0376bf6 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core/Features/Operations/SearchParameterState/SearchParameterStateUpdateHandler.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core/Features/Operations/SearchParameterState/SearchParameterStateUpdateHandler.cs @@ -15,7 +15,6 @@ using Microsoft.Health.Core; using Microsoft.Health.Core.Features.Audit; using Microsoft.Health.Core.Features.Security.Authorization; -using Microsoft.Health.Extensions.DependencyInjection; using Microsoft.Health.Fhir.Core.Exceptions; using Microsoft.Health.Fhir.Core.Extensions; using Microsoft.Health.Fhir.Core.Features.Audit; @@ -24,7 +23,6 @@ using Microsoft.Health.Fhir.Core.Features.Security; using Microsoft.Health.Fhir.Core.Messages.SearchParameterState; using Microsoft.Health.Fhir.Core.Models; -using Microsoft.Health.JobManagement; using static Hl7.Fhir.Model.Parameters; namespace Microsoft.Health.Fhir.Core.Features.Operations.SearchParameterState @@ -34,27 +32,24 @@ public class SearchParameterStateUpdateHandler : IRequestHandler _authorizationService; private readonly SearchParameterStatusManager _searchParameterStatusManager; + private readonly ISearchParameterOperations _searchParameterOperations; private IReadOnlyCollection _resourceSearchParameterStatus = null; private readonly ILogger _logger; - private readonly IQueueClient _queueClient; private readonly IAuditLogger _auditLogger; - private readonly Func> _fhirOperationDataStoreFactory; - public SearchParameterStateUpdateHandler(IAuthorizationService authorizationService, SearchParameterStatusManager searchParameterStatusManager, ILogger logger, IQueueClient queueClient, IAuditLogger auditLogger, Func> fhirOperationDataStoreFactory) + public SearchParameterStateUpdateHandler(IAuthorizationService authorizationService, SearchParameterStatusManager searchParameterStatusManager, ISearchParameterOperations searchParameterOperations, ILogger logger, IAuditLogger auditLogger) { EnsureArg.IsNotNull(authorizationService, nameof(authorizationService)); EnsureArg.IsNotNull(searchParameterStatusManager, nameof(searchParameterStatusManager)); + EnsureArg.IsNotNull(searchParameterOperations, nameof(searchParameterOperations)); EnsureArg.IsNotNull(logger, nameof(logger)); - EnsureArg.IsNotNull(queueClient, nameof(queueClient)); EnsureArg.IsNotNull(auditLogger, nameof(auditLogger)); - EnsureArg.IsNotNull(fhirOperationDataStoreFactory, nameof(fhirOperationDataStoreFactory)); _authorizationService = authorizationService; _searchParameterStatusManager = searchParameterStatusManager; + _searchParameterOperations = searchParameterOperations; _logger = logger; - _queueClient = queueClient; _auditLogger = auditLogger; - _fhirOperationDataStoreFactory = fhirOperationDataStoreFactory; } public async Task Handle(SearchParameterStateUpdateRequest request, CancellationToken cancellationToken) @@ -66,22 +61,14 @@ public async Task Handle(SearchParameterStat throw new UnauthorizedFhirActionException(); } - if (await IsReindexRunningAsync(cancellationToken)) - { - throw new PreconditionFailedException("A Reindex Job is currently running. Wait till it has completed before trying again."); - } + await _searchParameterOperations.EnsureNoActiveReindexJobAsync(cancellationToken); _resourceSearchParameterStatus = await _searchParameterStatusManager.GetAllSearchParameterStatus(cancellationToken); Dictionary> searchParametersToUpdate = ParseRequestForUpdate(request, out List invalidSearchParameters); - if (await IsReindexRunningAsync(cancellationToken)) - { - return CreateBundleResponse(new Dictionary>() { }, new List() { }, true); - } - foreach (var statusGroup in searchParametersToUpdate) { - await _searchParameterStatusManager.UpdateSearchParameterStatusAsync(statusGroup.Value, statusGroup.Key, cancellationToken); + await _searchParameterOperations.UpdateSearchParameterStatusAsync(statusGroup.Value, statusGroup.Key, cancellationToken); } return CreateBundleResponse(searchParametersToUpdate, invalidSearchParameters); @@ -240,11 +227,5 @@ private SearchParameterStateUpdateResponse CreateBundleResponse(Dictionary IsReindexRunningAsync(CancellationToken cancellationToken) - { - var activeJobs = await _queueClient.GetActiveJobsByQueueTypeAsync((byte)QueueType.Reindex, true, cancellationToken); - return activeJobs.Any(); - } } } diff --git a/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/Parameters/SearchParameterValidator.cs b/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/Parameters/SearchParameterValidator.cs index 6a79822f7b..4f7b28a443 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/Parameters/SearchParameterValidator.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/Parameters/SearchParameterValidator.cs @@ -14,11 +14,9 @@ using Hl7.Fhir.Rest; using Microsoft.Extensions.Logging; using Microsoft.Health.Core.Features.Security.Authorization; -using Microsoft.Health.Extensions.DependencyInjection; using Microsoft.Health.Fhir.Core; using Microsoft.Health.Fhir.Core.Exceptions; using Microsoft.Health.Fhir.Core.Features.Definition; -using Microsoft.Health.Fhir.Core.Features.Operations; using Microsoft.Health.Fhir.Core.Features.Search; using Microsoft.Health.Fhir.Core.Features.Search.Parameters; using Microsoft.Health.Fhir.Core.Features.Search.Registry; @@ -34,7 +32,6 @@ namespace Microsoft.Health.Fhir.Shared.Core.Features.Search.Parameters { public class SearchParameterValidator : ISearchParameterValidator { - private readonly Func> _fhirOperationDataStoreFactory; private readonly IAuthorizationService _authorizationService; private readonly ISearchParameterDefinitionManager _searchParameterDefinitionManager; private readonly IModelInfoProvider _modelInfoProvider; @@ -49,7 +46,6 @@ public class SearchParameterValidator : ISearchParameterValidator private const string HttpPatchName = "PATCH"; public SearchParameterValidator( - Func> fhirOperationDataStoreFactory, IAuthorizationService authorizationService, ISearchParameterDefinitionManager searchParameterDefinitionManager, IModelInfoProvider modelInfoProvider, @@ -57,14 +53,12 @@ public SearchParameterValidator( ISearchParameterComparer searchParameterComparer, ILogger logger) { - EnsureArg.IsNotNull(fhirOperationDataStoreFactory, nameof(fhirOperationDataStoreFactory)); EnsureArg.IsNotNull(authorizationService, nameof(authorizationService)); EnsureArg.IsNotNull(searchParameterDefinitionManager, nameof(searchParameterDefinitionManager)); EnsureArg.IsNotNull(modelInfoProvider, nameof(modelInfoProvider)); EnsureArg.IsNotNull(searchParameterOperations, nameof(searchParameterOperations)); EnsureArg.IsNotNull(searchParameterComparer, nameof(searchParameterComparer)); - _fhirOperationDataStoreFactory = fhirOperationDataStoreFactory; _authorizationService = authorizationService; _searchParameterDefinitionManager = searchParameterDefinitionManager; _modelInfoProvider = modelInfoProvider; @@ -80,15 +74,7 @@ public async Task ValidateSearchParameterInput(SearchParameter searchParam, stri throw new UnauthorizedFhirActionException(); } - // check if reindex job is running - using (IScoped fhirOperationDataStore = _fhirOperationDataStoreFactory()) - { - (var activeReindexJobs, var reindexJobId) = await fhirOperationDataStore.Value.CheckActiveReindexJobsAsync(cancellationToken); - if (activeReindexJobs) - { - throw new JobConflictException(string.Format(Resources.ChangesToSearchParametersNotAllowedWhileReindexing, reindexJobId)); - } - } + await _searchParameterOperations.EnsureNoActiveReindexJobAsync(cancellationToken); if (string.IsNullOrEmpty(searchParam.Url) && (method.Equals(HttpDeleteName, StringComparison.Ordinal) || method.Equals(HttpPatchName, StringComparison.Ordinal))) { diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/109.diff.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/109.diff.sql new file mode 100644 index 0000000000..b734c39720 --- /dev/null +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/109.diff.sql @@ -0,0 +1,165 @@ +IF object_id('UpsertSearchParamsWithOptimisticConcurrency') IS NOT NULL DROP PROCEDURE UpsertSearchParamsWithOptimisticConcurrency +GO +ALTER PROCEDURE dbo.MergeSearchParams @SearchParams dbo.SearchParamList READONLY + ,@IsResourceChangeCaptureEnabled bit = 0 + ,@TransactionId bigint = NULL + ,@Resources dbo.ResourceList READONLY + ,@ResourceWriteClaims dbo.ResourceWriteClaimList READONLY + ,@ReferenceSearchParams dbo.ReferenceSearchParamList READONLY + ,@TokenSearchParams dbo.TokenSearchParamList READONLY + ,@TokenTexts dbo.TokenTextList READONLY + ,@StringSearchParams dbo.StringSearchParamList READONLY + ,@UriSearchParams dbo.UriSearchParamList READONLY + ,@NumberSearchParams dbo.NumberSearchParamList READONLY + ,@QuantitySearchParams dbo.QuantitySearchParamList READONLY + ,@DateTimeSearchParms dbo.DateTimeSearchParamList READONLY + ,@ReferenceTokenCompositeSearchParams dbo.ReferenceTokenCompositeSearchParamList READONLY + ,@TokenTokenCompositeSearchParams dbo.TokenTokenCompositeSearchParamList READONLY + ,@TokenDateTimeCompositeSearchParams dbo.TokenDateTimeCompositeSearchParamList READONLY + ,@TokenQuantityCompositeSearchParams dbo.TokenQuantityCompositeSearchParamList READONLY + ,@TokenStringCompositeSearchParams dbo.TokenStringCompositeSearchParamList READONLY + ,@TokenNumberNumberCompositeSearchParams dbo.TokenNumberNumberCompositeSearchParamList READONLY +AS +set nocount on +DECLARE @SP varchar(100) = object_name(@@procid) + ,@Mode varchar(200) = 'Cnt='+convert(varchar,(SELECT count(*) FROM @SearchParams)) + ,@st datetime = getUTCdate() + ,@LastUpdated datetimeoffset(7) = sysdatetimeoffset() + ,@msg varchar(4000) + ,@Rows int + ,@AffectedRows int = 0 + ,@Uri varchar(4000) + ,@Status varchar(20) + +DECLARE @SearchParamsCopy dbo.SearchParamList +INSERT INTO @SearchParamsCopy SELECT * FROM @SearchParams +WHILE EXISTS (SELECT * FROM @SearchParamsCopy) +BEGIN + SELECT TOP 1 @Uri = Uri, @Status = Status FROM @SearchParamsCopy + SET @msg = 'Uri='+@Uri+' Status='+@Status + EXECUTE dbo.LogEvent @Process=@SP,@Mode=@Mode,@Status='Start',@Text=@msg + DELETE FROM @SearchParamsCopy WHERE Uri = @Uri +END + +DECLARE @SummaryOfChanges TABLE (Uri varchar(128) COLLATE Latin1_General_100_CS_AS NOT NULL, Operation varchar(20) NOT NULL) + +BEGIN TRY + SET TRANSACTION ISOLATION LEVEL SERIALIZABLE + + BEGIN TRANSACTION + + -- Check for concurrency conflicts first using LastUpdated + -- Only the top 60 are included in the message to avoid hitting the 8000 character limit, but all conflicts will cause the transaction to roll back + SELECT TOP 60 @msg = string_agg(S.Uri, ', ') + FROM @SearchParams I JOIN dbo.SearchParam S ON S.Uri = I.Uri + WHERE I.LastUpdated != S.LastUpdated + IF @msg IS NOT NULL + BEGIN + SET @msg = concat('Optimistic concurrency conflict detected for search parameters: ', @msg) + ROLLBACK TRANSACTION; + THROW 50001, @msg, 1 + END + + IF EXISTS (SELECT * FROM @Resources) + BEGIN + EXECUTE dbo.MergeResources + @AffectedRows = @AffectedRows OUTPUT + ,@RaiseExceptionOnConflict = 1 + ,@IsResourceChangeCaptureEnabled = @IsResourceChangeCaptureEnabled + ,@TransactionId = @TransactionId + ,@SingleTransaction = 1 + ,@Resources = @Resources + ,@ResourceWriteClaims = @ResourceWriteClaims + ,@ReferenceSearchParams = @ReferenceSearchParams + ,@TokenSearchParams = @TokenSearchParams + ,@TokenTexts = @TokenTexts + ,@StringSearchParams = @StringSearchParams + ,@UriSearchParams = @UriSearchParams + ,@NumberSearchParams = @NumberSearchParams + ,@QuantitySearchParams = @QuantitySearchParams + ,@DateTimeSearchParms = @DateTimeSearchParms + ,@ReferenceTokenCompositeSearchParams = @ReferenceTokenCompositeSearchParams + ,@TokenTokenCompositeSearchParams = @TokenTokenCompositeSearchParams + ,@TokenDateTimeCompositeSearchParams = @TokenDateTimeCompositeSearchParams + ,@TokenQuantityCompositeSearchParams = @TokenQuantityCompositeSearchParams + ,@TokenStringCompositeSearchParams = @TokenStringCompositeSearchParams + ,@TokenNumberNumberCompositeSearchParams = @TokenNumberNumberCompositeSearchParams; + + SET @Rows = @Rows + @AffectedRows; + END + + MERGE INTO dbo.SearchParam S + USING @SearchParams I ON I.Uri = S.Uri + WHEN MATCHED THEN + UPDATE + SET Status = I.Status + ,LastUpdated = @LastUpdated + ,IsPartiallySupported = I.IsPartiallySupported + WHEN NOT MATCHED BY TARGET THEN + INSERT ( Uri, Status, LastUpdated, IsPartiallySupported) + VALUES (I.Uri, I.Status, @LastUpdated, I.IsPartiallySupported) + OUTPUT I.Uri, $action INTO @SummaryOfChanges; + SET @Rows = @@rowcount + + SELECT S.SearchParamId + ,S.Uri + ,S.LastUpdated + FROM dbo.SearchParam S JOIN @SummaryOfChanges C ON C.Uri = S.Uri + WHERE C.Operation = 'INSERT' + SET @msg = 'LastUpdated='+substring(convert(varchar,@LastUpdated),1,23)+' INSERT='+convert(varchar,@@rowcount) + + COMMIT TRANSACTION + + EXECUTE dbo.LogEvent @Process=@SP,@Mode=@Mode,@Status='End',@Start=@st,@Action='Merge',@Rows=@Rows,@Text=@msg +END TRY +BEGIN CATCH + IF @@trancount > 0 ROLLBACK TRANSACTION; + EXECUTE dbo.LogEvent @Process=@SP,@Mode=@Mode,@Status='Error',@Start=@st; + THROW +END CATCH +GO +INSERT INTO Parameters (Id,Char) SELECT 'MergeSearchParams','LogEvent' WHERE NOT EXISTS (SELECT * FROM dbo.Parameters WHERE Id = 'MergeSearchParams') +GO +-- Enable event logging for DequeueJob to allow active host discovery via EventLog.HostName +INSERT INTO dbo.Parameters (Id, Char) SELECT 'DequeueJob', 'LogEvent' WHERE NOT EXISTS (SELECT * FROM dbo.Parameters WHERE Id = 'DequeueJob') +GO +-- Enable event logging for cache refresh convergence tracking and diagnostics +INSERT INTO dbo.Parameters (Id, Char) SELECT 'SearchParameterCacheRefresh', 'LogEvent' WHERE NOT EXISTS (SELECT * FROM dbo.Parameters WHERE Id = 'SearchParameterCacheRefresh') +GO +CREATE OR ALTER PROCEDURE dbo.CheckSearchParamCacheConsistency + @TargetSearchParamLastUpdated varchar(100) + ,@SyncStartDate datetime2(7) + ,@ActiveHostsSince datetime2(7) + ,@StalenessThresholdMinutes int = 10 +AS +set nocount on +SELECT HostName + ,CAST(NULL AS datetime2(7)) AS SyncEventDate + ,CAST(NULL AS nvarchar(3500)) AS EventText + FROM dbo.EventLog + WHERE EventDate >= @ActiveHostsSince + AND HostName IS NOT NULL + AND Process = 'DequeueJob' + +UNION ALL + +SELECT HostName + ,EventDate + ,EventText + FROM dbo.EventLog + WHERE EventDate >= @SyncStartDate + AND HostName IS NOT NULL + AND Process = 'SearchParameterCacheRefresh' + AND Status = 'End' +GO +--DECLARE @SearchParams dbo.SearchParamList +--INSERT INTO @SearchParams +-- --SELECT 'http://example.org/fhir/SearchParameter/custom-mixed-base-d9e18fc8', 'Enabled', 0, '2026-01-26 17:15:43.0364438 -08:00' +-- SELECT 'Test', 'Enabled', 0, '2026-01-26 17:15:43.0364438 -08:00' +--INSERT INTO @SearchParams +-- SELECT 'Test2', 'Enabled', 0, '2026-01-26 17:15:43.0364438 -08:00' +--SELECT * FROM @SearchParams +--EXECUTE dbo.MergeSearchParams @SearchParams +--SELECT TOP 100 * FROM SearchParam ORDER BY SearchParamId DESC +--DELETE FROM SearchParam WHERE Uri LIKE 'Test%' +--SELECT TOP 10 * FROM EventLog ORDER BY EventDate DESC diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/109.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/109.sql new file mode 100644 index 0000000000..5d0a84638b --- /dev/null +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/109.sql @@ -0,0 +1,6687 @@ + +/************************************************************************************************* + Auto-Generated from Sql build task. Do not manually edit it. +**************************************************************************************************/ +SET XACT_ABORT ON +BEGIN TRAN +IF EXISTS (SELECT * + FROM sys.tables + WHERE name = 'ClaimType') + BEGIN + ROLLBACK; + RETURN; + END + +CREATE PARTITION FUNCTION PartitionFunction_ResourceTypeId(SMALLINT) + AS RANGE RIGHT + FOR VALUES (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150); + +CREATE PARTITION SCHEME PartitionScheme_ResourceTypeId + AS PARTITION PartitionFunction_ResourceTypeId + ALL TO ([PRIMARY]); + + +GO +CREATE PARTITION FUNCTION PartitionFunction_ResourceChangeData_Timestamp(DATETIME2 (7)) + AS RANGE RIGHT + FOR VALUES (N'1970-01-01T00:00:00.0000000'); + +CREATE PARTITION SCHEME PartitionScheme_ResourceChangeData_Timestamp + AS PARTITION PartitionFunction_ResourceChangeData_Timestamp + ALL TO ([PRIMARY]); + +DECLARE @numberOfHistoryPartitions AS INT = 48; + +DECLARE @numberOfFuturePartitions AS INT = 720; + +DECLARE @rightPartitionBoundary AS DATETIME2 (7); + +DECLARE @currentDateTime AS DATETIME2 (7) = sysutcdatetime(); + +WHILE @numberOfHistoryPartitions >= -@numberOfFuturePartitions + BEGIN + SET @rightPartitionBoundary = DATEADD(hour, DATEDIFF(hour, 0, @currentDateTime) - @numberOfHistoryPartitions, 0); + ALTER PARTITION SCHEME PartitionScheme_ResourceChangeData_Timestamp NEXT USED [Primary]; + ALTER PARTITION FUNCTION PartitionFunction_ResourceChangeData_Timestamp( ) + SPLIT RANGE (@rightPartitionBoundary); + SET @numberOfHistoryPartitions -= 1; + END + +CREATE SEQUENCE dbo.ResourceSurrogateIdUniquifierSequence + AS INT + START WITH 0 + INCREMENT BY 1 + MINVALUE 0 + MAXVALUE 79999 + CYCLE + CACHE 1000000; + +CREATE TYPE dbo.BigintList AS TABLE ( + Id BIGINT NOT NULL PRIMARY KEY); + +CREATE TYPE dbo.DateTimeSearchParamList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + StartDateTime DATETIMEOFFSET (7) NOT NULL, + EndDateTime DATETIMEOFFSET (7) NOT NULL, + IsLongerThanADay BIT NOT NULL, + IsMin BIT NOT NULL, + IsMax BIT NOT NULL UNIQUE (ResourceTypeId, ResourceSurrogateId, SearchParamId, StartDateTime, EndDateTime, IsLongerThanADay, IsMin, IsMax)); + +CREATE TYPE dbo.NumberSearchParamList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SingleValue DECIMAL (36, 18) NULL, + LowValue DECIMAL (36, 18) NULL, + HighValue DECIMAL (36, 18) NULL UNIQUE (ResourceTypeId, ResourceSurrogateId, SearchParamId, SingleValue, LowValue, HighValue)); + +CREATE TYPE dbo.QuantitySearchParamList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId INT NULL, + QuantityCodeId INT NULL, + SingleValue DECIMAL (36, 18) NULL, + LowValue DECIMAL (36, 18) NULL, + HighValue DECIMAL (36, 18) NULL UNIQUE (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, QuantityCodeId, SingleValue, LowValue, HighValue)); + +CREATE TYPE dbo.ReferenceSearchParamList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + BaseUri VARCHAR (128) COLLATE Latin1_General_100_CS_AS NULL, + ReferenceResourceTypeId SMALLINT NULL, + ReferenceResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + ReferenceResourceVersion INT NULL UNIQUE (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri, ReferenceResourceTypeId, ReferenceResourceId)); + +CREATE TYPE dbo.ReferenceTokenCompositeSearchParamList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + BaseUri1 VARCHAR (128) COLLATE Latin1_General_100_CS_AS NULL, + ReferenceResourceTypeId1 SMALLINT NULL, + ReferenceResourceId1 VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + ReferenceResourceVersion1 INT NULL, + SystemId2 INT NULL, + Code2 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow2 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL); + +CREATE TYPE dbo.ResourceDateKeyList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + ResourceSurrogateId BIGINT NOT NULL PRIMARY KEY (ResourceTypeId, ResourceId, ResourceSurrogateId)); + +CREATE TYPE dbo.ResourceKeyList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + Version INT NULL UNIQUE (ResourceTypeId, ResourceId, Version)); + +CREATE TYPE dbo.ResourceList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + ResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + Version INT NOT NULL, + HasVersionToCompare BIT NOT NULL, + IsDeleted BIT NOT NULL, + IsHistory BIT NOT NULL, + KeepHistory BIT NOT NULL, + RawResource VARBINARY (MAX) NOT NULL, + IsRawResourceMetaSet BIT NOT NULL, + RequestMethod VARCHAR (10) NULL, + SearchParamHash VARCHAR (64) NULL PRIMARY KEY (ResourceTypeId, ResourceSurrogateId), + UNIQUE (ResourceTypeId, ResourceId, Version)); + +CREATE TYPE dbo.ResourceWriteClaimList AS TABLE ( + ResourceSurrogateId BIGINT NOT NULL, + ClaimTypeId TINYINT NOT NULL, + ClaimValue NVARCHAR (128) NOT NULL); + +CREATE TYPE dbo.SearchParamList AS TABLE ( + Uri VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL, + Status VARCHAR (20) NOT NULL, + IsPartiallySupported BIT NOT NULL, + LastUpdated DATETIMEOFFSET (7) NOT NULL UNIQUE (Uri)); + +CREATE TYPE dbo.StringList AS TABLE ( + String VARCHAR (MAX)); + +CREATE TYPE dbo.StringSearchParamList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + Text NVARCHAR (256) COLLATE Latin1_General_100_CI_AI_SC NOT NULL, + TextOverflow NVARCHAR (MAX) COLLATE Latin1_General_100_CI_AI_SC NULL, + IsMin BIT NOT NULL, + IsMax BIT NOT NULL); + +CREATE TYPE dbo.TokenDateTimeCompositeSearchParamList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL, + StartDateTime2 DATETIMEOFFSET (7) NOT NULL, + EndDateTime2 DATETIMEOFFSET (7) NOT NULL, + IsLongerThanADay2 BIT NOT NULL); + +CREATE TYPE dbo.TokenList AS TABLE ( + Code VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL, + SystemId INT NULL, + SystemValue NVARCHAR (256) NULL); + +CREATE TYPE dbo.TokenNumberNumberCompositeSearchParamList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL, + SingleValue2 DECIMAL (36, 18) NULL, + LowValue2 DECIMAL (36, 18) NULL, + HighValue2 DECIMAL (36, 18) NULL, + SingleValue3 DECIMAL (36, 18) NULL, + LowValue3 DECIMAL (36, 18) NULL, + HighValue3 DECIMAL (36, 18) NULL, + HasRange BIT NOT NULL); + +CREATE TYPE dbo.TokenQuantityCompositeSearchParamList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL, + SystemId2 INT NULL, + QuantityCodeId2 INT NULL, + SingleValue2 DECIMAL (36, 18) NULL, + LowValue2 DECIMAL (36, 18) NULL, + HighValue2 DECIMAL (36, 18) NULL); + +CREATE TYPE dbo.TokenSearchParamList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId INT NULL, + Code VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL); + +CREATE TYPE dbo.TokenStringCompositeSearchParamList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL, + Text2 NVARCHAR (256) COLLATE Latin1_General_100_CI_AI_SC NOT NULL, + TextOverflow2 NVARCHAR (MAX) COLLATE Latin1_General_100_CI_AI_SC NULL); + +CREATE TYPE dbo.TokenTextList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + Text NVARCHAR (400) COLLATE Latin1_General_CI_AI NOT NULL); + +CREATE TYPE dbo.TokenTokenCompositeSearchParamList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL, + SystemId2 INT NULL, + Code2 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow2 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL); + +CREATE TYPE dbo.UriSearchParamList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + Uri VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL PRIMARY KEY (ResourceTypeId, ResourceSurrogateId, SearchParamId, Uri)); + +CREATE TABLE dbo.ClaimType ( + ClaimTypeId TINYINT IDENTITY (1, 1) NOT NULL, + Name VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL, + CONSTRAINT UQ_ClaimType_ClaimTypeId UNIQUE (ClaimTypeId), + CONSTRAINT PKC_ClaimType PRIMARY KEY CLUSTERED (Name) WITH (DATA_COMPRESSION = PAGE) +); + +CREATE TABLE dbo.CompartmentAssignment ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + CompartmentTypeId TINYINT NOT NULL, + ReferenceResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + IsHistory BIT NOT NULL, + CONSTRAINT PKC_CompartmentAssignment PRIMARY KEY CLUSTERED (ResourceTypeId, ResourceSurrogateId, CompartmentTypeId, ReferenceResourceId) WITH (DATA_COMPRESSION = PAGE) ON PartitionScheme_ResourceTypeId (ResourceTypeId) +); + + +GO +ALTER TABLE dbo.CompartmentAssignment + ADD CONSTRAINT DF_CompartmentAssignment_IsHistory DEFAULT 0 FOR IsHistory; + + +GO +ALTER TABLE dbo.CompartmentAssignment SET (LOCK_ESCALATION = AUTO); + + +GO +CREATE NONCLUSTERED INDEX IX_CompartmentAssignment_CompartmentTypeId_ReferenceResourceId + ON dbo.CompartmentAssignment(ResourceTypeId, CompartmentTypeId, ReferenceResourceId, ResourceSurrogateId) WHERE IsHistory = 0 WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.CompartmentType ( + CompartmentTypeId TINYINT IDENTITY (1, 1) NOT NULL, + Name VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL, + CONSTRAINT UQ_CompartmentType_CompartmentTypeId UNIQUE (CompartmentTypeId), + CONSTRAINT PKC_CompartmentType PRIMARY KEY CLUSTERED (Name) WITH (DATA_COMPRESSION = PAGE) +); + +CREATE TABLE dbo.DateTimeSearchParam ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + StartDateTime DATETIME2 (7) NOT NULL, + EndDateTime DATETIME2 (7) NOT NULL, + IsLongerThanADay BIT NOT NULL, + IsMin BIT CONSTRAINT date_IsMin_Constraint DEFAULT 0 NOT NULL, + IsMax BIT CONSTRAINT date_IsMax_Constraint DEFAULT 0 NOT NULL +); + +ALTER TABLE dbo.DateTimeSearchParam SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_DateTimeSearchParam + ON dbo.DateTimeSearchParam(ResourceTypeId, ResourceSurrogateId, SearchParamId) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE INDEX IX_SearchParamId_StartDateTime_EndDateTime_INCLUDE_IsLongerThanADay_IsMin_IsMax + ON dbo.DateTimeSearchParam(SearchParamId, StartDateTime, EndDateTime) + INCLUDE(IsLongerThanADay, IsMin, IsMax) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE INDEX IX_SearchParamId_EndDateTime_StartDateTime_INCLUDE_IsLongerThanADay_IsMin_IsMax + ON dbo.DateTimeSearchParam(SearchParamId, EndDateTime, StartDateTime) + INCLUDE(IsLongerThanADay, IsMin, IsMax) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE INDEX IX_SearchParamId_StartDateTime_EndDateTime_INCLUDE_IsMin_IsMax_WHERE_IsLongerThanADay_1 + ON dbo.DateTimeSearchParam(SearchParamId, StartDateTime, EndDateTime) + INCLUDE(IsMin, IsMax) WHERE IsLongerThanADay = 1 + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE INDEX IX_SearchParamId_EndDateTime_StartDateTime_INCLUDE_IsMin_IsMax_WHERE_IsLongerThanADay_1 + ON dbo.DateTimeSearchParam(SearchParamId, EndDateTime, StartDateTime) + INCLUDE(IsMin, IsMax) WHERE IsLongerThanADay = 1 + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +IF NOT EXISTS (SELECT 1 + FROM sys.tables + WHERE name = 'EventAgentCheckpoint') + BEGIN + CREATE TABLE dbo.EventAgentCheckpoint ( + CheckpointId VARCHAR (64) NOT NULL, + LastProcessedDateTime DATETIMEOFFSET (7), + LastProcessedIdentifier VARCHAR (64) , + UpdatedOn DATETIME2 (7) DEFAULT sysutcdatetime() NOT NULL, + CONSTRAINT PK_EventAgentCheckpoint PRIMARY KEY CLUSTERED (CheckpointId) + ) ON [PRIMARY]; + END + +CREATE PARTITION FUNCTION EventLogPartitionFunction(TINYINT) + AS RANGE RIGHT + FOR VALUES (0, 1, 2, 3, 4, 5, 6, 7); + + +GO +CREATE PARTITION SCHEME EventLogPartitionScheme + AS PARTITION EventLogPartitionFunction + ALL TO ([PRIMARY]); + + +GO +CREATE TABLE dbo.EventLog ( + PartitionId AS isnull(CONVERT (TINYINT, EventId % 8), 0) PERSISTED, + EventId BIGINT IDENTITY (1, 1) NOT NULL, + EventDate DATETIME NOT NULL, + Process VARCHAR (100) NOT NULL, + Status VARCHAR (10) NOT NULL, + Mode VARCHAR (200) NULL, + Action VARCHAR (20) NULL, + Target VARCHAR (100) NULL, + Rows BIGINT NULL, + Milliseconds INT NULL, + EventText NVARCHAR (3500) NULL, + SPID SMALLINT NOT NULL, + HostName VARCHAR (64) NOT NULL CONSTRAINT PKC_EventLog_EventDate_EventId_PartitionId PRIMARY KEY CLUSTERED (EventDate, EventId, PartitionId) ON EventLogPartitionScheme (PartitionId) +); + +CREATE TABLE dbo.IndexProperties ( + TableName VARCHAR (100) NOT NULL, + IndexName VARCHAR (200) NOT NULL, + PropertyName VARCHAR (100) NOT NULL, + PropertyValue VARCHAR (100) NOT NULL, + CreateDate DATETIME CONSTRAINT DF_IndexProperties_CreateDate DEFAULT getUTCdate() NOT NULL CONSTRAINT PKC_IndexProperties_TableName_IndexName_PropertyName PRIMARY KEY CLUSTERED (TableName, IndexName, PropertyName) +); + +CREATE PARTITION FUNCTION TinyintPartitionFunction(TINYINT) + AS RANGE RIGHT + FOR VALUES (0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255); + + +GO +CREATE PARTITION SCHEME TinyintPartitionScheme + AS PARTITION TinyintPartitionFunction + ALL TO ([PRIMARY]); + + +GO +CREATE TABLE dbo.JobQueue ( + QueueType TINYINT NOT NULL, + GroupId BIGINT NOT NULL, + JobId BIGINT NOT NULL, + PartitionId AS CONVERT (TINYINT, JobId % 16) PERSISTED, + Definition VARCHAR (MAX) NOT NULL, + DefinitionHash VARBINARY (20) NOT NULL, + Version BIGINT CONSTRAINT DF_JobQueue_Version DEFAULT datediff_big(millisecond, '0001-01-01', getUTCdate()) NOT NULL, + Status TINYINT CONSTRAINT DF_JobQueue_Status DEFAULT 0 NOT NULL, + Priority TINYINT CONSTRAINT DF_JobQueue_Priority DEFAULT 100 NOT NULL, + Data BIGINT NULL, + Result VARCHAR (MAX) NULL, + CreateDate DATETIME CONSTRAINT DF_JobQueue_CreateDate DEFAULT getUTCdate() NOT NULL, + StartDate DATETIME NULL, + EndDate DATETIME NULL, + HeartbeatDate DATETIME CONSTRAINT DF_JobQueue_HeartbeatDate DEFAULT getUTCdate() NOT NULL, + Worker VARCHAR (100) NULL, + Info VARCHAR (1000) NULL, + CancelRequested BIT CONSTRAINT DF_JobQueue_CancelRequested DEFAULT 0 NOT NULL CONSTRAINT PKC_JobQueue_QueueType_PartitionId_JobId PRIMARY KEY CLUSTERED (QueueType, PartitionId, JobId) ON TinyintPartitionScheme (QueueType), + CONSTRAINT U_JobQueue_QueueType_JobId UNIQUE (QueueType, JobId) +); + + +GO +CREATE INDEX IX_QueueType_PartitionId_Status_Priority + ON dbo.JobQueue(PartitionId, Status, Priority) + ON TinyintPartitionScheme (QueueType); + + +GO +CREATE INDEX IX_QueueType_GroupId + ON dbo.JobQueue(QueueType, GroupId) + ON TinyintPartitionScheme (QueueType); + + +GO +CREATE INDEX IX_QueueType_DefinitionHash + ON dbo.JobQueue(QueueType, DefinitionHash) + ON TinyintPartitionScheme (QueueType); + +CREATE TABLE dbo.NumberSearchParam ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SingleValue DECIMAL (36, 18) NULL, + LowValue DECIMAL (36, 18) NOT NULL, + HighValue DECIMAL (36, 18) NOT NULL +); + +ALTER TABLE dbo.NumberSearchParam SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_NumberSearchParam + ON dbo.NumberSearchParam(ResourceTypeId, ResourceSurrogateId, SearchParamId) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE INDEX IX_SearchParamId_SingleValue_WHERE_SingleValue_NOT_NULL + ON dbo.NumberSearchParam(SearchParamId, SingleValue) WHERE SingleValue IS NOT NULL + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE INDEX IX_SearchParamId_LowValue_HighValue + ON dbo.NumberSearchParam(SearchParamId, LowValue, HighValue) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE INDEX IX_SearchParamId_HighValue_LowValue + ON dbo.NumberSearchParam(SearchParamId, HighValue, LowValue) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.Parameters ( + Id VARCHAR (100) NOT NULL, + Date DATETIME NULL, + Number FLOAT NULL, + Bigint BIGINT NULL, + Char VARCHAR (4000) NULL, + Binary VARBINARY (MAX) NULL, + UpdatedDate DATETIME NULL, + UpdatedBy NVARCHAR (255) NULL CONSTRAINT PKC_Parameters_Id PRIMARY KEY CLUSTERED (Id) WITH (IGNORE_DUP_KEY = ON) +); + + +GO +CREATE TABLE dbo.ParametersHistory ( + ChangeId INT IDENTITY (1, 1) NOT NULL, + Id VARCHAR (100) NOT NULL, + Date DATETIME NULL, + Number FLOAT NULL, + Bigint BIGINT NULL, + Char VARCHAR (4000) NULL, + Binary VARBINARY (MAX) NULL, + UpdatedDate DATETIME NULL, + UpdatedBy NVARCHAR (255) NULL +); + +CREATE TABLE dbo.QuantityCode ( + QuantityCodeId INT IDENTITY (1, 1) NOT NULL, + Value NVARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CONSTRAINT UQ_QuantityCode_QuantityCodeId UNIQUE (QuantityCodeId), + CONSTRAINT PKC_QuantityCode PRIMARY KEY CLUSTERED (Value) WITH (DATA_COMPRESSION = PAGE) +); + +CREATE TABLE dbo.QuantitySearchParam ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId INT NULL, + QuantityCodeId INT NULL, + SingleValue DECIMAL (36, 18) NULL, + LowValue DECIMAL (36, 18) NOT NULL, + HighValue DECIMAL (36, 18) NOT NULL +); + +ALTER TABLE dbo.QuantitySearchParam SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_QuantitySearchParam + ON dbo.QuantitySearchParam(ResourceTypeId, ResourceSurrogateId, SearchParamId) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE INDEX IX_SearchParamId_QuantityCodeId_SingleValue_INCLUDE_SystemId_WHERE_SingleValue_NOT_NULL + ON dbo.QuantitySearchParam(SearchParamId, QuantityCodeId, SingleValue) + INCLUDE(SystemId) WHERE SingleValue IS NOT NULL + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE INDEX IX_SearchParamId_QuantityCodeId_LowValue_HighValue_INCLUDE_SystemId + ON dbo.QuantitySearchParam(SearchParamId, QuantityCodeId, LowValue, HighValue) + INCLUDE(SystemId) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE INDEX IX_SearchParamId_QuantityCodeId_HighValue_LowValue_INCLUDE_SystemId + ON dbo.QuantitySearchParam(SearchParamId, QuantityCodeId, HighValue, LowValue) + INCLUDE(SystemId) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.ReferenceSearchParam ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + BaseUri VARCHAR (128) COLLATE Latin1_General_100_CS_AS NULL, + ReferenceResourceTypeId SMALLINT NULL, + ReferenceResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + ReferenceResourceVersion INT NULL +); + +ALTER TABLE dbo.ReferenceSearchParam SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_ReferenceSearchParam + ON dbo.ReferenceSearchParam(ResourceTypeId, ResourceSurrogateId, SearchParamId) WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE UNIQUE INDEX IXU_ReferenceResourceId_ReferenceResourceTypeId_SearchParamId_BaseUri_ResourceSurrogateId_ResourceTypeId + ON dbo.ReferenceSearchParam(ReferenceResourceId, ReferenceResourceTypeId, SearchParamId, BaseUri, ResourceSurrogateId, ResourceTypeId) WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.ReferenceTokenCompositeSearchParam ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + BaseUri1 VARCHAR (128) COLLATE Latin1_General_100_CS_AS NULL, + ReferenceResourceTypeId1 SMALLINT NULL, + ReferenceResourceId1 VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + ReferenceResourceVersion1 INT NULL, + SystemId2 INT NULL, + Code2 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow2 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL +); + +ALTER TABLE dbo.ReferenceTokenCompositeSearchParam SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_ReferenceTokenCompositeSearchParam + ON dbo.ReferenceTokenCompositeSearchParam(ResourceTypeId, ResourceSurrogateId, SearchParamId) WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE INDEX IX_SearchParamId_ReferenceResourceId1_Code2_INCLUDE_ReferenceResourceTypeId1_BaseUri1_SystemId2 + ON dbo.ReferenceTokenCompositeSearchParam(SearchParamId, ReferenceResourceId1, Code2) + INCLUDE(ReferenceResourceTypeId1, BaseUri1, SystemId2) WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.ReindexJob ( + Id VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + Status VARCHAR (10) NOT NULL, + HeartbeatDateTime DATETIME2 (7) NULL, + RawJobRecord VARCHAR (MAX) NOT NULL, + JobVersion ROWVERSION NOT NULL, + CONSTRAINT PKC_ReindexJob PRIMARY KEY CLUSTERED (Id) +); + +CREATE TABLE dbo.CurrentResource ( + ResourceTypeId SMALLINT NOT NULL, + ResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + Version INT NOT NULL, + IsHistory BIT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + IsDeleted BIT NOT NULL, + RequestMethod VARCHAR (10) NULL, + RawResource VARBINARY (MAX) NOT NULL, + IsRawResourceMetaSet BIT NOT NULL, + SearchParamHash VARCHAR (64) NULL, + TransactionId BIGINT NULL, + HistoryTransactionId BIGINT NULL +); + + +GO +DROP TABLE dbo.CurrentResource; + + +GO +CREATE TABLE dbo.Resource ( + ResourceTypeId SMALLINT NOT NULL, + ResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + Version INT NOT NULL, + IsHistory BIT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + IsDeleted BIT NOT NULL, + RequestMethod VARCHAR (10) NULL, + RawResource VARBINARY (MAX) NOT NULL, + IsRawResourceMetaSet BIT DEFAULT 0 NOT NULL, + SearchParamHash VARCHAR (64) NULL, + TransactionId BIGINT NULL, + HistoryTransactionId BIGINT NULL CONSTRAINT PKC_Resource PRIMARY KEY CLUSTERED (ResourceTypeId, ResourceSurrogateId) WITH (DATA_COMPRESSION = PAGE) ON PartitionScheme_ResourceTypeId (ResourceTypeId), + CONSTRAINT CH_Resource_RawResource_Length CHECK (RawResource > 0x0) +); + +ALTER TABLE dbo.Resource SET (LOCK_ESCALATION = AUTO); + +CREATE INDEX IX_ResourceTypeId_TransactionId + ON dbo.Resource(ResourceTypeId, TransactionId) WHERE TransactionId IS NOT NULL + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE INDEX IX_ResourceTypeId_HistoryTransactionId + ON dbo.Resource(ResourceTypeId, HistoryTransactionId) WHERE HistoryTransactionId IS NOT NULL + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE UNIQUE NONCLUSTERED INDEX IX_Resource_ResourceTypeId_ResourceId_Version + ON dbo.Resource(ResourceTypeId, ResourceId, Version) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE UNIQUE NONCLUSTERED INDEX IX_Resource_ResourceTypeId_ResourceId + ON dbo.Resource(ResourceTypeId, ResourceId) + INCLUDE(Version, IsDeleted) WHERE IsHistory = 0 + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE UNIQUE NONCLUSTERED INDEX IX_Resource_ResourceTypeId_ResourceSurrgateId + ON dbo.Resource(ResourceTypeId, ResourceSurrogateId) WHERE IsHistory = 0 + AND IsDeleted = 0 + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.ResourceChangeData ( + Id BIGINT IDENTITY (1, 1) NOT NULL, + Timestamp DATETIME2 (7) CONSTRAINT DF_ResourceChangeData_Timestamp DEFAULT sysutcdatetime() NOT NULL, + ResourceId VARCHAR (64) NOT NULL, + ResourceTypeId SMALLINT NOT NULL, + ResourceVersion INT NOT NULL, + ResourceChangeTypeId TINYINT NOT NULL +) ON PartitionScheme_ResourceChangeData_Timestamp (Timestamp); + +CREATE CLUSTERED INDEX IXC_ResourceChangeData + ON dbo.ResourceChangeData(Id ASC) WITH (ONLINE = ON) + ON PartitionScheme_ResourceChangeData_Timestamp (Timestamp); + +CREATE TABLE dbo.ResourceChangeDataStaging ( + Id BIGINT IDENTITY (1, 1) NOT NULL, + Timestamp DATETIME2 (7) CONSTRAINT DF_ResourceChangeDataStaging_Timestamp DEFAULT sysutcdatetime() NOT NULL, + ResourceId VARCHAR (64) NOT NULL, + ResourceTypeId SMALLINT NOT NULL, + ResourceVersion INT NOT NULL, + ResourceChangeTypeId TINYINT NOT NULL +) ON [PRIMARY]; + +CREATE CLUSTERED INDEX IXC_ResourceChangeDataStaging + ON dbo.ResourceChangeDataStaging(Id ASC, Timestamp ASC) WITH (ONLINE = ON) + ON [PRIMARY]; + +ALTER TABLE dbo.ResourceChangeDataStaging WITH CHECK + ADD CONSTRAINT CHK_ResourceChangeDataStaging_partition CHECK (Timestamp < CONVERT (DATETIME2 (7), N'9999-12-31 23:59:59.9999999')); + +ALTER TABLE dbo.ResourceChangeDataStaging CHECK CONSTRAINT CHK_ResourceChangeDataStaging_partition; + +CREATE TABLE dbo.ResourceChangeType ( + ResourceChangeTypeId TINYINT NOT NULL, + Name NVARCHAR (50) NOT NULL, + CONSTRAINT PK_ResourceChangeType PRIMARY KEY CLUSTERED (ResourceChangeTypeId), + CONSTRAINT UQ_ResourceChangeType_Name UNIQUE NONCLUSTERED (Name) +) ON [PRIMARY]; + + +GO +INSERT dbo.ResourceChangeType (ResourceChangeTypeId, Name) +VALUES (0, N'Creation'); + +INSERT dbo.ResourceChangeType (ResourceChangeTypeId, Name) +VALUES (1, N'Update'); + +INSERT dbo.ResourceChangeType (ResourceChangeTypeId, Name) +VALUES (2, N'Deletion'); + +CREATE TABLE dbo.ResourceType ( + ResourceTypeId SMALLINT IDENTITY (1, 1) NOT NULL, + Name NVARCHAR (50) COLLATE Latin1_General_100_CS_AS NOT NULL, + CONSTRAINT UQ_ResourceType_ResourceTypeId UNIQUE (ResourceTypeId), + CONSTRAINT PKC_ResourceType PRIMARY KEY CLUSTERED (Name) WITH (DATA_COMPRESSION = PAGE) +); + +CREATE TABLE dbo.ResourceWriteClaim ( + ResourceSurrogateId BIGINT NOT NULL, + ClaimTypeId TINYINT NOT NULL, + ClaimValue NVARCHAR (128) NOT NULL +) +WITH (DATA_COMPRESSION = PAGE); + +CREATE CLUSTERED INDEX IXC_ResourceWriteClaim + ON dbo.ResourceWriteClaim(ResourceSurrogateId, ClaimTypeId); + +CREATE TABLE dbo.SchemaMigrationProgress ( + Timestamp DATETIME2 (3) DEFAULT CURRENT_TIMESTAMP, + Message NVARCHAR (MAX) +); + +CREATE TABLE dbo.SearchParam ( + SearchParamId SMALLINT IDENTITY (1, 1) NOT NULL, + Uri VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL, + Status VARCHAR (20) NOT NULL, + LastUpdated DATETIMEOFFSET (7) NOT NULL, + IsPartiallySupported BIT NOT NULL, + CONSTRAINT UQ_SearchParam_SearchParamId UNIQUE (SearchParamId), + CONSTRAINT PKC_SearchParam PRIMARY KEY CLUSTERED (Uri) WITH (DATA_COMPRESSION = PAGE) +); + +CREATE NONCLUSTERED INDEX IX_LastUpdated + ON dbo.SearchParam(LastUpdated); + +CREATE TABLE dbo.StringSearchParam ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + Text NVARCHAR (256) COLLATE Latin1_General_100_CI_AI_SC NOT NULL, + TextOverflow NVARCHAR (MAX) COLLATE Latin1_General_100_CI_AI_SC NULL, + IsMin BIT CONSTRAINT string_IsMin_Constraint DEFAULT 0 NOT NULL, + IsMax BIT CONSTRAINT string_IsMax_Constraint DEFAULT 0 NOT NULL +); + +ALTER TABLE dbo.StringSearchParam SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_StringSearchParam + ON dbo.StringSearchParam(ResourceTypeId, ResourceSurrogateId, SearchParamId) WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE INDEX IX_SearchParamId_Text_INCLUDE_TextOverflow_IsMin_IsMax + ON dbo.StringSearchParam(SearchParamId, Text) + INCLUDE(TextOverflow, IsMin, IsMax) WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE INDEX IX_SearchParamId_Text_INCLUDE_IsMin_IsMax_WHERE_TextOverflow_NOT_NULL + ON dbo.StringSearchParam(SearchParamId, Text) + INCLUDE(IsMin, IsMax) WHERE TextOverflow IS NOT NULL WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.System ( + SystemId INT IDENTITY (1, 1) NOT NULL, + Value NVARCHAR (256) NOT NULL, + CONSTRAINT UQ_System_SystemId UNIQUE (SystemId), + CONSTRAINT PKC_System PRIMARY KEY CLUSTERED (Value) WITH (DATA_COMPRESSION = PAGE) +); + +CREATE TABLE [dbo].[TaskInfo] ( + [TaskId] VARCHAR (64) NOT NULL, + [QueueId] VARCHAR (64) NOT NULL, + [Status] SMALLINT NOT NULL, + [TaskTypeId] SMALLINT NOT NULL, + [RunId] VARCHAR (50) NULL, + [IsCanceled] BIT NOT NULL, + [RetryCount] SMALLINT NOT NULL, + [MaxRetryCount] SMALLINT NOT NULL, + [HeartbeatDateTime] DATETIME2 (7) NULL, + [InputData] VARCHAR (MAX) NOT NULL, + [TaskContext] VARCHAR (MAX) NULL, + [Result] VARCHAR (MAX) NULL, + [CreateDateTime] DATETIME2 (7) CONSTRAINT DF_TaskInfo_CreateDate DEFAULT SYSUTCDATETIME() NOT NULL, + [StartDateTime] DATETIME2 (7) NULL, + [EndDateTime] DATETIME2 (7) NULL, + [Worker] VARCHAR (100) NULL, + [RestartInfo] VARCHAR (MAX) NULL, + [ParentTaskId] VARCHAR (64) NULL, + CONSTRAINT PKC_TaskInfo PRIMARY KEY CLUSTERED (TaskId) WITH (DATA_COMPRESSION = PAGE) +) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY]; + + +GO +CREATE NONCLUSTERED INDEX IX_QueueId_Status + ON dbo.TaskInfo(QueueId, Status); + + +GO +CREATE NONCLUSTERED INDEX IX_QueueId_ParentTaskId + ON dbo.TaskInfo(QueueId, ParentTaskId); + +CREATE TABLE dbo.TokenDateTimeCompositeSearchParam ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + StartDateTime2 DATETIME2 (7) NOT NULL, + EndDateTime2 DATETIME2 (7) NOT NULL, + IsLongerThanADay2 BIT NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL +); + +ALTER TABLE dbo.TokenDateTimeCompositeSearchParam SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_TokenDateTimeCompositeSearchParam + ON dbo.TokenDateTimeCompositeSearchParam(ResourceTypeId, ResourceSurrogateId, SearchParamId) WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE INDEX IX_SearchParamId_Code1_StartDateTime2_EndDateTime2_INCLUDE_SystemId1_IsLongerThanADay2 + ON dbo.TokenDateTimeCompositeSearchParam(SearchParamId, Code1, StartDateTime2, EndDateTime2) + INCLUDE(SystemId1, IsLongerThanADay2) WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE INDEX IX_SearchParamId_Code1_EndDateTime2_StartDateTime2_INCLUDE_SystemId1_IsLongerThanADay2 + ON dbo.TokenDateTimeCompositeSearchParam(SearchParamId, Code1, EndDateTime2, StartDateTime2) + INCLUDE(SystemId1, IsLongerThanADay2) WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE INDEX IX_SearchParamId_Code1_StartDateTime2_EndDateTime2_INCLUDE_SystemId1_WHERE_IsLongerThanADay2_1 + ON dbo.TokenDateTimeCompositeSearchParam(SearchParamId, Code1, StartDateTime2, EndDateTime2) + INCLUDE(SystemId1) WHERE IsLongerThanADay2 = 1 WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE INDEX IX_SearchParamId_Code1_EndDateTime2_StartDateTime2_INCLUDE_SystemId1_WHERE_IsLongerThanADay2_1 + ON dbo.TokenDateTimeCompositeSearchParam(SearchParamId, Code1, EndDateTime2, StartDateTime2) + INCLUDE(SystemId1) WHERE IsLongerThanADay2 = 1 WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.TokenNumberNumberCompositeSearchParam ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + SingleValue2 DECIMAL (36, 18) NULL, + LowValue2 DECIMAL (36, 18) NULL, + HighValue2 DECIMAL (36, 18) NULL, + SingleValue3 DECIMAL (36, 18) NULL, + LowValue3 DECIMAL (36, 18) NULL, + HighValue3 DECIMAL (36, 18) NULL, + HasRange BIT NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL +); + +ALTER TABLE dbo.TokenNumberNumberCompositeSearchParam SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_TokenNumberNumberCompositeSearchParam + ON dbo.TokenNumberNumberCompositeSearchParam(ResourceTypeId, ResourceSurrogateId, SearchParamId) WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE INDEX IX_SearchParamId_Code1_SingleValue2_SingleValue3_INCLUDE_SystemId1_WHERE_HasRange_0 + ON dbo.TokenNumberNumberCompositeSearchParam(SearchParamId, Code1, SingleValue2, SingleValue3) + INCLUDE(SystemId1) WHERE HasRange = 0 WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE INDEX IX_SearchParamId_Code1_LowValue2_HighValue2_LowValue3_HighValue3_INCLUDE_SystemId1_WHERE_HasRange_1 + ON dbo.TokenNumberNumberCompositeSearchParam(SearchParamId, Code1, LowValue2, HighValue2, LowValue3, HighValue3) + INCLUDE(SystemId1) WHERE HasRange = 1 WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.TokenQuantityCompositeSearchParam ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + SystemId2 INT NULL, + QuantityCodeId2 INT NULL, + SingleValue2 DECIMAL (36, 18) NULL, + LowValue2 DECIMAL (36, 18) NULL, + HighValue2 DECIMAL (36, 18) NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL +); + +ALTER TABLE dbo.TokenQuantityCompositeSearchParam SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_TokenQuantityCompositeSearchParam + ON dbo.TokenQuantityCompositeSearchParam(ResourceTypeId, ResourceSurrogateId, SearchParamId) WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE INDEX IX_SearchParamId_Code1_SingleValue2_INCLUDE_QuantityCodeId2_SystemId1_SystemId2_WHERE_SingleValue2_NOT_NULL + ON dbo.TokenQuantityCompositeSearchParam(SearchParamId, Code1, SingleValue2) + INCLUDE(QuantityCodeId2, SystemId1, SystemId2) WHERE SingleValue2 IS NOT NULL WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE INDEX IX_SearchParamId_Code1_LowValue2_HighValue2_INCLUDE_QuantityCodeId2_SystemId1_SystemId2_WHERE_LowValue2_NOT_NULL + ON dbo.TokenQuantityCompositeSearchParam(SearchParamId, Code1, LowValue2, HighValue2) + INCLUDE(QuantityCodeId2, SystemId1, SystemId2) WHERE LowValue2 IS NOT NULL WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE INDEX IX_SearchParamId_Code1_HighValue2_LowValue2_INCLUDE_QuantityCodeId2_SystemId1_SystemId2_WHERE_LowValue2_NOT_NULL + ON dbo.TokenQuantityCompositeSearchParam(SearchParamId, Code1, HighValue2, LowValue2) + INCLUDE(QuantityCodeId2, SystemId1, SystemId2) WHERE LowValue2 IS NOT NULL WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.TokenSearchParam ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId INT NULL, + Code VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL +); + +ALTER TABLE dbo.TokenSearchParam SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_TokenSearchParam + ON dbo.TokenSearchParam(ResourceTypeId, ResourceSurrogateId, SearchParamId) WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE INDEX IX_SearchParamId_Code_INCLUDE_SystemId + ON dbo.TokenSearchParam(SearchParamId, Code) + INCLUDE(SystemId) WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.TokenStringCompositeSearchParam ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + Text2 NVARCHAR (256) COLLATE Latin1_General_CI_AI NOT NULL, + TextOverflow2 NVARCHAR (MAX) COLLATE Latin1_General_CI_AI NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL +); + +ALTER TABLE dbo.TokenStringCompositeSearchParam SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_TokenStringCompositeSearchParam + ON dbo.TokenStringCompositeSearchParam(ResourceSurrogateId, SearchParamId) WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE INDEX IX_SearchParamId_Code1_Text2_INCLUDE_SystemId1_TextOverflow2 + ON dbo.TokenStringCompositeSearchParam(SearchParamId, Code1, Text2) + INCLUDE(SystemId1, TextOverflow2) WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE INDEX IX_SearchParamId_Code1_Text2_INCLUDE_SystemId1_WHERE_TextOverflow2_NOT_NULL + ON dbo.TokenStringCompositeSearchParam(SearchParamId, Code1, Text2) + INCLUDE(SystemId1) WHERE TextOverflow2 IS NOT NULL WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.TokenText ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + Text NVARCHAR (400) COLLATE Latin1_General_CI_AI NOT NULL, + IsHistory BIT NOT NULL +); + +ALTER TABLE dbo.TokenText + ADD CONSTRAINT DF_TokenText_IsHistory DEFAULT 0 FOR IsHistory; + +ALTER TABLE dbo.TokenText SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_TokenText + ON dbo.TokenText(ResourceTypeId, ResourceSurrogateId, SearchParamId) WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_TokenText_SearchParamId_Text + ON dbo.TokenText(ResourceTypeId, SearchParamId, Text, ResourceSurrogateId) WHERE IsHistory = 0 WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.TokenTokenCompositeSearchParam ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + SystemId2 INT NULL, + Code2 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL, + CodeOverflow2 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL +); + +ALTER TABLE dbo.TokenTokenCompositeSearchParam SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_TokenTokenCompositeSearchParam + ON dbo.TokenTokenCompositeSearchParam(ResourceSurrogateId, SearchParamId) WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE INDEX IX_SearchParamId_Code1_Code2_INCLUDE_SystemId1_SystemId2 + ON dbo.TokenTokenCompositeSearchParam(SearchParamId, Code1, Code2) + INCLUDE(SystemId1, SystemId2) WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.Transactions ( + SurrogateIdRangeFirstValue BIGINT NOT NULL, + SurrogateIdRangeLastValue BIGINT NOT NULL, + Definition VARCHAR (2000) NULL, + IsCompleted BIT CONSTRAINT DF_Transactions_IsCompleted DEFAULT 0 NOT NULL, + IsSuccess BIT CONSTRAINT DF_Transactions_IsSuccess DEFAULT 0 NOT NULL, + IsVisible BIT CONSTRAINT DF_Transactions_IsVisible DEFAULT 0 NOT NULL, + IsHistoryMoved BIT CONSTRAINT DF_Transactions_IsHistoryMoved DEFAULT 0 NOT NULL, + CreateDate DATETIME CONSTRAINT DF_Transactions_CreateDate DEFAULT getUTCdate() NOT NULL, + EndDate DATETIME NULL, + VisibleDate DATETIME NULL, + HistoryMovedDate DATETIME NULL, + HeartbeatDate DATETIME CONSTRAINT DF_Transactions_HeartbeatDate DEFAULT getUTCdate() NOT NULL, + FailureReason VARCHAR (MAX) NULL, + IsControlledByClient BIT CONSTRAINT DF_Transactions_IsControlledByClient DEFAULT 1 NOT NULL, + InvisibleHistoryRemovedDate DATETIME NULL CONSTRAINT PKC_Transactions_SurrogateIdRangeFirstValue PRIMARY KEY CLUSTERED (SurrogateIdRangeFirstValue) +); + +CREATE INDEX IX_IsVisible + ON dbo.Transactions(IsVisible); + +CREATE TABLE dbo.UriSearchParam ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + Uri VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL +); + +ALTER TABLE dbo.UriSearchParam SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_UriSearchParam + ON dbo.UriSearchParam(ResourceTypeId, ResourceSurrogateId, SearchParamId) WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE INDEX IX_SearchParamId_Uri + ON dbo.UriSearchParam(SearchParamId, Uri) WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.WatchdogLeases ( + Watchdog VARCHAR (100) NOT NULL, + LeaseHolder VARCHAR (100) CONSTRAINT DF_WatchdogLeases_LeaseHolder DEFAULT '' NOT NULL, + LeaseEndTime DATETIME CONSTRAINT DF_WatchdogLeases_LeaseEndTime DEFAULT 0 NOT NULL, + RemainingLeaseTimeSec AS datediff(second, getUTCdate(), LeaseEndTime), + LeaseRequestor VARCHAR (100) CONSTRAINT DF_WatchdogLeases_LeaseRequestor DEFAULT '' NOT NULL, + LeaseRequestTime DATETIME CONSTRAINT DF_WatchdogLeases_LeaseRequestTime DEFAULT 0 NOT NULL CONSTRAINT PKC_WatchdogLeases_Watchdog PRIMARY KEY CLUSTERED (Watchdog) +); + +COMMIT +GO +CREATE PROCEDURE dbo.AcquireReindexJobs +@jobHeartbeatTimeoutThresholdInSeconds BIGINT, @maximumNumberOfConcurrentJobsAllowed INT +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +SET TRANSACTION ISOLATION LEVEL SERIALIZABLE; +BEGIN TRANSACTION; +DECLARE @expirationDateTime AS DATETIME2 (7); +SELECT @expirationDateTime = DATEADD(second, -@jobHeartbeatTimeoutThresholdInSeconds, SYSUTCDATETIME()); +DECLARE @numberOfRunningJobs AS INT; +SELECT @numberOfRunningJobs = COUNT(*) +FROM dbo.ReindexJob WITH (TABLOCKX) +WHERE Status = 'Running' + AND HeartbeatDateTime > @expirationDateTime; +DECLARE @limit AS INT = @maximumNumberOfConcurrentJobsAllowed - @numberOfRunningJobs; +IF (@limit > 0) + BEGIN + DECLARE @availableJobs TABLE ( + Id VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + JobVersion BINARY (8) NOT NULL); + INSERT INTO @availableJobs + SELECT TOP (@limit) Id, + JobVersion + FROM dbo.ReindexJob + WHERE (Status = 'Queued' + OR (Status = 'Running' + AND HeartbeatDateTime <= @expirationDateTime)) + ORDER BY HeartbeatDateTime; + DECLARE @heartbeatDateTime AS DATETIME2 (7) = SYSUTCDATETIME(); + UPDATE dbo.ReindexJob + SET Status = 'Running', + HeartbeatDateTime = @heartbeatDateTime, + RawJobRecord = JSON_MODIFY(RawJobRecord, '$.status', 'Running') + OUTPUT inserted.RawJobRecord, inserted.JobVersion + FROM dbo.ReindexJob AS job + INNER JOIN + @availableJobs AS availableJob + ON job.Id = availableJob.Id + AND job.JobVersion = availableJob.JobVersion; + END +COMMIT TRANSACTION; + +GO +CREATE PROCEDURE dbo.AcquireWatchdogLease +@Watchdog VARCHAR (100), @Worker VARCHAR (100), @AllowRebalance BIT=1, @ForceAcquire BIT=0, @LeasePeriodSec FLOAT, @WorkerIsRunning BIT=0, @LeaseEndTime DATETIME OUTPUT, @IsAcquired BIT OUTPUT, @CurrentLeaseHolder VARCHAR (100)=NULL OUTPUT +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +DECLARE @SP AS VARCHAR (100) = 'AcquireWatchdogLease', @Mode AS VARCHAR (100), @msg AS VARCHAR (1000), @MyLeasesNumber AS INT, @OtherValidRequestsOrLeasesNumber AS INT, @MyValidRequestsOrLeasesNumber AS INT, @DesiredLeasesNumber AS INT, @NotLeasedWatchdogNumber AS INT, @WatchdogNumber AS INT, @Now AS DATETIME, @MyLastChangeTime AS DATETIME, @PreviousLeaseHolder AS VARCHAR (100), @Rows AS INT = 0, @NumberOfWorkers AS INT, @st AS DATETIME = getUTCdate(), @RowsInt AS INT, @Pattern AS VARCHAR (100); +BEGIN TRY + SET @Mode = 'R=' + isnull(@Watchdog, 'NULL') + ' W=' + isnull(@Worker, 'NULL') + ' F=' + isnull(CONVERT (VARCHAR, @ForceAcquire), 'NULL') + ' LP=' + isnull(CONVERT (VARCHAR, @LeasePeriodSec), 'NULL'); + SET @CurrentLeaseHolder = ''; + SET @IsAcquired = 0; + SET @Now = getUTCdate(); + SET @LeaseEndTime = @Now; + SET @Pattern = NULLIF ((SELECT Char + FROM dbo.Parameters + WHERE Id = 'WatchdogLeaseHolderIncludePatternFor' + @Watchdog), ''); + IF @Pattern IS NULL + SET @Pattern = NULLIF ((SELECT Char + FROM dbo.Parameters + WHERE Id = 'WatchdogLeaseHolderIncludePattern'), ''); + IF @Pattern IS NOT NULL + AND @Worker NOT LIKE @Pattern + BEGIN + SET @msg = 'Worker does not match include pattern=' + @Pattern; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows, @Text = @msg; + SET @CurrentLeaseHolder = isnull((SELECT LeaseHolder + FROM dbo.WatchdogLeases + WHERE Watchdog = @Watchdog), ''); + RETURN; + END + SET @Pattern = NULLIF ((SELECT Char + FROM dbo.Parameters + WHERE Id = 'WatchdogLeaseHolderExcludePatternFor' + @Watchdog), ''); + IF @Pattern IS NULL + SET @Pattern = NULLIF ((SELECT Char + FROM dbo.Parameters + WHERE Id = 'WatchdogLeaseHolderExcludePattern'), ''); + IF @Pattern IS NOT NULL + AND @Worker LIKE @Pattern + BEGIN + SET @msg = 'Worker matches exclude pattern=' + @Pattern; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows, @Text = @msg; + SET @CurrentLeaseHolder = isnull((SELECT LeaseHolder + FROM dbo.WatchdogLeases + WHERE Watchdog = @Watchdog), ''); + RETURN; + END + DECLARE @Watchdogs TABLE ( + Watchdog VARCHAR (100) PRIMARY KEY); + INSERT INTO @Watchdogs + SELECT Watchdog + FROM dbo.WatchdogLeases WITH (NOLOCK) + WHERE RemainingLeaseTimeSec * (-1) > 10 * @LeasePeriodSec + OR @ForceAcquire = 1 + AND Watchdog = @Watchdog + AND LeaseHolder <> @Worker; + IF @@rowcount > 0 + BEGIN + DELETE dbo.WatchdogLeases + WHERE Watchdog IN (SELECT Watchdog + FROM @Watchdogs); + SET @Rows += @@rowcount; + IF @Rows > 0 + BEGIN + SET @msg = ''; + SELECT @msg = CONVERT (VARCHAR (1000), @msg + CASE WHEN @msg = '' THEN '' ELSE ',' END + Watchdog) + FROM @Watchdogs; + SET @msg = CONVERT (VARCHAR (1000), 'Remove old/forced leases:' + @msg); + EXECUTE dbo.LogEvent @Process = 'AcquireWatchdogLease', @Status = 'Info', @Mode = @Mode, @Target = 'WatchdogLeases', @Action = 'Delete', @Rows = @Rows, @Text = @msg; + END + END + SET @NumberOfWorkers = 1 + (SELECT count(*) + FROM (SELECT LeaseHolder + FROM dbo.WatchdogLeases WITH (NOLOCK) + WHERE LeaseHolder <> @Worker + UNION + SELECT LeaseRequestor + FROM dbo.WatchdogLeases WITH (NOLOCK) + WHERE LeaseRequestor <> @Worker + AND LeaseRequestor <> '') AS A); + SET @Mode = CONVERT (VARCHAR (100), @Mode + ' N=' + CONVERT (VARCHAR (10), @NumberOfWorkers)); + IF NOT EXISTS (SELECT * + FROM dbo.WatchdogLeases WITH (NOLOCK) + WHERE Watchdog = @Watchdog) + INSERT INTO dbo.WatchdogLeases (Watchdog, LeaseEndTime, LeaseRequestTime) + SELECT @Watchdog, + dateadd(day, -10, @Now), + dateadd(day, -10, @Now) + WHERE NOT EXISTS (SELECT * + FROM dbo.WatchdogLeases WITH (TABLOCKX) + WHERE Watchdog = @Watchdog); + SET @LeaseEndTime = dateadd(second, @LeasePeriodSec, @Now); + SET @WatchdogNumber = (SELECT count(*) + FROM dbo.WatchdogLeases WITH (NOLOCK)); + SET @NotLeasedWatchdogNumber = (SELECT count(*) + FROM dbo.WatchdogLeases WITH (NOLOCK) + WHERE LeaseHolder = '' + OR LeaseEndTime < @Now); + SET @MyLeasesNumber = (SELECT count(*) + FROM dbo.WatchdogLeases WITH (NOLOCK) + WHERE LeaseHolder = @Worker + AND LeaseEndTime > @Now); + SET @OtherValidRequestsOrLeasesNumber = (SELECT count(*) + FROM dbo.WatchdogLeases WITH (NOLOCK) + WHERE LeaseHolder <> @Worker + AND LeaseEndTime > @Now + OR LeaseRequestor <> @Worker + AND datediff(second, LeaseRequestTime, @Now) < @LeasePeriodSec); + SET @MyValidRequestsOrLeasesNumber = (SELECT count(*) + FROM dbo.WatchdogLeases WITH (NOLOCK) + WHERE LeaseHolder = @Worker + AND LeaseEndTime > @Now + OR LeaseRequestor = @Worker + AND datediff(second, LeaseRequestTime, @Now) < @LeasePeriodSec); + SET @DesiredLeasesNumber = ceiling(1.0 * @WatchdogNumber / @NumberOfWorkers); + IF @DesiredLeasesNumber = 0 + SET @DesiredLeasesNumber = 1; + IF @DesiredLeasesNumber = 1 + AND @OtherValidRequestsOrLeasesNumber = 1 + AND @WatchdogNumber = 1 + SET @DesiredLeasesNumber = 0; + IF @MyValidRequestsOrLeasesNumber = floor(1.0 * @WatchdogNumber / @NumberOfWorkers) + AND @OtherValidRequestsOrLeasesNumber + @MyValidRequestsOrLeasesNumber = @WatchdogNumber + SET @DesiredLeasesNumber = @DesiredLeasesNumber - 1; + UPDATE dbo.WatchdogLeases + SET LeaseHolder = @Worker, + LeaseEndTime = @LeaseEndTime, + LeaseRequestor = '', + @PreviousLeaseHolder = LeaseHolder + WHERE Watchdog = @Watchdog + AND NOT (LeaseRequestor <> @Worker + AND datediff(second, LeaseRequestTime, @Now) < @LeasePeriodSec) + AND (LeaseHolder = @Worker + AND (LeaseEndTime > @Now + OR @WorkerIsRunning = 1) + OR LeaseEndTime < @Now + AND (@DesiredLeasesNumber > @MyLeasesNumber + OR @OtherValidRequestsOrLeasesNumber < @WatchdogNumber)); + IF @@rowcount > 0 + BEGIN + SET @IsAcquired = 1; + SET @msg = 'Lease holder changed from [' + isnull(@PreviousLeaseHolder, '') + '] to [' + @Worker + ']'; + IF @PreviousLeaseHolder <> @Worker + EXECUTE dbo.LogEvent @Process = 'AcquireWatchdogLease', @Status = 'Info', @Mode = @Mode, @Text = @msg; + END + ELSE + IF @AllowRebalance = 1 + BEGIN + SET @CurrentLeaseHolder = (SELECT LeaseHolder + FROM dbo.WatchdogLeases + WHERE Watchdog = @Watchdog); + UPDATE dbo.WatchdogLeases + SET LeaseRequestTime = @Now + WHERE Watchdog = @Watchdog + AND LeaseRequestor = @Worker + AND datediff(second, LeaseRequestTime, @Now) < @LeasePeriodSec; + IF @DesiredLeasesNumber > @MyValidRequestsOrLeasesNumber + BEGIN + UPDATE A + SET LeaseRequestor = @Worker, + LeaseRequestTime = @Now + FROM dbo.WatchdogLeases AS A + WHERE Watchdog = @Watchdog + AND NOT (LeaseRequestor <> @Worker + AND datediff(second, LeaseRequestTime, @Now) < @LeasePeriodSec) + AND @NotLeasedWatchdogNumber = 0 + AND (SELECT count(*) + FROM dbo.WatchdogLeases AS B + WHERE B.LeaseHolder = A.LeaseHolder + AND datediff(second, B.LeaseEndTime, @Now) < @LeasePeriodSec) > @DesiredLeasesNumber; + SET @RowsInt = @@rowcount; + SET @msg = '@DesiredLeasesNumber=[' + CONVERT (VARCHAR (10), @DesiredLeasesNumber) + '] > @MyValidRequestsOrLeasesNumber=[' + CONVERT (VARCHAR (10), @MyValidRequestsOrLeasesNumber) + ']'; + EXECUTE dbo.LogEvent @Process = 'AcquireWatchdogLease', @Status = 'Info', @Mode = @Mode, @Rows = @RowsInt, @Text = @msg; + END + END + SET @Mode = CONVERT (VARCHAR (100), @Mode + ' A=' + CONVERT (VARCHAR (1), @IsAcquired)); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows; +END TRY +BEGIN CATCH + IF @@trancount > 0 + ROLLBACK; + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = 'AcquireWatchdogLease', @Status = 'Error', @Mode = @Mode; + THROW; +END CATCH + +GO +CREATE OR ALTER PROCEDURE dbo.AddPartitionOnResourceChanges +@partitionBoundary DATETIME2 (7) OUTPUT +AS +BEGIN + SET XACT_ABORT ON; + BEGIN TRANSACTION; + DECLARE @rightPartitionBoundary AS DATETIME2 (7) = CAST ((SELECT TOP (1) value + FROM sys.partition_range_values AS prv + INNER JOIN + sys.partition_functions AS pf + ON pf.function_id = prv.function_id + WHERE pf.name = N'PartitionFunction_ResourceChangeData_Timestamp' + ORDER BY prv.boundary_id DESC) AS DATETIME2 (7)); + DECLARE @timestamp AS DATETIME2 (7) = DATEADD(hour, DATEDIFF(hour, 0, sysutcdatetime()), 0); + IF (@rightPartitionBoundary < @timestamp) + BEGIN + SET @rightPartitionBoundary = @timestamp; + END + SET @rightPartitionBoundary = DATEADD(hour, 1, @rightPartitionBoundary); + ALTER PARTITION SCHEME PartitionScheme_ResourceChangeData_Timestamp NEXT USED [Primary]; + ALTER PARTITION FUNCTION PartitionFunction_ResourceChangeData_Timestamp( ) + SPLIT RANGE (@rightPartitionBoundary); + SET @partitionBoundary = @rightPartitionBoundary; + COMMIT TRANSACTION; +END + +GO +CREATE PROCEDURE dbo.ArchiveJobs +@QueueType TINYINT +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'ArchiveJobs', @Mode AS VARCHAR (100) = '', @st AS DATETIME = getUTCdate(), @Rows AS INT = 0, @PartitionId AS TINYINT, @MaxPartitions AS TINYINT = 16, @LookedAtPartitions AS TINYINT = 0, @InflightRows AS INT = 0, @Lock AS VARCHAR (100) = 'DequeueJob_' + CONVERT (VARCHAR, @QueueType); +BEGIN TRY + SET @PartitionId = @MaxPartitions * rand(); + BEGIN TRANSACTION; + EXECUTE sp_getapplock @Lock, 'Exclusive'; + WHILE @LookedAtPartitions <= @MaxPartitions + BEGIN + SET @InflightRows += (SELECT count(*) + FROM dbo.JobQueue + WHERE PartitionId = @PartitionId + AND QueueType = @QueueType + AND Status IN (0, 1)); + SET @PartitionId = CASE WHEN @PartitionId = 15 THEN 0 ELSE @PartitionId + 1 END; + SET @LookedAtPartitions = @LookedAtPartitions + 1; + END + IF @InflightRows = 0 + BEGIN + SET @LookedAtPartitions = 0; + WHILE @LookedAtPartitions <= @MaxPartitions + BEGIN + UPDATE dbo.JobQueue + SET Status = 5 + WHERE PartitionId = @PartitionId + AND QueueType = @QueueType + AND Status IN (2, 3, 4); + SET @Rows += @@rowcount; + SET @PartitionId = CASE WHEN @PartitionId = 15 THEN 0 ELSE @PartitionId + 1 END; + SET @LookedAtPartitions = @LookedAtPartitions + 1; + END + END + COMMIT TRANSACTION; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows; +END TRY +BEGIN CATCH + IF @@trancount > 0 + ROLLBACK; + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.CaptureResourceChanges +@isDeleted BIT, @version INT, @resourceId VARCHAR (64), @resourceTypeId SMALLINT +AS +BEGIN + DECLARE @changeType AS SMALLINT; + IF (@isDeleted = 1) + BEGIN + SET @changeType = 2; + END + ELSE + BEGIN + IF (@version = 1) + BEGIN + SET @changeType = 0; + END + ELSE + BEGIN + SET @changeType = 1; + END + END + INSERT INTO dbo.ResourceChangeData (ResourceId, ResourceTypeId, ResourceVersion, ResourceChangeTypeId) + VALUES (@resourceId, @resourceTypeId, @version, @changeType); +END + +GO +CREATE PROCEDURE dbo.CaptureResourceIdsForChanges +@Resources dbo.ResourceList READONLY +AS +SET NOCOUNT ON; +INSERT INTO dbo.ResourceChangeData (ResourceId, ResourceTypeId, ResourceVersion, ResourceChangeTypeId) +SELECT ResourceId, + ResourceTypeId, + Version, + CASE WHEN IsDeleted = 1 THEN 2 WHEN Version > 1 THEN 1 ELSE 0 END +FROM @Resources +WHERE IsHistory = 0; + +GO +CREATE PROCEDURE dbo.CheckActiveReindexJobs +AS +SET NOCOUNT ON; +SELECT Id +FROM dbo.ReindexJob +WHERE Status = 'Running' + OR Status = 'Queued' + OR Status = 'Paused'; + +GO +CREATE OR ALTER PROCEDURE dbo.CheckSearchParamCacheConsistency +@TargetSearchParamLastUpdated VARCHAR (100), @SyncStartDate DATETIME2 (7), @ActiveHostsSince DATETIME2 (7), @StalenessThresholdMinutes INT=10 +AS +SET NOCOUNT ON; +SELECT HostName, + CAST (NULL AS DATETIME2 (7)) AS SyncEventDate, + CAST (NULL AS NVARCHAR (3500)) AS EventText +FROM dbo.EventLog +WHERE EventDate >= @ActiveHostsSince + AND HostName IS NOT NULL + AND Process = 'DequeueJob' +UNION ALL +SELECT HostName, + EventDate, + EventText +FROM dbo.EventLog +WHERE EventDate >= @SyncStartDate + AND HostName IS NOT NULL + AND Process = 'SearchParameterCacheRefresh' + AND Status = 'End'; + + +GO +INSERT INTO dbo.Parameters (Id, Char) +SELECT 'DequeueJob', + 'LogEvent' +WHERE NOT EXISTS (SELECT * + FROM dbo.Parameters + WHERE Id = 'DequeueJob'); + + +GO +INSERT INTO Parameters (Id, Char) +SELECT 'SearchParameterCacheRefresh', + 'LogEvent'; + +GO +CREATE PROCEDURE dbo.CleanupEventLog +WITH EXECUTE AS 'dbo' +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'CleanupEventLog', @Mode AS VARCHAR (100) = '', @MaxDeleteRows AS INT, @MaxAllowedRows AS BIGINT, @RetentionPeriodSecond AS INT, @DeletedRows AS INT, @TotalDeletedRows AS INT = 0, @TotalRows AS INT, @Now AS DATETIME = getUTCdate(); +EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start'; +BEGIN TRY + SET @MaxDeleteRows = (SELECT Number + FROM dbo.Parameters + WHERE Id = 'CleanupEventLog.DeleteBatchSize'); + IF @MaxDeleteRows IS NULL + RAISERROR ('Cannot get Parameter.CleanupEventLog.DeleteBatchSize', 18, 127); + SET @MaxAllowedRows = (SELECT Number + FROM dbo.Parameters + WHERE Id = 'CleanupEventLog.AllowedRows'); + IF @MaxAllowedRows IS NULL + RAISERROR ('Cannot get Parameter.CleanupEventLog.AllowedRows', 18, 127); + SET @RetentionPeriodSecond = (SELECT Number * 24 * 60 * 60 + FROM dbo.Parameters + WHERE Id = 'CleanupEventLog.RetentionPeriodDay'); + IF @RetentionPeriodSecond IS NULL + RAISERROR ('Cannot get Parameter.CleanupEventLog.RetentionPeriodDay', 18, 127); + SET @TotalRows = (SELECT sum(row_count) + FROM sys.dm_db_partition_stats + WHERE object_id = object_id('EventLog') + AND index_id IN (0, 1)); + SET @DeletedRows = 1; + WHILE @DeletedRows > 0 + AND EXISTS (SELECT * + FROM dbo.Parameters + WHERE Id = 'CleanupEventLog.IsEnabled' + AND Number = 1) + BEGIN + SET @DeletedRows = 0; + IF @TotalRows - @TotalDeletedRows > @MaxAllowedRows + BEGIN + DELETE TOP (@MaxDeleteRows) + dbo.EventLog WITH (PAGLOCK) + WHERE EventDate <= dateadd(second, -@RetentionPeriodSecond, @Now); + SET @DeletedRows = @@rowcount; + SET @TotalDeletedRows += @DeletedRows; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Run', @Target = 'EventLog', @Action = 'Delete', @Rows = @DeletedRows, @Text = @TotalDeletedRows; + END + END + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @Now; +END TRY +BEGIN CATCH + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE OR ALTER PROCEDURE dbo.ConfigurePartitionOnResourceChanges +@numberOfFuturePartitionsToAdd INT +AS +BEGIN + SET XACT_ABORT ON; + BEGIN TRANSACTION; + DECLARE @partitionBoundary AS DATETIME2 (7) = DATEADD(hour, DATEDIFF(hour, 0, sysutcdatetime()), 0); + DECLARE @startingRightPartitionBoundary AS DATETIME2 (7) = CAST ((SELECT TOP (1) value + FROM sys.partition_range_values AS prv + INNER JOIN + sys.partition_functions AS pf + ON pf.function_id = prv.function_id + WHERE pf.name = N'PartitionFunction_ResourceChangeData_Timestamp' + ORDER BY prv.boundary_id DESC) AS DATETIME2 (7)); + DECLARE @numberOfPartitionsToAdd AS INT = @numberOfFuturePartitionsToAdd + 1; + WHILE @numberOfPartitionsToAdd > 0 + BEGIN + IF (@startingRightPartitionBoundary < @partitionBoundary) + BEGIN + ALTER PARTITION SCHEME PartitionScheme_ResourceChangeData_Timestamp NEXT USED [PRIMARY]; + ALTER PARTITION FUNCTION PartitionFunction_ResourceChangeData_Timestamp( ) + SPLIT RANGE (@partitionBoundary); + END + SET @partitionBoundary = DATEADD(hour, 1, @partitionBoundary); + SET @numberOfPartitionsToAdd -= 1; + END + COMMIT TRANSACTION; +END + +GO +CREATE PROCEDURE dbo.CreateReindexJob +@id VARCHAR (64), @status VARCHAR (10), @rawJobRecord VARCHAR (MAX) +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +BEGIN TRANSACTION; +DECLARE @heartbeatDateTime AS DATETIME2 (7) = SYSUTCDATETIME(); +INSERT INTO dbo.ReindexJob (Id, Status, HeartbeatDateTime, RawJobRecord) +VALUES (@id, @status, @heartbeatDateTime, @rawJobRecord); +SELECT CAST (MIN_ACTIVE_ROWVERSION() AS INT); +COMMIT TRANSACTION; + +GO +CREATE PROCEDURE dbo.CreateResourceSearchParamStats +@Table VARCHAR (100), @Column VARCHAR (100), @ResourceTypeId SMALLINT, @SearchParamId SMALLINT +WITH EXECUTE AS 'dbo' +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = object_name(@@procid), @Mode AS VARCHAR (200) = 'T=' + isnull(@Table, 'NULL') + ' C=' + isnull(@Column, 'NULL') + ' RT=' + isnull(CONVERT (VARCHAR, @ResourceTypeId), 'NULL') + ' SP=' + isnull(CONVERT (VARCHAR, @SearchParamId), 'NULL'), @st AS DATETIME = getUTCdate(); +BEGIN TRY + IF @Table IS NULL + OR @Column IS NULL + OR @ResourceTypeId IS NULL + OR @SearchParamId IS NULL + RAISERROR ('@TableName IS NULL OR @KeyColumn IS NULL OR @ResourceTypeId IS NULL OR @SearchParamId IS NULL', 18, 127); + EXECUTE ('CREATE STATISTICS ST_' + @Column + '_WHERE_ResourceTypeId_' + @ResourceTypeId + '_SearchParamId_' + @SearchParamId + ' ON dbo.' + @Table + ' (' + @Column + ') WHERE ResourceTypeId = ' + @ResourceTypeId + ' AND SearchParamId = ' + @SearchParamId); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Text = 'Stats created'; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + IF error_number() = 1927 + BEGIN + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st; + RETURN; + END + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.Defrag +@TableName VARCHAR (100), @IndexName VARCHAR (200), @PartitionNumber INT, @IsPartitioned BIT +WITH EXECUTE AS 'dbo' +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = object_name(@@procid), @Mode AS VARCHAR (200) = @TableName + '.' + @IndexName + '.' + CONVERT (VARCHAR, @PartitionNumber) + '.' + CONVERT (VARCHAR, @IsPartitioned), @st AS DATETIME = getUTCdate(), @SQL AS VARCHAR (3500), @msg AS VARCHAR (1000), @SizeBefore AS FLOAT, @SizeAfter AS FLOAT, @IndexId AS INT, @Operation AS VARCHAR (50) = CASE WHEN EXISTS (SELECT * + FROM dbo.Parameters + WHERE Id = 'Defrag.IndexRebuild.IsEnabled' + AND Number = 1) THEN 'REBUILD' ELSE 'REORGANIZE' END; +SET @Mode = @Mode + ' ' + @Operation; +BEGIN TRY + SET @IndexId = (SELECT index_id + FROM sys.indexes + WHERE object_id = object_id(@TableName) + AND name = @IndexName); + SET @Sql = 'ALTER INDEX ' + quotename(@IndexName) + ' ON dbo.' + quotename(@TableName) + ' ' + @Operation + CASE WHEN @IsPartitioned = 1 THEN ' PARTITION = ' + CONVERT (VARCHAR, @PartitionNumber) ELSE '' END + CASE WHEN @Operation = 'REBUILD' THEN ' WITH (ONLINE = ON' + CASE WHEN EXISTS (SELECT * + FROM sys.partitions + WHERE object_id = object_id(@TableName) + AND index_id = @IndexId + AND data_compression_desc = 'PAGE') THEN ', DATA_COMPRESSION = PAGE' ELSE '' END + ')' ELSE '' END; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start', @Text = @Sql; + SET @SizeBefore = (SELECT sum(reserved_page_count) + FROM sys.dm_db_partition_stats + WHERE object_id = object_id(@TableName) + AND index_id = @IndexId + AND partition_number = @PartitionNumber) * 8.0 / 1024 / 1024; + SET @msg = 'Size[GB] before=' + CONVERT (VARCHAR, @SizeBefore); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Run', @Text = @msg; + BEGIN TRY + EXECUTE (@Sql); + SET @SizeAfter = (SELECT sum(reserved_page_count) + FROM sys.dm_db_partition_stats + WHERE object_id = object_id(@TableName) + AND index_id = @IndexId + AND partition_number = @PartitionNumber) * 8.0 / 1024 / 1024; + SET @msg = 'Size[GB] before=' + CONVERT (VARCHAR, @SizeBefore) + ', after=' + CONVERT (VARCHAR, @SizeAfter) + ', reduced by=' + CONVERT (VARCHAR, @SizeBefore - @SizeAfter); + EXECUTE dbo.LogEvent @Process = @SP, @Status = 'End', @Mode = @Mode, @Action = @Operation, @Start = @st, @Text = @msg; + END TRY + BEGIN CATCH + EXECUTE dbo.LogEvent @Process = @SP, @Status = 'Error', @Mode = @Mode, @Action = @Operation, @Start = @st; + THROW; + END CATCH +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.DefragChangeDatabaseSettings +@IsOn BIT +WITH EXECUTE AS 'dbo' +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'DefragChangeDatabaseSettings', @Mode AS VARCHAR (200) = 'On=' + CONVERT (VARCHAR, @IsOn), @st AS DATETIME = getUTCdate(), @SQL AS VARCHAR (3500); +BEGIN TRY + EXECUTE dbo.LogEvent @Process = @SP, @Status = 'Start', @Mode = @Mode; + SET @SQL = 'ALTER DATABASE CURRENT SET AUTO_UPDATE_STATISTICS ' + CASE WHEN @IsOn = 1 THEN 'ON' ELSE 'OFF' END; + EXECUTE (@SQL); + EXECUTE dbo.LogEvent @Process = @SP, @Status = 'Run', @Mode = @Mode, @Text = @SQL; + SET @SQL = 'ALTER DATABASE CURRENT SET AUTO_CREATE_STATISTICS ' + CASE WHEN @IsOn = 1 THEN 'ON' ELSE 'OFF' END; + EXECUTE (@SQL); + EXECUTE dbo.LogEvent @Process = @SP, @Status = 'End', @Mode = @Mode, @Start = @st, @Text = @SQL; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.DefragGetFragmentation +@TableName VARCHAR (200), @IndexName VARCHAR (200)=NULL, @PartitionNumber INT=NULL +WITH EXECUTE AS 'dbo' +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = object_name(@@procid), @st AS DATETIME = getUTCdate(), @msg AS VARCHAR (1000), @Rows AS INT, @MinFragPct AS INT = isnull((SELECT Number + FROM dbo.Parameters + WHERE Id = 'Defrag.MinFragPct'), 10), @MinSizeGB AS FLOAT = isnull((SELECT Number + FROM dbo.Parameters + WHERE Id = 'Defrag.MinSizeGB'), 0.1), @PreviousGroupId AS BIGINT, @IndexId AS INT; +DECLARE @Mode AS VARCHAR (200) = 'T=' + @TableName + ' I=' + isnull(@IndexName, 'NULL') + ' P=' + isnull(CONVERT (VARCHAR, @PartitionNumber), 'NULL') + ' MF=' + CONVERT (VARCHAR, @MinFragPct) + ' MS=' + CONVERT (VARCHAR, @MinSizeGB); +BEGIN TRY + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start'; + IF object_id(@TableName) IS NULL + RAISERROR ('Table does not exist', 18, 127); + SET @IndexId = (SELECT index_id + FROM sys.indexes + WHERE object_id = object_id(@TableName) + AND name = @IndexName); + IF @IndexName IS NOT NULL + AND @IndexId IS NULL + RAISERROR ('Index does not exist', 18, 127); + SET @PreviousGroupId = (SELECT TOP 1 GroupId + FROM dbo.JobQueue + WHERE QueueType = 3 + AND Status = 5 + AND Definition = @TableName + ORDER BY GroupId DESC); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Run', @Target = '@PreviousGroupId', @Text = @PreviousGroupId; + SELECT TableName, + IndexName, + partition_number, + frag_in_percent + FROM (SELECT @TableName AS TableName, + I.name AS IndexName, + partition_number, + avg_fragmentation_in_percent AS frag_in_percent, + isnull(CONVERT (FLOAT, Result), 0) AS prev_frag_in_percent + FROM (SELECT object_id, + index_id, + partition_number, + avg_fragmentation_in_percent + FROM sys.dm_db_index_physical_stats(db_id(), object_id(@TableName), @IndexId, @PartitionNumber, 'LIMITED') AS A + WHERE index_id > 0 + AND (@PartitionNumber IS NOT NULL + OR avg_fragmentation_in_percent >= @MinFragPct + AND A.page_count > @MinSizeGB * 1024 * 1024 / 8)) AS A + INNER JOIN + sys.indexes AS I + ON I.object_id = A.object_id + AND I.index_id = A.index_id + LEFT OUTER JOIN + dbo.JobQueue + ON QueueType = 3 + AND Status = 5 + AND GroupId = @PreviousGroupId + AND Definition = I.name + ';' + CONVERT (VARCHAR, partition_number)) AS A + WHERE @PartitionNumber IS NOT NULL + OR frag_in_percent >= prev_frag_in_percent + @MinFragPct; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.DeleteHistory +@DeleteResources BIT=0, @Reset BIT=0, @DisableLogEvent BIT=0 +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'DeleteHistory', @Mode AS VARCHAR (100) = 'D=' + isnull(CONVERT (VARCHAR, @DeleteResources), 'NULL') + ' R=' + isnull(CONVERT (VARCHAR, @Reset), 'NULL'), @st AS DATETIME = getUTCdate(), @Id AS VARCHAR (100) = 'DeleteHistory.LastProcessed.TypeId.SurrogateId', @ResourceTypeId AS SMALLINT, @SurrogateId AS BIGINT, @RowsToProcess AS INT, @ProcessedResources AS INT = 0, @DeletedResources AS INT = 0, @DeletedSearchParams AS INT = 0, @ReportDate AS DATETIME = getUTCdate(); +BEGIN TRY + IF @DisableLogEvent = 0 + INSERT INTO dbo.Parameters (Id, Char) + SELECT @SP, + 'LogEvent'; + ELSE + DELETE dbo.Parameters + WHERE Id = @SP; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start'; + INSERT INTO dbo.Parameters (Id, Char) + SELECT @Id, + '0.0' + WHERE NOT EXISTS (SELECT * + FROM dbo.Parameters + WHERE Id = @Id); + DECLARE @LastProcessed AS VARCHAR (100) = CASE WHEN @Reset = 0 THEN (SELECT Char + FROM dbo.Parameters + WHERE Id = @Id) ELSE '0.0' END; + DECLARE @Types TABLE ( + ResourceTypeId SMALLINT PRIMARY KEY, + Name VARCHAR (100)); + DECLARE @SurrogateIds TABLE ( + ResourceSurrogateId BIGINT PRIMARY KEY, + IsHistory BIT ); + INSERT INTO @Types + EXECUTE dbo.GetUsedResourceTypes ; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Run', @Target = '@Types', @Action = 'Insert', @Rows = @@rowcount; + SET @ResourceTypeId = substring(@LastProcessed, 1, charindex('.', @LastProcessed) - 1); + SET @SurrogateId = substring(@LastProcessed, charindex('.', @LastProcessed) + 1, 255); + DELETE @Types + WHERE ResourceTypeId < @ResourceTypeId; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Run', @Target = '@Types', @Action = 'Delete', @Rows = @@rowcount; + WHILE EXISTS (SELECT * + FROM @Types) + BEGIN + SET @ResourceTypeId = (SELECT TOP 1 ResourceTypeId + FROM @Types + ORDER BY ResourceTypeId); + SET @ProcessedResources = 0; + SET @DeletedResources = 0; + SET @DeletedSearchParams = 0; + SET @RowsToProcess = 1; + WHILE @RowsToProcess > 0 + BEGIN + DELETE @SurrogateIds; + INSERT INTO @SurrogateIds + SELECT TOP 10000 ResourceSurrogateId, + IsHistory + FROM dbo.Resource + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId > @SurrogateId + ORDER BY ResourceSurrogateId; + SET @RowsToProcess = @@rowcount; + SET @ProcessedResources += @RowsToProcess; + IF @RowsToProcess > 0 + SET @SurrogateId = (SELECT max(ResourceSurrogateId) + FROM @SurrogateIds); + SET @LastProcessed = CONVERT (VARCHAR, @ResourceTypeId) + '.' + CONVERT (VARCHAR, @SurrogateId); + DELETE @SurrogateIds + WHERE IsHistory = 0; + IF EXISTS (SELECT * + FROM @SurrogateIds) + BEGIN + DELETE dbo.ResourceWriteClaim + WHERE ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.CompartmentAssignment + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.ReferenceSearchParam + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.TokenSearchParam + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.TokenText + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.StringSearchParam + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.UriSearchParam + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.NumberSearchParam + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.QuantitySearchParam + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.DateTimeSearchParam + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.ReferenceTokenCompositeSearchParam + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.TokenTokenCompositeSearchParam + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.TokenDateTimeCompositeSearchParam + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.TokenQuantityCompositeSearchParam + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.TokenStringCompositeSearchParam + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.TokenNumberNumberCompositeSearchParam + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + IF @DeleteResources = 1 + BEGIN + DELETE dbo.Resource + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedResources += @@rowcount; + END + END + UPDATE dbo.Parameters + SET Char = @LastProcessed + WHERE Id = @Id; + IF datediff(second, @ReportDate, getUTCdate()) > 60 + BEGIN + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Run', @Target = 'Resource', @Action = 'Select', @Rows = @ProcessedResources, @Text = @LastProcessed; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Run', @Target = '*SearchParam', @Action = 'Delete', @Rows = @DeletedSearchParams, @Text = @LastProcessed; + IF @DeleteResources = 1 + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Run', @Target = 'Resource', @Action = 'Delete', @Rows = @DeletedResources, @Text = @LastProcessed; + SET @ReportDate = getUTCdate(); + SET @ProcessedResources = 0; + SET @DeletedSearchParams = 0; + SET @DeletedResources = 0; + END + END + DELETE @Types + WHERE ResourceTypeId = @ResourceTypeId; + SET @SurrogateId = 0; + END + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Run', @Target = 'Resource', @Action = 'Select', @Rows = @ProcessedResources, @Text = @LastProcessed; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Run', @Target = '*SearchParam', @Action = 'Delete', @Rows = @DeletedSearchParams, @Text = @LastProcessed; + IF @DeleteResources = 1 + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Run', @Target = 'Resource', @Action = 'Delete', @Rows = @DeletedResources, @Text = @LastProcessed; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.DequeueJob +@QueueType TINYINT, @Worker VARCHAR (100), @HeartbeatTimeoutSec INT, @InputJobId BIGINT=NULL, @CheckTimeoutJobs BIT=0 +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'DequeueJob', @Mode AS VARCHAR (100) = 'Q=' + isnull(CONVERT (VARCHAR, @QueueType), 'NULL') + ' H=' + isnull(CONVERT (VARCHAR, @HeartbeatTimeoutSec), 'NULL') + ' W=' + isnull(@Worker, 'NULL') + ' IJ=' + isnull(CONVERT (VARCHAR, @InputJobId), 'NULL') + ' T=' + isnull(CONVERT (VARCHAR, @CheckTimeoutJobs), 'NULL'), @Rows AS INT = 0, @st AS DATETIME = getUTCdate(), @JobId AS BIGINT, @msg AS VARCHAR (100), @Lock AS VARCHAR (100), @PartitionId AS TINYINT, @MaxPartitions AS TINYINT = 16, @LookedAtPartitions AS TINYINT = 0; +BEGIN TRY + IF EXISTS (SELECT * + FROM dbo.Parameters + WHERE Id = 'DequeueJobStop' + AND Number = 1) + BEGIN + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = 0, @Text = 'Skipped'; + RETURN; + END + IF @InputJobId IS NULL + SET @PartitionId = @MaxPartitions * rand(); + ELSE + SET @PartitionId = @InputJobId % 16; + SET TRANSACTION ISOLATION LEVEL READ COMMITTED; + WHILE @InputJobId IS NULL + AND @JobId IS NULL + AND @LookedAtPartitions < @MaxPartitions + AND @CheckTimeoutJobs = 0 + BEGIN + SET @Lock = 'DequeueJob_0_' + CONVERT (VARCHAR, @QueueType) + '_' + CONVERT (VARCHAR, @PartitionId); + BEGIN TRANSACTION; + EXECUTE sp_getapplock @Lock, 'Exclusive'; + UPDATE T + SET StartDate = getUTCdate(), + HeartbeatDate = getUTCdate(), + Worker = @Worker, + Status = 1, + Version = datediff_big(millisecond, '0001-01-01', getUTCdate()), + @JobId = T.JobId + FROM dbo.JobQueue AS T WITH (PAGLOCK) + INNER JOIN + (SELECT TOP 1 JobId + FROM dbo.JobQueue WITH (INDEX (IX_QueueType_PartitionId_Status_Priority)) + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND Status = 0 + ORDER BY Priority, JobId) AS S + ON QueueType = @QueueType + AND PartitionId = @PartitionId + AND T.JobId = S.JobId; + SET @Rows += @@rowcount; + COMMIT TRANSACTION; + IF @JobId IS NULL + BEGIN + SET @PartitionId = CASE WHEN @PartitionId = 15 THEN 0 ELSE @PartitionId + 1 END; + SET @LookedAtPartitions = @LookedAtPartitions + 1; + END + END + SET @LookedAtPartitions = 0; + WHILE @InputJobId IS NULL + AND @JobId IS NULL + AND @LookedAtPartitions < @MaxPartitions + BEGIN + SET @Lock = 'DequeueJob_1_' + CONVERT (VARCHAR, @QueueType) + '_' + CONVERT (VARCHAR, @PartitionId); + BEGIN TRANSACTION; + EXECUTE sp_getapplock @Lock, 'Exclusive'; + UPDATE T + SET StartDate = getUTCdate(), + HeartbeatDate = getUTCdate(), + Worker = @Worker, + Status = CASE WHEN CancelRequested = 0 THEN 1 ELSE 4 END, + Version = datediff_big(millisecond, '0001-01-01', getUTCdate()), + @JobId = CASE WHEN CancelRequested = 0 THEN T.JobId END, + Info = CONVERT (VARCHAR (1000), isnull(Info, '') + ' Prev: Worker=' + Worker + ' Start=' + CONVERT (VARCHAR, StartDate, 121)) + FROM dbo.JobQueue AS T WITH (PAGLOCK) + INNER JOIN + (SELECT TOP 1 JobId + FROM dbo.JobQueue WITH (INDEX (IX_QueueType_PartitionId_Status_Priority)) + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND Status = 1 + AND datediff(second, HeartbeatDate, getUTCdate()) > @HeartbeatTimeoutSec + ORDER BY Priority, JobId) AS S + ON QueueType = @QueueType + AND PartitionId = @PartitionId + AND T.JobId = S.JobId; + SET @Rows += @@rowcount; + COMMIT TRANSACTION; + IF @JobId IS NULL + BEGIN + SET @PartitionId = CASE WHEN @PartitionId = 15 THEN 0 ELSE @PartitionId + 1 END; + SET @LookedAtPartitions = @LookedAtPartitions + 1; + END + END + IF @InputJobId IS NOT NULL + BEGIN + UPDATE dbo.JobQueue WITH (PAGLOCK) + SET StartDate = getUTCdate(), + HeartbeatDate = getUTCdate(), + Worker = @Worker, + Status = 1, + Version = datediff_big(millisecond, '0001-01-01', getUTCdate()), + @JobId = JobId + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND Status = 0 + AND JobId = @InputJobId; + SET @Rows += @@rowcount; + IF @JobId IS NULL + BEGIN + UPDATE dbo.JobQueue WITH (PAGLOCK) + SET StartDate = getUTCdate(), + HeartbeatDate = getUTCdate(), + Worker = @Worker, + Status = 1, + Version = datediff_big(millisecond, '0001-01-01', getUTCdate()), + @JobId = JobId + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND Status = 1 + AND JobId = @InputJobId + AND datediff(second, HeartbeatDate, getUTCdate()) > @HeartbeatTimeoutSec; + SET @Rows += @@rowcount; + END + END + IF @JobId IS NOT NULL + EXECUTE dbo.GetJobs @QueueType = @QueueType, @JobId = @JobId; + SET @msg = 'J=' + isnull(CONVERT (VARCHAR, @JobId), 'NULL') + ' P=' + CONVERT (VARCHAR, @PartitionId); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows, @Text = @msg; +END TRY +BEGIN CATCH + IF @@trancount > 0 + ROLLBACK; + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + + +GO +INSERT INTO Parameters (Id, Char) +SELECT 'DequeueJob', + 'LogEvent'; + +GO +CREATE PROCEDURE dbo.DisableIndex +@tableName NVARCHAR (128), @indexName NVARCHAR (128) +WITH EXECUTE AS 'dbo' +AS +DECLARE @errorTxt AS VARCHAR (1000), @sql AS NVARCHAR (1000), @isDisabled AS BIT; +IF object_id(@tableName) IS NULL + BEGIN + SET @errorTxt = @tableName + ' does not exist or you don''t have permissions.'; + RAISERROR (@errorTxt, 18, 127); + END +SET @isDisabled = (SELECT is_disabled + FROM sys.indexes + WHERE object_id = object_id(@tableName) + AND name = @indexName); +IF @isDisabled IS NULL + BEGIN + SET @errorTxt = @indexName + ' does not exist or you don''t have permissions.'; + RAISERROR (@errorTxt, 18, 127); + END +IF @isDisabled = 0 + BEGIN + SET @sql = N'ALTER INDEX ' + QUOTENAME(@indexName) + N' on ' + @tableName + ' Disable'; + EXECUTE sp_executesql @sql; + END + +GO +CREATE PROCEDURE dbo.DisableIndexes +WITH EXECUTE AS 'dbo' +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'DisableIndexes', @Mode AS VARCHAR (200) = '', @st AS DATETIME = getUTCdate(), @Tbl AS VARCHAR (100), @Ind AS VARCHAR (200), @Txt AS VARCHAR (4000); +BEGIN TRY + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start'; + DECLARE @Tables TABLE ( + Tbl VARCHAR (100) PRIMARY KEY, + Supported BIT ); + INSERT INTO @Tables + EXECUTE dbo.GetPartitionedTables @IncludeNotDisabled = 1, @IncludeNotSupported = 0; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@Tables', @Action = 'Insert', @Rows = @@rowcount; + DECLARE @Indexes TABLE ( + Tbl VARCHAR (100), + Ind VARCHAR (200), + TblId INT , + IndId INT PRIMARY KEY (Tbl, Ind)); + INSERT INTO @Indexes + SELECT Tbl, + I.Name, + TblId, + I.index_id + FROM (SELECT object_id(Tbl) AS TblId, + Tbl + FROM @Tables) AS O + INNER JOIN + sys.indexes AS I + ON I.object_id = TblId; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@Indexes', @Action = 'Insert', @Rows = @@rowcount; + INSERT INTO dbo.IndexProperties (TableName, IndexName, PropertyName, PropertyValue) + SELECT Tbl, + Ind, + 'DATA_COMPRESSION', + data_comp + FROM (SELECT Tbl, + Ind, + isnull((SELECT TOP 1 CASE WHEN data_compression_desc = 'PAGE' THEN 'PAGE' END + FROM sys.partitions + WHERE object_id = TblId + AND index_id = IndId), 'NONE') AS data_comp + FROM @Indexes) AS A + WHERE NOT EXISTS (SELECT * + FROM dbo.IndexProperties + WHERE TableName = Tbl + AND IndexName = Ind); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = 'IndexProperties', @Action = 'Insert', @Rows = @@rowcount; + DELETE @Indexes + WHERE Tbl = 'Resource' + OR IndId = 1; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@Indexes', @Action = 'Delete', @Rows = @@rowcount; + WHILE EXISTS (SELECT * + FROM @Indexes) + BEGIN + SELECT TOP 1 @Tbl = Tbl, + @Ind = Ind + FROM @Indexes; + SET @Txt = 'ALTER INDEX ' + @Ind + ' ON dbo.' + @Tbl + ' DISABLE'; + EXECUTE (@Txt); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @Ind, @Action = 'Disable', @Text = @Txt; + DELETE @Indexes + WHERE Tbl = @Tbl + AND Ind = @Ind; + END + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.EnqueueJobs +@QueueType TINYINT, @Definitions StringList READONLY, @GroupId BIGINT=NULL, @ForceOneActiveJobGroup BIT=1, @Status TINYINT=NULL, @Result VARCHAR (MAX)=NULL, @StartDate DATETIME=NULL, @ReturnJobs BIT=1 +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'EnqueueJobs', @Mode AS VARCHAR (100) = 'Q=' + isnull(CONVERT (VARCHAR, @QueueType), 'NULL') + ' D=' + CONVERT (VARCHAR, (SELECT count(*) + FROM @Definitions)) + ' G=' + isnull(CONVERT (VARCHAR, @GroupId), 'NULL') + ' F=' + isnull(CONVERT (VARCHAR, @ForceOneActiveJobGroup), 'NULL') + ' S=' + isnull(CONVERT (VARCHAR, @Status), 'NULL'), @st AS DATETIME = getUTCdate(), @Lock AS VARCHAR (100) = 'EnqueueJobs_' + CONVERT (VARCHAR, @QueueType), @MaxJobId AS BIGINT, @Rows AS INT, @msg AS VARCHAR (1000), @JobIds AS BigintList, @InputRows AS INT; +BEGIN TRY + DECLARE @Input TABLE ( + DefinitionHash VARBINARY (20) PRIMARY KEY, + Definition VARCHAR (MAX) ); + INSERT INTO @Input + SELECT hashbytes('SHA1', String) AS DefinitionHash, + String AS Definition + FROM @Definitions; + SET @InputRows = @@rowcount; + INSERT INTO @JobIds + SELECT JobId + FROM @Input AS A + INNER JOIN + dbo.JobQueue AS B + ON B.QueueType = @QueueType + AND B.DefinitionHash = A.DefinitionHash + AND B.Status <> 5; + IF @@rowcount < @InputRows + BEGIN + BEGIN TRANSACTION; + EXECUTE sp_getapplock @Lock, 'Exclusive'; + IF @ForceOneActiveJobGroup = 1 + AND EXISTS (SELECT * + FROM dbo.JobQueue + WHERE QueueType = @QueueType + AND Status IN (0, 1) + AND (@GroupId IS NULL + OR GroupId <> @GroupId)) + RAISERROR ('There are other active job groups', 18, 127); + IF @GroupId IS NOT NULL + AND isnull(@Status, 0) <> 6 + AND EXISTS (SELECT * + FROM dbo.JobQueue + WHERE QueueType = @QueueType + AND JobId = @GroupId + AND CancelRequested = 1) + RAISERROR ('The specified job group is cancelled', 18, 127); + SET @MaxJobId = isnull((SELECT TOP 1 JobId + FROM dbo.JobQueue + WHERE QueueType = @QueueType + ORDER BY JobId DESC), 0); + INSERT INTO dbo.JobQueue (QueueType, GroupId, JobId, Definition, DefinitionHash, Status, Result, StartDate, EndDate) + OUTPUT inserted.JobId INTO @JobIds + SELECT @QueueType, + isnull(@GroupId, @MaxJobId + 1) AS GroupId, + JobId, + Definition, + DefinitionHash, + isnull(@Status, 0) AS Status, + CASE WHEN @Status = 2 THEN @Result ELSE NULL END AS Result, + CASE WHEN @Status = 1 THEN getUTCdate() ELSE @StartDate END AS StartDate, + CASE WHEN @Status = 2 THEN getUTCdate() ELSE NULL END AS EndDate + FROM (SELECT @MaxJobId + row_number() OVER (ORDER BY Dummy) AS JobId, + * + FROM (SELECT *, + 0 AS Dummy + FROM @Input) AS A) AS A + WHERE NOT EXISTS (SELECT * + FROM dbo.JobQueue AS B WITH (INDEX (IX_QueueType_DefinitionHash)) + WHERE B.QueueType = @QueueType + AND B.DefinitionHash = A.DefinitionHash + AND B.Status <> 5); + SET @Rows = @@rowcount; + COMMIT TRANSACTION; + END + IF @ReturnJobs = 1 + EXECUTE dbo.GetJobs @QueueType = @QueueType, @JobIds = @JobIds; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows; +END TRY +BEGIN CATCH + IF @@trancount > 0 + ROLLBACK; + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + + +GO +INSERT INTO Parameters (Id, Char) +SELECT 'EnqueueJobs', + 'LogEvent'; + +GO +CREATE PROCEDURE dbo.ExecuteCommandForRebuildIndexes +@Tbl VARCHAR (100), @Ind VARCHAR (1000), @Cmd VARCHAR (MAX) +WITH EXECUTE AS 'dbo' +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'ExecuteCommandForRebuildIndexes', @Mode AS VARCHAR (200) = 'Tbl=' + isnull(@Tbl, 'NULL'), @st AS DATETIME, @Retries AS INT = 0, @Action AS VARCHAR (100), @msg AS VARCHAR (1000); +RetryOnTempdbError: +BEGIN TRY + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start', @Text = @Cmd; + SET @st = getUTCdate(); + IF @Tbl IS NULL + RAISERROR ('@Tbl IS NULL', 18, 127); + IF @Cmd IS NULL + RAISERROR ('@Cmd IS NULL', 18, 127); + SET @Action = CASE WHEN @Cmd LIKE 'UPDATE STAT%' THEN 'Update statistics' WHEN @Cmd LIKE 'CREATE%INDEX%' THEN 'Create Index' WHEN @Cmd LIKE 'ALTER%INDEX%REBUILD%' THEN 'Rebuild Index' WHEN @Cmd LIKE 'ALTER%TABLE%ADD%' THEN 'Add Constraint' END; + IF @Action IS NULL + BEGIN + SET @msg = 'Not supported command = ' + CONVERT (VARCHAR (900), @Cmd); + RAISERROR (@msg, 18, 127); + END + IF @Action = 'Create Index' + WAITFOR DELAY '00:00:05'; + EXECUTE (@Cmd); + SELECT @Ind; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Action = @Action, @Status = 'End', @Start = @st, @Text = @Cmd; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + IF error_number() = 40544 + BEGIN + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st, @Retry = @Retries; + SET @Retries = @Retries + 1; + IF @Tbl = 'TokenText_96' + WAITFOR DELAY '01:00:00'; + ELSE + WAITFOR DELAY '00:10:00'; + GOTO RetryOnTempdbError; + END + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE OR ALTER PROCEDURE dbo.FetchEventAgentCheckpoint +@CheckpointId VARCHAR (64) +AS +BEGIN + SELECT TOP (1) CheckpointId, + LastProcessedDateTime, + LastProcessedIdentifier + FROM dbo.EventAgentCheckpoint + WHERE CheckpointId = @CheckpointId; +END + +GO +CREATE PROCEDURE dbo.FetchResourceChanges_3 +@startId BIGINT, @lastProcessedUtcDateTime DATETIME2 (7), @pageSize SMALLINT +AS +BEGIN + SET NOCOUNT ON; + DECLARE @precedingPartitionBoundary AS DATETIME2 (7) = (SELECT TOP (1) CAST (prv.value AS DATETIME2 (7)) AS value + FROM sys.partition_range_values AS prv WITH (NOLOCK) + INNER JOIN + sys.partition_functions AS pf WITH (NOLOCK) + ON pf.function_id = prv.function_id + WHERE pf.name = N'PartitionFunction_ResourceChangeData_Timestamp' + AND SQL_VARIANT_PROPERTY(prv.Value, 'BaseType') = 'datetime2' + AND CAST (prv.value AS DATETIME2 (7)) < DATEADD(HOUR, DATEDIFF(HOUR, 0, @lastProcessedUtcDateTime), 0) + ORDER BY prv.boundary_id DESC); + IF (@precedingPartitionBoundary IS NULL) + BEGIN + SET @precedingPartitionBoundary = CONVERT (DATETIME2 (7), N'1970-01-01T00:00:00.0000000'); + END + DECLARE @endDateTimeToFilter AS DATETIME2 (7) = DATEADD(HOUR, 1, SYSUTCDATETIME()); + WITH PartitionBoundaries + AS (SELECT CAST (prv.value AS DATETIME2 (7)) AS PartitionBoundary + FROM sys.partition_range_values AS prv WITH (NOLOCK) + INNER JOIN + sys.partition_functions AS pf WITH (NOLOCK) + ON pf.function_id = prv.function_id + WHERE pf.name = N'PartitionFunction_ResourceChangeData_Timestamp' + AND SQL_VARIANT_PROPERTY(prv.Value, 'BaseType') = 'datetime2' + AND CAST (prv.value AS DATETIME2 (7)) BETWEEN @precedingPartitionBoundary AND @endDateTimeToFilter) + SELECT TOP (@pageSize) Id, + Timestamp, + ResourceId, + ResourceTypeId, + ResourceVersion, + ResourceChangeTypeId + FROM PartitionBoundaries AS p CROSS APPLY (SELECT TOP (@pageSize) Id, + Timestamp, + ResourceId, + ResourceTypeId, + ResourceVersion, + ResourceChangeTypeId + FROM dbo.ResourceChangeData WITH (TABLOCK, HOLDLOCK) + WHERE Id >= @startId + AND $PARTITION.PartitionFunction_ResourceChangeData_Timestamp (Timestamp) = $PARTITION.PartitionFunction_ResourceChangeData_Timestamp (p.PartitionBoundary) + ORDER BY Id ASC) AS rcd + ORDER BY rcd.Id ASC; +END + +GO +CREATE PROCEDURE dbo.GetActiveJobs +@QueueType TINYINT, @GroupId BIGINT=NULL, @ReturnParentOnly BIT=0 +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'GetActiveJobs', @Mode AS VARCHAR (100) = 'Q=' + isnull(CONVERT (VARCHAR, @QueueType), 'NULL') + ' G=' + isnull(CONVERT (VARCHAR, @GroupId), 'NULL') + ' R=' + CONVERT (VARCHAR, @ReturnParentOnly), @st AS DATETIME = getUTCdate(), @JobIds AS BigintList, @PartitionId AS TINYINT, @MaxPartitions AS TINYINT = 16, @LookedAtPartitions AS TINYINT = 0, @Rows AS INT = 0; +BEGIN TRY + SET @PartitionId = @MaxPartitions * rand(); + WHILE @LookedAtPartitions < @MaxPartitions + BEGIN + IF @GroupId IS NULL + INSERT INTO @JobIds + SELECT JobId + FROM dbo.JobQueue + WHERE PartitionId = @PartitionId + AND QueueType = @QueueType + AND Status IN (0, 1); + ELSE + INSERT INTO @JobIds + SELECT JobId + FROM dbo.JobQueue + WHERE PartitionId = @PartitionId + AND QueueType = @QueueType + AND GroupId = @GroupId + AND Status IN (0, 1); + SET @Rows += @@rowcount; + SET @PartitionId = CASE WHEN @PartitionId = 15 THEN 0 ELSE @PartitionId + 1 END; + SET @LookedAtPartitions += 1; + END + IF @Rows > 0 + BEGIN + IF @ReturnParentOnly = 1 + BEGIN + DECLARE @TopGroupId AS BIGINT; + SELECT TOP 1 @TopGroupId = GroupId + FROM dbo.JobQueue + WHERE JobId IN (SELECT Id + FROM @JobIds) + ORDER BY GroupId DESC; + DELETE @JobIds + WHERE Id NOT IN (SELECT JobId + FROM dbo.JobQueue + WHERE JobId = @TopGroupId); + END + EXECUTE dbo.GetJobs @QueueType = @QueueType, @JobIds = @JobIds; + END + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.GetCommandsForRebuildIndexes +@RebuildClustered BIT +WITH EXECUTE AS 'dbo' +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'GetCommandsForRebuildIndexes', @Mode AS VARCHAR (200) = 'PS=PartitionScheme_ResourceTypeId RC=' + isnull(CONVERT (VARCHAR, @RebuildClustered), 'NULL'), @st AS DATETIME = getUTCdate(), @Tbl AS VARCHAR (100), @TblInt AS VARCHAR (100), @Ind AS VARCHAR (200), @IndId AS INT, @Supported AS BIT, @Txt AS VARCHAR (MAX), @Rows AS BIGINT, @Pages AS BIGINT, @ResourceTypeId AS SMALLINT, @IndexesCnt AS INT, @DataComp AS VARCHAR (100); +BEGIN TRY + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start'; + DECLARE @Commands TABLE ( + Tbl VARCHAR (100), + Ind VARCHAR (200), + Txt VARCHAR (MAX), + Pages BIGINT ); + DECLARE @ResourceTypes TABLE ( + ResourceTypeId SMALLINT PRIMARY KEY); + DECLARE @Indexes TABLE ( + Ind VARCHAR (200) PRIMARY KEY, + IndId INT ); + DECLARE @Tables TABLE ( + name VARCHAR (100) PRIMARY KEY, + Supported BIT ); + INSERT INTO @Tables + EXECUTE dbo.GetPartitionedTables @IncludeNotDisabled = 1, @IncludeNotSupported = 1; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@Tables', @Action = 'Insert', @Rows = @@rowcount; + WHILE EXISTS (SELECT * + FROM @Tables) + BEGIN + SELECT TOP 1 @Tbl = name, + @Supported = Supported + FROM @Tables + ORDER BY name; + IF @Supported = 0 + BEGIN + INSERT INTO @Commands + SELECT @Tbl, + name, + 'ALTER INDEX ' + name + ' ON dbo.' + @Tbl + ' REBUILD' + CASE WHEN (SELECT PropertyValue + FROM dbo.IndexProperties + WHERE TableName = @Tbl + AND IndexName = name) = 'PAGE' THEN ' PARTITION = ALL WITH (DATA_COMPRESSION = PAGE)' ELSE '' END, + CONVERT (BIGINT, 9e18) + FROM sys.indexes + WHERE object_id = object_id(@Tbl) + AND (is_disabled = 1 + AND index_id > 1 + AND @RebuildClustered = 0 + OR index_id = 1 + AND @RebuildClustered = 1); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@Commands', @Action = 'Insert', @Rows = @@rowcount, @Text = 'Not supported tables with disabled indexes'; + END + ELSE + BEGIN + DELETE @ResourceTypes; + INSERT INTO @ResourceTypes + SELECT CONVERT (SMALLINT, substring(name, charindex('_', name) + 1, 6)) AS ResourceTypeId + FROM sys.sysobjects + WHERE name LIKE @Tbl + '[_]%'; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@ResourceTypes', @Action = 'Insert', @Rows = @@rowcount; + WHILE EXISTS (SELECT * + FROM @ResourceTypes) + BEGIN + SET @ResourceTypeId = (SELECT TOP 1 ResourceTypeId + FROM @ResourceTypes + ORDER BY ResourceTypeId); + SET @TblInt = @Tbl + '_' + CONVERT (VARCHAR, @ResourceTypeId); + SET @Pages = (SELECT dpages + FROM sysindexes + WHERE id = object_id(@TblInt) + AND indid IN (0, 1)); + DELETE @Indexes; + INSERT INTO @Indexes + SELECT name, + index_id + FROM sys.indexes + WHERE object_id = object_id(@Tbl) + AND (index_id > 1 + AND @RebuildClustered = 0 + OR index_id = 1 + AND @RebuildClustered = 1); + SET @IndexesCnt = 0; + WHILE EXISTS (SELECT * + FROM @Indexes) + BEGIN + SELECT TOP 1 @Ind = Ind, + @IndId = IndId + FROM @Indexes + ORDER BY Ind; + IF @IndId = 1 + BEGIN + SET @Txt = 'ALTER INDEX ' + @Ind + ' ON dbo.' + @TblInt + ' REBUILD' + CASE WHEN (SELECT PropertyValue + FROM dbo.IndexProperties + WHERE TableName = @Tbl + AND IndexName = @Ind) = 'PAGE' THEN ' PARTITION = ALL WITH (DATA_COMPRESSION = PAGE)' ELSE '' END; + INSERT INTO @Commands + SELECT @TblInt, + @Ind, + @Txt, + @Pages; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @TblInt, @Action = 'Add command', @Rows = @@rowcount, @Text = @Txt; + END + ELSE + IF NOT EXISTS (SELECT * + FROM sys.indexes + WHERE object_id = object_id(@TblInt) + AND name = @Ind) + BEGIN + EXECUTE dbo.GetIndexCommands @Tbl = @Tbl, @Ind = @Ind, @AddPartClause = 0, @IncludeClustered = 0, @Txt = @Txt OUTPUT; + SET @Txt = replace(@Txt, '[' + @Tbl + ']', @TblInt); + IF @Txt IS NOT NULL + BEGIN + SET @IndexesCnt = @IndexesCnt + 1; + INSERT INTO @Commands + SELECT @TblInt, + @Ind, + @Txt, + @Pages; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @TblInt, @Action = 'Add command', @Rows = @@rowcount, @Text = @Txt; + END + END + DELETE @Indexes + WHERE Ind = @Ind; + END + IF @IndexesCnt > 1 + BEGIN + INSERT INTO @Commands + SELECT @TblInt, + 'UPDATE STAT', + 'UPDATE STATISTICS dbo.' + @TblInt, + @Pages; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @TblInt, @Action = 'Add command', @Rows = @@rowcount, @Text = 'Add stats update'; + END + DELETE @ResourceTypes + WHERE ResourceTypeId = @ResourceTypeId; + END + END + DELETE @Tables + WHERE name = @Tbl; + END + SELECT Tbl, + Ind, + Txt + FROM @Commands + ORDER BY Pages DESC, Tbl, CASE WHEN Txt LIKE 'UPDATE STAT%' THEN 0 ELSE 1 END; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@Commands', @Action = 'Select', @Rows = @@rowcount; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE OR ALTER PROCEDURE dbo.GetGeoReplicationLag +AS +BEGIN + SET NOCOUNT ON; + SELECT replication_state_desc, + replication_lag_sec, + last_replication + FROM sys.dm_geo_replication_link_status + WHERE role_desc = 'PRIMARY'; +END + +GO +CREATE PROCEDURE dbo.GetIndexCommands +@Tbl VARCHAR (100), @Ind VARCHAR (200), @AddPartClause BIT, @IncludeClustered BIT, @Txt VARCHAR (MAX)=NULL OUTPUT +WITH EXECUTE AS 'dbo' +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'GetIndexCommands', @Mode AS VARCHAR (200) = 'Tbl=' + isnull(@Tbl, 'NULL') + ' Ind=' + isnull(@Ind, 'NULL'), @st AS DATETIME = getUTCdate(); +DECLARE @Indexes TABLE ( + Ind VARCHAR (200) PRIMARY KEY, + Txt VARCHAR (MAX)); +BEGIN TRY + IF @Tbl IS NULL + RAISERROR ('@Tbl IS NULL', 18, 127); + INSERT INTO @Indexes + SELECT Ind, + CASE WHEN is_primary_key = 1 THEN 'ALTER TABLE dbo.[' + Tbl + '] ADD PRIMARY KEY ' + CASE WHEN type = 1 THEN ' CLUSTERED' ELSE '' END ELSE 'CREATE' + CASE WHEN is_unique = 1 THEN ' UNIQUE' ELSE '' END + CASE WHEN type = 1 THEN ' CLUSTERED' ELSE '' END + ' INDEX ' + Ind + ' ON dbo.[' + Tbl + ']' END + ' (' + KeyCols + ')' + IncClause + CASE WHEN filter_def IS NOT NULL THEN ' WHERE ' + filter_def ELSE '' END + CASE WHEN data_comp IS NOT NULL THEN ' WITH (DATA_COMPRESSION = ' + data_comp + ')' ELSE '' END + CASE WHEN @AddPartClause = 1 THEN PartClause ELSE '' END + FROM (SELECT O.Name AS Tbl, + I.Name AS Ind, + isnull((SELECT TOP 1 CASE WHEN data_compression_desc = 'PAGE' THEN 'PAGE' END + FROM sys.partitions AS P + WHERE P.object_id = I.object_id + AND I.index_id = P.index_id), (SELECT NULLIF (PropertyValue, 'NONE') + FROM dbo.IndexProperties + WHERE TableName = O.Name + AND IndexName = I.Name + AND PropertyName = 'DATA_COMPRESSION')) AS data_comp, + replace(replace(replace(replace(I.filter_definition, '[', ''), ']', ''), '(', ''), ')', '') AS filter_def, + I.is_unique, + I.is_primary_key, + I.type, + KeyCols, + CASE WHEN IncCols IS NOT NULL THEN ' INCLUDE (' + IncCols + ')' ELSE '' END AS IncClause, + CASE WHEN EXISTS (SELECT * + FROM sys.partition_schemes AS S + WHERE S.data_space_id = I.data_space_id + AND name = 'PartitionScheme_ResourceTypeId') THEN ' ON PartitionScheme_ResourceTypeId (ResourceTypeId)' ELSE '' END AS PartClause + FROM sys.indexes AS I + INNER JOIN + sys.objects AS O + ON O.object_id = I.object_id CROSS APPLY (SELECT string_agg(CASE WHEN IC.key_ordinal > 0 + AND IC.is_included_column = 0 THEN C.name END, ',') WITHIN GROUP (ORDER BY key_ordinal) AS KeyCols, + string_agg(CASE WHEN IC.is_included_column = 1 THEN C.name END, ',') WITHIN GROUP (ORDER BY key_ordinal) AS IncCols + FROM sys.index_columns AS IC + INNER JOIN + sys.columns AS C + ON C.object_id = IC.object_id + AND C.column_id = IC.column_id + WHERE IC.object_id = I.object_id + AND IC.index_id = I.index_id + GROUP BY IC.object_id, IC.index_id) AS IC + WHERE O.name = @Tbl + AND (@Ind IS NULL + OR I.name = @Ind) + AND (@IncludeClustered = 1 + OR index_id > 1)) AS A; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@Indexes', @Action = 'Insert', @Rows = @@rowcount; + IF @Ind IS NULL + SELECT Ind, + Txt + FROM @Indexes; + ELSE + SET @Txt = (SELECT Txt + FROM @Indexes); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Text = @Txt; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.GetJobs +@QueueType TINYINT, @JobId BIGINT=NULL, @JobIds BigintList READONLY, @GroupId BIGINT=NULL, @ReturnDefinition BIT=1 +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'GetJobs', @Mode AS VARCHAR (100) = 'Q=' + isnull(CONVERT (VARCHAR, @QueueType), 'NULL') + ' J=' + isnull(CONVERT (VARCHAR, @JobId), 'NULL') + ' G=' + isnull(CONVERT (VARCHAR, @GroupId), 'NULL'), @st AS DATETIME = getUTCdate(), @PartitionId AS TINYINT = @JobId % 16; +BEGIN TRY + IF @JobId IS NULL + AND @GroupId IS NULL + AND NOT EXISTS (SELECT * + FROM @JobIds) + RAISERROR ('@JobId = NULL and @GroupId = NULL and @JobIds is empty', 18, 127); + IF @JobId IS NOT NULL + SELECT GroupId, + JobId, + CASE WHEN @ReturnDefinition = 1 THEN Definition ELSE NULL END AS Definition, + Version, + Status, + Priority, + Data, + Result, + CreateDate, + StartDate, + EndDate, + HeartbeatDate, + CancelRequested + FROM dbo.JobQueue + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND JobId = isnull(@JobId, -1) + AND Status <> 5; + ELSE + IF @GroupId IS NOT NULL + SELECT GroupId, + JobId, + CASE WHEN @ReturnDefinition = 1 THEN Definition ELSE NULL END AS Definition, + Version, + Status, + Priority, + Data, + Result, + CreateDate, + StartDate, + EndDate, + HeartbeatDate, + CancelRequested + FROM dbo.JobQueue WITH (INDEX (IX_QueueType_GroupId)) + WHERE QueueType = @QueueType + AND GroupId = isnull(@GroupId, -1) + AND Status <> 5; + ELSE + SELECT GroupId, + JobId, + CASE WHEN @ReturnDefinition = 1 THEN Definition ELSE NULL END AS Definition, + Version, + Status, + Priority, + Data, + Result, + CreateDate, + StartDate, + EndDate, + HeartbeatDate, + CancelRequested + FROM dbo.JobQueue + WHERE QueueType = @QueueType + AND JobId IN (SELECT Id + FROM @JobIds) + AND PartitionId = JobId % 16 + AND Status <> 5; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.GetPartitionedTables +@IncludeNotDisabled BIT=1, @IncludeNotSupported BIT=1 +WITH EXECUTE AS 'dbo' +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'GetPartitionedTables', @Mode AS VARCHAR (200) = 'PS=PartitionScheme_ResourceTypeId D=' + isnull(CONVERT (VARCHAR, @IncludeNotDisabled), 'NULL') + ' S=' + isnull(CONVERT (VARCHAR, @IncludeNotSupported), 'NULL'), @st AS DATETIME = getUTCdate(); +DECLARE @NotSupportedTables TABLE ( + id INT PRIMARY KEY); +BEGIN TRY + INSERT INTO @NotSupportedTables + SELECT DISTINCT O.object_id + FROM sys.indexes AS I + INNER JOIN + sys.objects AS O + ON O.object_id = I.object_id + WHERE O.type = 'u' + AND EXISTS (SELECT * + FROM sys.partition_schemes AS PS + WHERE PS.data_space_id = I.data_space_id + AND name = 'PartitionScheme_ResourceTypeId') + AND (NOT EXISTS (SELECT * + FROM sys.index_columns AS IC + INNER JOIN + sys.columns AS C + ON C.object_id = IC.object_id + AND C.column_id = IC.column_id + WHERE IC.object_id = I.object_id + AND IC.index_id = I.index_id + AND IC.key_ordinal > 0 + AND IC.is_included_column = 0 + AND C.name = 'ResourceTypeId') + OR EXISTS (SELECT * + FROM sys.indexes AS NSI + WHERE NSI.object_id = O.object_id + AND NOT EXISTS (SELECT * + FROM sys.partition_schemes AS PS + WHERE PS.data_space_id = NSI.data_space_id + AND name = 'PartitionScheme_ResourceTypeId'))); + SELECT CONVERT (VARCHAR (100), O.name), + CONVERT (BIT, CASE WHEN EXISTS (SELECT * + FROM @NotSupportedTables AS NSI + WHERE NSI.id = O.object_id) THEN 0 ELSE 1 END) + FROM sys.indexes AS I + INNER JOIN + sys.objects AS O + ON O.object_id = I.object_id + WHERE O.type = 'u' + AND I.index_id IN (0, 1) + AND EXISTS (SELECT * + FROM sys.partition_schemes AS PS + WHERE PS.data_space_id = I.data_space_id + AND name = 'PartitionScheme_ResourceTypeId') + AND EXISTS (SELECT * + FROM sys.index_columns AS IC + INNER JOIN + sys.columns AS C + ON C.object_id = I.object_id + AND C.column_id = IC.column_id + AND IC.is_included_column = 0 + AND C.name = 'ResourceTypeId') + AND (@IncludeNotSupported = 1 + OR NOT EXISTS (SELECT * + FROM @NotSupportedTables AS NSI + WHERE NSI.id = O.object_id)) + AND (@IncludeNotDisabled = 1 + OR EXISTS (SELECT * + FROM sys.indexes AS D + WHERE D.object_id = O.object_id + AND D.is_disabled = 1)) + ORDER BY 1; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.GetQuantityCodeId +@stringValue NVARCHAR (255) +AS +SET TRANSACTION ISOLATION LEVEL SERIALIZABLE; +BEGIN TRANSACTION; +DECLARE @id AS INT = (SELECT QuantityCodeId + FROM dbo.QuantityCode WITH (UPDLOCK) + WHERE Value = @stringValue); +IF (@id IS NULL) + BEGIN + INSERT INTO dbo.QuantityCode (Value) + VALUES (@stringValue); + SET @id = SCOPE_IDENTITY(); + END +COMMIT TRANSACTION; +SELECT @id; + +GO +CREATE PROCEDURE dbo.GetReindexJobById +@id VARCHAR (64) +AS +SET NOCOUNT ON; +SELECT RawJobRecord, + JobVersion +FROM dbo.ReindexJob +WHERE Id = @id; + +GO +CREATE PROCEDURE dbo.GetResources +@ResourceKeys dbo.ResourceKeyList READONLY +AS +SET NOCOUNT ON; +DECLARE @st AS DATETIME = getUTCdate(), @SP AS VARCHAR (100) = 'GetResources', @InputRows AS INT, @DummyTop AS BIGINT = 9223372036854775807, @NotNullVersionExists AS BIT, @NullVersionExists AS BIT, @MinRT AS SMALLINT, @MaxRT AS SMALLINT; +SELECT @MinRT = min(ResourceTypeId), + @MaxRT = max(ResourceTypeId), + @InputRows = count(*), + @NotNullVersionExists = max(CASE WHEN Version IS NOT NULL THEN 1 ELSE 0 END), + @NullVersionExists = max(CASE WHEN Version IS NULL THEN 1 ELSE 0 END) +FROM @ResourceKeys; +DECLARE @Mode AS VARCHAR (100) = 'RT=[' + CONVERT (VARCHAR, @MinRT) + ',' + CONVERT (VARCHAR, @MaxRT) + '] Cnt=' + CONVERT (VARCHAR, @InputRows) + ' NNVE=' + CONVERT (VARCHAR, @NotNullVersionExists) + ' NVE=' + CONVERT (VARCHAR, @NullVersionExists); +BEGIN TRY + IF @NotNullVersionExists = 1 + IF @NullVersionExists = 0 + SELECT B.ResourceTypeId, + B.ResourceId, + ResourceSurrogateId, + B.Version, + IsDeleted, + IsHistory, + RawResource, + IsRawResourceMetaSet, + SearchParamHash + FROM (SELECT TOP (@DummyTop) * + FROM @ResourceKeys) AS A + INNER JOIN + dbo.Resource AS B WITH (INDEX (IX_Resource_ResourceTypeId_ResourceId_Version)) + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceId = A.ResourceId + AND B.Version = A.Version + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + ELSE + SELECT * + FROM (SELECT B.ResourceTypeId, + B.ResourceId, + ResourceSurrogateId, + B.Version, + IsDeleted, + IsHistory, + RawResource, + IsRawResourceMetaSet, + SearchParamHash + FROM (SELECT TOP (@DummyTop) * + FROM @ResourceKeys + WHERE Version IS NOT NULL) AS A + INNER JOIN + dbo.Resource AS B WITH (INDEX (IX_Resource_ResourceTypeId_ResourceId_Version)) + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceId = A.ResourceId + AND B.Version = A.Version + UNION ALL + SELECT B.ResourceTypeId, + B.ResourceId, + ResourceSurrogateId, + B.Version, + IsDeleted, + IsHistory, + RawResource, + IsRawResourceMetaSet, + SearchParamHash + FROM (SELECT TOP (@DummyTop) * + FROM @ResourceKeys + WHERE Version IS NULL) AS A + INNER JOIN + dbo.Resource AS B WITH (INDEX (IX_Resource_ResourceTypeId_ResourceId)) + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceId = A.ResourceId + WHERE IsHistory = 0) AS A + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + ELSE + SELECT B.ResourceTypeId, + B.ResourceId, + ResourceSurrogateId, + B.Version, + IsDeleted, + IsHistory, + RawResource, + IsRawResourceMetaSet, + SearchParamHash + FROM (SELECT TOP (@DummyTop) * + FROM @ResourceKeys) AS A + INNER JOIN + dbo.Resource AS B WITH (INDEX (IX_Resource_ResourceTypeId_ResourceId)) + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceId = A.ResourceId + WHERE IsHistory = 0 + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.GetResourcesByTokens +@ResourceTypeId SMALLINT, @SearchParamId SMALLINT, @Tokens dbo.TokenList READONLY, @Top INT +AS +SET NOCOUNT ON; +DECLARE @st AS DATETIME = getUTCdate(), @SP AS VARCHAR (100) = 'GetResourcesByTokens', @Mode AS VARCHAR (100) = 'RT=' + CONVERT (VARCHAR, @ResourceTypeId) + ' SP=' + CONVERT (VARCHAR, @SearchParamId) + ' Tokens=' + CONVERT (VARCHAR, (SELECT count(*) + FROM @Tokens)) + ' T=' + CONVERT (VARCHAR, @Top), @DummyTop AS BIGINT = 9223372036854775807; +BEGIN TRY + IF NOT EXISTS (SELECT * + FROM @Tokens + WHERE CodeOverflow IS NOT NULL + OR SystemValue IS NOT NULL) + SELECT ResourceTypeId, + ResourceId, + Version, + IsDeleted, + ResourceSurrogateId, + RequestMethod, + CONVERT (BIT, 1) AS IsMatch, + CONVERT (BIT, 0) AS IsPartial, + IsRawResourceMetaSet, + SearchParamHash, + RawResource + FROM (SELECT DISTINCT TOP (@Top) ResourceSurrogateId AS Sid1 + FROM (SELECT TOP (@DummyTop) * + FROM @Tokens) AS A + INNER JOIN + dbo.TokenSearchParam AS B + ON B.Code = A.Code + AND (B.SystemId = A.SystemId + OR A.SystemId IS NULL) + WHERE ResourceTypeId = @ResourceTypeId + AND SearchParamId = @SearchParamId + ORDER BY ResourceSurrogateId) AS A + INNER JOIN + dbo.Resource + ON ResourceSurrogateId = Sid1 + WHERE ResourceTypeId = @ResourceTypeId + ORDER BY ResourceSurrogateId + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + ELSE + IF NOT EXISTS (SELECT * + FROM @Tokens + WHERE CodeOverflow IS NOT NULL) + SELECT ResourceTypeId, + ResourceId, + Version, + IsDeleted, + ResourceSurrogateId, + RequestMethod, + CONVERT (BIT, 1) AS IsMatch, + CONVERT (BIT, 0) AS IsPartial, + IsRawResourceMetaSet, + SearchParamHash, + RawResource + FROM (SELECT DISTINCT TOP (@Top) ResourceSurrogateId AS Sid1 + FROM (SELECT TOP (@DummyTop) Code, + CodeOverflow, + CASE WHEN SystemValue IS NOT NULL THEN (SELECT SystemId + FROM dbo.System + WHERE Value = SystemValue) ELSE SystemId END AS SystemId, + SystemValue + FROM @Tokens) AS A + INNER JOIN + dbo.TokenSearchParam AS B + ON B.Code = A.Code + AND (B.SystemId = A.SystemId + OR A.SystemId IS NULL + AND A.SystemValue IS NULL) + WHERE ResourceTypeId = @ResourceTypeId + AND SearchParamId = @SearchParamId + ORDER BY ResourceSurrogateId) AS A + INNER JOIN + dbo.Resource + ON ResourceSurrogateId = Sid1 + WHERE ResourceTypeId = @ResourceTypeId + ORDER BY ResourceSurrogateId + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + ELSE + SELECT ResourceTypeId, + ResourceId, + Version, + IsDeleted, + ResourceSurrogateId, + RequestMethod, + CONVERT (BIT, 1) AS IsMatch, + CONVERT (BIT, 0) AS IsPartial, + IsRawResourceMetaSet, + SearchParamHash, + RawResource + FROM (SELECT DISTINCT TOP (@Top) ResourceSurrogateId AS Sid1 + FROM (SELECT TOP (@DummyTop) Code, + CodeOverflow, + CASE WHEN SystemValue IS NOT NULL THEN (SELECT SystemId + FROM dbo.System + WHERE Value = SystemValue) ELSE SystemId END AS SystemId, + SystemValue + FROM @Tokens) AS A + INNER JOIN + dbo.TokenSearchParam AS B + ON B.Code = A.Code + AND (B.CodeOverflow = A.CodeOverflow + OR B.CodeOverflow IS NULL + AND A.CodeOverflow IS NULL) + AND (B.SystemId = A.SystemId + OR A.SystemId IS NULL + AND A.SystemValue IS NULL) + WHERE ResourceTypeId = @ResourceTypeId + AND SearchParamId = @SearchParamId + ORDER BY ResourceSurrogateId) AS A + INNER JOIN + dbo.Resource + ON ResourceSurrogateId = Sid1 + WHERE ResourceTypeId = @ResourceTypeId + ORDER BY ResourceSurrogateId + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.GetResourcesByTransactionId +@TransactionId BIGINT, @IncludeHistory BIT=0, @ReturnResourceKeysOnly BIT=0 +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = object_name(@@procid), @Mode AS VARCHAR (100) = 'T=' + CONVERT (VARCHAR, @TransactionId) + ' H=' + CONVERT (VARCHAR, @IncludeHistory), @st AS DATETIME = getUTCdate(), @DummyTop AS BIGINT = 9223372036854775807, @TypeId AS SMALLINT; +BEGIN TRY + DECLARE @Types TABLE ( + TypeId SMALLINT PRIMARY KEY, + Name VARCHAR (100)); + INSERT INTO @Types + EXECUTE dbo.GetUsedResourceTypes ; + DECLARE @Keys TABLE ( + TypeId SMALLINT, + SurrogateId BIGINT PRIMARY KEY (TypeId, SurrogateId)); + WHILE EXISTS (SELECT * + FROM @Types) + BEGIN + SET @TypeId = (SELECT TOP 1 TypeId + FROM @Types + ORDER BY TypeId); + INSERT INTO @Keys + SELECT @TypeId, + ResourceSurrogateId + FROM dbo.Resource + WHERE ResourceTypeId = @TypeId + AND TransactionId = @TransactionId; + DELETE @Types + WHERE TypeId = @TypeId; + END + IF @ReturnResourceKeysOnly = 0 + SELECT ResourceTypeId, + ResourceId, + ResourceSurrogateId, + Version, + IsDeleted, + IsHistory, + RawResource, + IsRawResourceMetaSet, + SearchParamHash, + RequestMethod + FROM (SELECT TOP (@DummyTop) * + FROM @Keys) AS A + INNER JOIN + dbo.Resource AS B + ON ResourceTypeId = TypeId + AND ResourceSurrogateId = SurrogateId + WHERE IsHistory = 0 + OR @IncludeHistory = 1 + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + ELSE + SELECT ResourceTypeId, + ResourceId, + ResourceSurrogateId, + Version, + IsDeleted + FROM (SELECT TOP (@DummyTop) * + FROM @Keys) AS A + INNER JOIN + dbo.Resource AS B + ON ResourceTypeId = TypeId + AND ResourceSurrogateId = SurrogateId + WHERE IsHistory = 0 + OR @IncludeHistory = 1 + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.GetResourcesByTypeAndSurrogateIdRange +@ResourceTypeId SMALLINT, @StartId BIGINT, @EndId BIGINT, @GlobalEndId BIGINT=NULL, @IncludeHistory BIT=0, @IncludeDeleted BIT=0 +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'GetResourcesByTypeAndSurrogateIdRange', @Mode AS VARCHAR (100) = 'RT=' + isnull(CONVERT (VARCHAR, @ResourceTypeId), 'NULL') + ' S=' + isnull(CONVERT (VARCHAR, @StartId), 'NULL') + ' E=' + isnull(CONVERT (VARCHAR, @EndId), 'NULL') + ' GE=' + isnull(CONVERT (VARCHAR, @GlobalEndId), 'NULL') + ' HI=' + isnull(CONVERT (VARCHAR, @IncludeHistory), 'NULL') + ' DE' + isnull(CONVERT (VARCHAR, @IncludeDeleted), 'NULL'), @st AS DATETIME = getUTCdate(), @DummyTop AS BIGINT = 9223372036854775807; +BEGIN TRY + DECLARE @ResourceIds TABLE ( + ResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS PRIMARY KEY); + DECLARE @SurrogateIds TABLE ( + MaxSurrogateId BIGINT PRIMARY KEY); + IF @GlobalEndId IS NOT NULL + AND @IncludeHistory = 0 + BEGIN + INSERT INTO @ResourceIds + SELECT DISTINCT ResourceId + FROM dbo.Resource + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId BETWEEN @StartId AND @EndId + AND IsHistory = 1 + AND (IsDeleted = 0 + OR @IncludeDeleted = 1) + OPTION (MAXDOP 1); + IF @@rowcount > 0 + INSERT INTO @SurrogateIds + SELECT ResourceSurrogateId + FROM (SELECT ResourceId, + ResourceSurrogateId, + row_number() OVER (PARTITION BY ResourceId ORDER BY ResourceSurrogateId DESC) AS RowId + FROM dbo.Resource WITH (INDEX (IX_Resource_ResourceTypeId_ResourceId_Version)) + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceId IN (SELECT TOP (@DummyTop) ResourceId + FROM @ResourceIds) + AND ResourceSurrogateId BETWEEN @StartId AND @GlobalEndId) AS A + WHERE RowId = 1 + AND ResourceSurrogateId BETWEEN @StartId AND @EndId + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + END + SELECT ResourceTypeId, + ResourceId, + Version, + IsDeleted, + ResourceSurrogateId, + RequestMethod, + CONVERT (BIT, 1) AS IsMatch, + CONVERT (BIT, 0) AS IsPartial, + IsRawResourceMetaSet, + SearchParamHash, + RawResource, + IsHistory + FROM dbo.Resource + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId BETWEEN @StartId AND @EndId + AND (IsHistory = 0 + OR @IncludeHistory = 1) + AND (IsDeleted = 0 + OR @IncludeDeleted = 1) + UNION ALL + SELECT ResourceTypeId, + ResourceId, + Version, + IsDeleted, + ResourceSurrogateId, + RequestMethod, + CONVERT (BIT, 1) AS IsMatch, + CONVERT (BIT, 0) AS IsPartial, + IsRawResourceMetaSet, + SearchParamHash, + RawResource, + IsHistory + FROM @SurrogateIds + INNER JOIN + dbo.Resource + ON ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId = MaxSurrogateId + WHERE IsHistory = 1 + AND (IsDeleted = 0 + OR @IncludeDeleted = 1) + OPTION (MAXDOP 1); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.GetResourceSearchParamStats +@Table VARCHAR (100)=NULL, @ResourceTypeId SMALLINT=NULL, @SearchParamId SMALLINT=NULL +WITH EXECUTE AS 'dbo' +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = object_name(@@procid), @Mode AS VARCHAR (200) = 'T=' + isnull(@Table, 'NULL') + ' RT=' + isnull(CONVERT (VARCHAR, @ResourceTypeId), 'NULL') + ' SP=' + isnull(CONVERT (VARCHAR, @SearchParamId), 'NULL'), @st AS DATETIME = getUTCdate(); +BEGIN TRY + SELECT T.name AS TableName, + S.name AS StatsName, + db_name() AS DatabaseName + FROM sys.stats AS S + INNER JOIN + sys.tables AS T + ON T.object_id = S.object_id + WHERE T.name LIKE '%SearchParam' + AND T.name <> 'SearchParam' + AND S.name LIKE 'ST[_]%' + AND (T.name LIKE @Table + OR @Table IS NULL) + AND (S.name LIKE '%ResourceTypeId[_]' + CONVERT (VARCHAR, @ResourceTypeId) + '[_]%' + OR @ResourceTypeId IS NULL) + AND (S.name LIKE '%SearchParamId[_]' + CONVERT (VARCHAR, @SearchParamId) + OR @SearchParamId IS NULL); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Rows = @@rowcount, @Start = @st; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.GetResourceSurrogateIdRanges +@ResourceTypeId SMALLINT, @StartId BIGINT, @EndId BIGINT, @RangeSize INT, @NumberOfRanges INT=100, @Up BIT=1, @ActiveOnly BIT=0 +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'GetResourceSurrogateIdRanges', @Mode AS VARCHAR (100) = 'RT=' + isnull(CONVERT (VARCHAR, @ResourceTypeId), 'NULL') + ' S=' + isnull(CONVERT (VARCHAR, @StartId), 'NULL') + ' E=' + isnull(CONVERT (VARCHAR, @EndId), 'NULL') + ' R=' + isnull(CONVERT (VARCHAR, @RangeSize), 'NULL') + ' UP=' + isnull(CONVERT (VARCHAR, @Up), 'NULL') + ' AO=' + isnull(CONVERT (VARCHAR, @ActiveOnly), 'NULL'), @st AS DATETIME = getUTCdate(); +BEGIN TRY + IF @Up = 1 + SELECT RangeId, + min(ResourceSurrogateId), + max(ResourceSurrogateId), + count(*) + FROM (SELECT isnull(CONVERT (INT, (row_number() OVER (ORDER BY ResourceSurrogateId) - 1) / @RangeSize), 0) AS RangeId, + ResourceSurrogateId + FROM (SELECT TOP (@RangeSize * @NumberOfRanges) ResourceSurrogateId + FROM dbo.Resource + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId >= @StartId + AND ResourceSurrogateId <= @EndId + AND (@ActiveOnly = 0 + OR (IsHistory = 0 + AND IsDeleted = 0)) + ORDER BY ResourceSurrogateId) AS A) AS A + GROUP BY RangeId + OPTION (MAXDOP 1); + ELSE + SELECT RangeId, + min(ResourceSurrogateId), + max(ResourceSurrogateId), + count(*) + FROM (SELECT isnull(CONVERT (INT, (row_number() OVER (ORDER BY ResourceSurrogateId) - 1) / @RangeSize), 0) AS RangeId, + ResourceSurrogateId + FROM (SELECT TOP (@RangeSize * @NumberOfRanges) ResourceSurrogateId + FROM dbo.Resource + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId >= @StartId + AND ResourceSurrogateId <= @EndId + AND (@ActiveOnly = 0 + OR (IsHistory = 0 + AND IsDeleted = 0)) + ORDER BY ResourceSurrogateId DESC) AS A) AS A + GROUP BY RangeId + OPTION (MAXDOP 1); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.GetResourceVersions +@ResourceDateKeys dbo.ResourceDateKeyList READONLY +AS +SET NOCOUNT ON; +DECLARE @st AS DATETIME = getUTCdate(), @SP AS VARCHAR (100) = 'GetResourceVersions', @Mode AS VARCHAR (100) = 'Rows=' + CONVERT (VARCHAR, (SELECT count(*) + FROM @ResourceDateKeys)), @DummyTop AS BIGINT = 9223372036854775807; +BEGIN TRY + SELECT A.ResourceTypeId, + A.ResourceId, + A.ResourceSurrogateId, + CASE WHEN D.Version IS NOT NULL THEN 0 WHEN isnull(U.Version, 1) - isnull(L.Version, 0) > ResourceIndex THEN isnull(U.Version, 1) - ResourceIndex ELSE isnull(M.Version, 0) - ResourceIndex END AS Version, + isnull(D.Version, 0) AS MatchedVersion, + D.RawResource AS MatchedRawResource + FROM (SELECT TOP (@DummyTop) *, + CONVERT (INT, row_number() OVER (PARTITION BY ResourceTypeId, ResourceId ORDER BY ResourceSurrogateId DESC)) AS ResourceIndex + FROM @ResourceDateKeys) AS A OUTER APPLY (SELECT TOP 1 * + FROM dbo.Resource AS B WITH (INDEX (IX_Resource_ResourceTypeId_ResourceId_Version)) + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceId = A.ResourceId + AND B.Version > 0 + AND B.ResourceSurrogateId < A.ResourceSurrogateId + ORDER BY B.ResourceSurrogateId DESC) AS L OUTER APPLY (SELECT TOP 1 * + FROM dbo.Resource AS B WITH (INDEX (IX_Resource_ResourceTypeId_ResourceId_Version)) + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceId = A.ResourceId + AND B.Version > 0 + AND B.ResourceSurrogateId > A.ResourceSurrogateId + ORDER BY B.ResourceSurrogateId) AS U OUTER APPLY (SELECT TOP 1 * + FROM dbo.Resource AS B WITH (INDEX (IX_Resource_ResourceTypeId_ResourceId_Version)) + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceId = A.ResourceId + AND B.Version < 0 + ORDER BY B.Version) AS M OUTER APPLY (SELECT TOP 1 * + FROM dbo.Resource AS B WITH (INDEX (IX_Resource_ResourceTypeId_ResourceId_Version)) + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceId = A.ResourceId + AND B.ResourceSurrogateId BETWEEN A.ResourceSurrogateId AND A.ResourceSurrogateId + 79999) AS D + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.GetSearchParamMaxLastUpdated +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = object_name(@@procid), @Mode AS VARCHAR (200) = 'SearchParam MaxLastUpdated Query', @st AS DATETIME = getUTCdate(), @MaxLastUpdated AS DATETIMEOFFSET (7); +BEGIN TRY + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start', @Start = @st; + SELECT @MaxLastUpdated = MAX(LastUpdated) + FROM dbo.SearchParam; + SELECT @MaxLastUpdated AS MaxLastUpdated; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@ROWCOUNT; +END TRY +BEGIN CATCH + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.GetSearchParamStatuses +@StartLastUpdated DATETIMEOFFSET (7)=NULL, @LastUpdated DATETIMEOFFSET (7)=NULL OUTPUT +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'GetSearchParamStatuses', @Mode AS VARCHAR (100) = 'S=' + isnull(substring(CONVERT (VARCHAR, @StartLastUpdated), 1, 23), 'NULL'), @st AS DATETIME = getUTCdate(), @msg AS VARCHAR (100), @Rows AS INT; +BEGIN TRY + SET TRANSACTION ISOLATION LEVEL REPEATABLE READ; + BEGIN TRANSACTION; + SET @LastUpdated = (SELECT max(LastUpdated) + FROM dbo.SearchParam); + SET @msg = 'LastUpdated=' + substring(CONVERT (VARCHAR, @LastUpdated), 1, 23); + IF @StartLastUpdated IS NULL + SELECT SearchParamId, + Uri, + Status, + LastUpdated, + IsPartiallySupported + FROM dbo.SearchParam; + ELSE + SELECT SearchParamId, + Uri, + Status, + LastUpdated, + IsPartiallySupported + FROM dbo.SearchParam + WHERE LastUpdated > @StartLastUpdated; + SET @Rows = @@rowcount; + COMMIT TRANSACTION; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows, @Action = 'Select', @Target = 'SearchParam', @Text = @msg; +END TRY +BEGIN CATCH + IF @@trancount > 0 + ROLLBACK; + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + + +GO +INSERT INTO dbo.Parameters (Id, Char) +SELECT 'GetSearchParamStatuses', + 'LogEvent'; + +GO +CREATE PROCEDURE dbo.GetSystemId +@stringValue NVARCHAR (255) +AS +SET TRANSACTION ISOLATION LEVEL SERIALIZABLE; +BEGIN TRANSACTION; +DECLARE @id AS INT = (SELECT SystemId + FROM dbo.System WITH (UPDLOCK) + WHERE Value = @stringValue); +IF (@id IS NULL) + BEGIN + INSERT INTO dbo.System (Value) + VALUES (@stringValue); + SET @id = SCOPE_IDENTITY(); + END +COMMIT TRANSACTION; +SELECT @id; + +GO +CREATE PROCEDURE dbo.GetTransactions +@StartNotInclusiveTranId BIGINT, @EndInclusiveTranId BIGINT, @EndDate DATETIME=NULL +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = object_name(@@procid), @Mode AS VARCHAR (100) = 'ST=' + CONVERT (VARCHAR, @StartNotInclusiveTranId) + ' ET=' + CONVERT (VARCHAR, @EndInclusiveTranId) + ' ED=' + isnull(CONVERT (VARCHAR, @EndDate, 121), 'NULL'), @st AS DATETIME = getUTCdate(); +IF @EndDate IS NULL + SET @EndDate = getUTCdate(); +SELECT TOP 10000 SurrogateIdRangeFirstValue, + VisibleDate, + InvisibleHistoryRemovedDate +FROM dbo.Transactions +WHERE SurrogateIdRangeFirstValue > @StartNotInclusiveTranId + AND SurrogateIdRangeFirstValue <= @EndInclusiveTranId + AND EndDate <= @EndDate +ORDER BY SurrogateIdRangeFirstValue; +EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; + +GO +CREATE PROCEDURE dbo.GetUsedResourceTypes +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'GetUsedResourceTypes', @Mode AS VARCHAR (100) = '', @st AS DATETIME = getUTCdate(); +BEGIN TRY + SELECT ResourceTypeId, + Name + FROM dbo.ResourceType AS A + WHERE EXISTS (SELECT * + FROM dbo.Resource AS B + WHERE B.ResourceTypeId = A.ResourceTypeId); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.HardDeleteResource +@ResourceTypeId SMALLINT, @ResourceId VARCHAR (64), @KeepCurrentVersion BIT, @IsResourceChangeCaptureEnabled BIT +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = object_name(@@procid), @Mode AS VARCHAR (200) = 'RT=' + CONVERT (VARCHAR, @ResourceTypeId) + ' R=' + @ResourceId + ' V=' + CONVERT (VARCHAR, @KeepCurrentVersion) + ' CC=' + CONVERT (VARCHAR, @IsResourceChangeCaptureEnabled), @st AS DATETIME = getUTCdate(), @TransactionId AS BIGINT; +BEGIN TRY + IF @IsResourceChangeCaptureEnabled = 1 + EXECUTE dbo.MergeResourcesBeginTransaction @Count = 1, @TransactionId = @TransactionId OUTPUT; + IF @KeepCurrentVersion = 0 + BEGIN TRANSACTION; + DECLARE @SurrogateIds TABLE ( + ResourceSurrogateId BIGINT NOT NULL); + IF @IsResourceChangeCaptureEnabled = 1 + AND NOT EXISTS (SELECT * + FROM dbo.Parameters + WHERE Id = 'InvisibleHistory.IsEnabled' + AND Number = 0) + UPDATE dbo.Resource + SET IsDeleted = 1, + RawResource = 0xF, + SearchParamHash = NULL, + HistoryTransactionId = @TransactionId + OUTPUT deleted.ResourceSurrogateId INTO @SurrogateIds + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceId = @ResourceId + AND (@KeepCurrentVersion = 0 + OR IsHistory = 1) + AND RawResource <> 0xF; + ELSE + DELETE dbo.Resource + OUTPUT deleted.ResourceSurrogateId INTO @SurrogateIds + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceId = @ResourceId + AND (@KeepCurrentVersion = 0 + OR IsHistory = 1) + AND RawResource <> 0xF; + IF @KeepCurrentVersion = 0 + BEGIN + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.ResourceWriteClaim AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.ReferenceSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.TokenSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.TokenText AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.StringSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.UriSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.NumberSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.QuantitySearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.DateTimeSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.ReferenceTokenCompositeSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.TokenTokenCompositeSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.TokenDateTimeCompositeSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.TokenQuantityCompositeSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.TokenStringCompositeSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.TokenNumberNumberCompositeSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + END + IF @@trancount > 0 + COMMIT TRANSACTION; + IF @IsResourceChangeCaptureEnabled = 1 + EXECUTE dbo.MergeResourcesCommitTransaction @TransactionId; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st; +END TRY +BEGIN CATCH + IF @@trancount > 0 + ROLLBACK; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.InitializeBase +@searchParams NVARCHAR (MAX), @resourceTypes NVARCHAR (3000), @claimTypes VARCHAR (100), @compartmentTypes VARCHAR (100) +AS +SET XACT_ABORT ON; +BEGIN TRANSACTION; +INSERT INTO dbo.ResourceType (Name) +SELECT value +FROM string_split (@resourceTypes, ',') +EXCEPT +SELECT Name +FROM dbo.ResourceType WITH (TABLOCKX); +SELECT ResourceTypeId, + Name +FROM dbo.ResourceType; +WITH Input +AS (SELECT DISTINCT j.Uri, + CAST (j.IsPartiallySupported AS BIT) AS IsPartiallySupported + FROM OPENJSON (@searchParams) WITH (Uri VARCHAR (128) '$.Uri', IsPartiallySupported BIT '$.IsPartiallySupported') AS j) +INSERT dbo.SearchParam (Uri, Status, LastUpdated, IsPartiallySupported) +SELECT i.Uri, + 'Initialized', + SYSDATETIMEOFFSET(), + i.IsPartiallySupported +FROM Input AS i +WHERE NOT EXISTS (SELECT 1 + FROM dbo.SearchParam AS sp + WHERE sp.Uri = i.Uri); +SELECT Uri, + SearchParamId +FROM dbo.SearchParam; +INSERT INTO dbo.ClaimType (Name) +SELECT value +FROM string_split (@claimTypes, ',') +EXCEPT +SELECT Name +FROM dbo.ClaimType; +SELECT ClaimTypeId, + Name +FROM dbo.ClaimType; +INSERT INTO dbo.CompartmentType (Name) +SELECT value +FROM string_split (@compartmentTypes, ',') +EXCEPT +SELECT Name +FROM dbo.CompartmentType; +SELECT CompartmentTypeId, + Name +FROM dbo.CompartmentType; +COMMIT TRANSACTION; +SELECT Value, + SystemId +FROM dbo.System; +SELECT Value, + QuantityCodeId +FROM dbo.QuantityCode; + +GO +CREATE PROCEDURE dbo.InitializeIndexProperties +AS +SET NOCOUNT ON; +INSERT INTO dbo.IndexProperties (TableName, IndexName, PropertyName, PropertyValue) +SELECT Tbl, + Ind, + 'DATA_COMPRESSION', + isnull(data_comp, 'NONE') +FROM (SELECT O.Name AS Tbl, + I.Name AS Ind, + (SELECT TOP 1 CASE WHEN data_compression_desc = 'PAGE' THEN 'PAGE' END + FROM sys.partitions AS P + WHERE P.object_id = I.object_id + AND I.index_id = P.index_id) AS data_comp + FROM sys.indexes AS I + INNER JOIN + sys.objects AS O + ON O.object_id = I.object_id + WHERE O.type = 'u' + AND EXISTS (SELECT * + FROM sys.partition_schemes AS PS + WHERE PS.data_space_id = I.data_space_id + AND name = 'PartitionScheme_ResourceTypeId')) AS A +WHERE NOT EXISTS (SELECT * + FROM dbo.IndexProperties + WHERE TableName = Tbl + AND IndexName = Ind); + +GO +CREATE PROCEDURE dbo.LogEvent +@Process VARCHAR (100), @Status VARCHAR (10), @Mode VARCHAR (200)=NULL, @Action VARCHAR (20)=NULL, @Target VARCHAR (100)=NULL, @Rows BIGINT=NULL, @Start DATETIME=NULL, @Text NVARCHAR (3500)=NULL, @EventId BIGINT=NULL OUTPUT, @Retry INT=NULL +AS +SET NOCOUNT ON; +DECLARE @ErrorNumber AS INT = error_number(), @ErrorMessage AS VARCHAR (1000) = '', @TranCount AS INT = @@trancount, @DoWork AS BIT = 0, @NumberAdded AS BIT; +IF @ErrorNumber IS NOT NULL + OR @Status IN ('Warn', 'Error') + SET @DoWork = 1; +IF @DoWork = 0 + SET @DoWork = CASE WHEN EXISTS (SELECT * + FROM dbo.Parameters + WHERE Id = isnull(@Process, '') + AND Char = 'LogEvent') THEN 1 ELSE 0 END; +IF @DoWork = 0 + RETURN; +IF @ErrorNumber IS NOT NULL + SET @ErrorMessage = CASE WHEN @Retry IS NOT NULL THEN 'Retry ' + CONVERT (VARCHAR, @Retry) + ', ' ELSE '' END + 'Error ' + CONVERT (VARCHAR, error_number()) + ': ' + CONVERT (VARCHAR (1000), error_message()) + ', Level ' + CONVERT (VARCHAR, error_severity()) + ', State ' + CONVERT (VARCHAR, error_state()) + CASE WHEN error_procedure() IS NOT NULL THEN ', Procedure ' + error_procedure() ELSE '' END + ', Line ' + CONVERT (VARCHAR, error_line()); +IF @TranCount > 0 + AND @ErrorNumber IS NOT NULL + ROLLBACK; +IF databasepropertyex(db_name(), 'UpdateAbility') = 'READ_WRITE' + BEGIN + INSERT INTO dbo.EventLog (Process, Status, Mode, Action, Target, Rows, Milliseconds, EventDate, EventText, SPID, HostName) + SELECT @Process, + @Status, + @Mode, + @Action, + @Target, + @Rows, + datediff(millisecond, @Start, getUTCdate()), + getUTCdate() AS EventDate, + CASE WHEN @ErrorNumber IS NULL THEN @Text ELSE @ErrorMessage + CASE WHEN isnull(@Text, '') <> '' THEN '. ' + @Text ELSE '' END END AS Text, + @@SPID, + host_name() AS HostName; + SET @EventId = scope_identity(); + END +IF @TranCount > 0 + AND @ErrorNumber IS NOT NULL + BEGIN TRANSACTION; + +GO +CREATE PROCEDURE dbo.LogSchemaMigrationProgress +@message VARCHAR (MAX) +AS +INSERT INTO dbo.SchemaMigrationProgress (Message) +VALUES (@message); + +GO +CREATE PROCEDURE dbo.MergeResources +@AffectedRows INT=0 OUTPUT, @RaiseExceptionOnConflict BIT=1, @IsResourceChangeCaptureEnabled BIT=0, @TransactionId BIGINT=NULL, @SingleTransaction BIT=1, @Resources dbo.ResourceList READONLY, @ResourceWriteClaims dbo.ResourceWriteClaimList READONLY, @ReferenceSearchParams dbo.ReferenceSearchParamList READONLY, @TokenSearchParams dbo.TokenSearchParamList READONLY, @TokenTexts dbo.TokenTextList READONLY, @StringSearchParams dbo.StringSearchParamList READONLY, @UriSearchParams dbo.UriSearchParamList READONLY, @NumberSearchParams dbo.NumberSearchParamList READONLY, @QuantitySearchParams dbo.QuantitySearchParamList READONLY, @DateTimeSearchParms dbo.DateTimeSearchParamList READONLY, @ReferenceTokenCompositeSearchParams dbo.ReferenceTokenCompositeSearchParamList READONLY, @TokenTokenCompositeSearchParams dbo.TokenTokenCompositeSearchParamList READONLY, @TokenDateTimeCompositeSearchParams dbo.TokenDateTimeCompositeSearchParamList READONLY, @TokenQuantityCompositeSearchParams dbo.TokenQuantityCompositeSearchParamList READONLY, @TokenStringCompositeSearchParams dbo.TokenStringCompositeSearchParamList READONLY, @TokenNumberNumberCompositeSearchParams dbo.TokenNumberNumberCompositeSearchParamList READONLY +AS +SET NOCOUNT ON; +DECLARE @st AS DATETIME = getUTCdate(), @SP AS VARCHAR (100) = object_name(@@procid), @DummyTop AS BIGINT = 9223372036854775807, @InitialTranCount AS INT = @@trancount, @IsRetry AS BIT = 0; +DECLARE @Mode AS VARCHAR (200) = isnull((SELECT 'RT=[' + CONVERT (VARCHAR, min(ResourceTypeId)) + ',' + CONVERT (VARCHAR, max(ResourceTypeId)) + '] Sur=[' + CONVERT (VARCHAR, min(ResourceSurrogateId)) + ',' + CONVERT (VARCHAR, max(ResourceSurrogateId)) + '] V=' + CONVERT (VARCHAR, max(Version)) + ' Rows=' + CONVERT (VARCHAR, count(*)) + FROM @Resources), 'Input=Empty'); +SET @Mode += ' E=' + CONVERT (VARCHAR, @RaiseExceptionOnConflict) + ' CC=' + CONVERT (VARCHAR, @IsResourceChangeCaptureEnabled) + ' IT=' + CONVERT (VARCHAR, @InitialTranCount) + ' T=' + isnull(CONVERT (VARCHAR, @TransactionId), 'NULL') + ' ST=' + CONVERT (VARCHAR, @SingleTransaction); +SET @AffectedRows = 0; +BEGIN TRY + DECLARE @Existing AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + SurrogateId BIGINT NOT NULL PRIMARY KEY (ResourceTypeId, SurrogateId)); + DECLARE @ResourceInfos AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + SurrogateId BIGINT NOT NULL, + Version INT NOT NULL, + KeepHistory BIT NOT NULL, + PreviousVersion INT NULL, + PreviousSurrogateId BIGINT NULL PRIMARY KEY (ResourceTypeId, SurrogateId)); + DECLARE @PreviousSurrogateIds AS TABLE ( + TypeId SMALLINT NOT NULL, + SurrogateId BIGINT NOT NULL PRIMARY KEY (TypeId, SurrogateId), + KeepHistory BIT ); + IF @InitialTranCount = 0 + BEGIN + IF EXISTS (SELECT * + FROM @Resources AS A + INNER JOIN + dbo.Resource AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId) + BEGIN + BEGIN TRANSACTION; + INSERT INTO @Existing (ResourceTypeId, SurrogateId) + SELECT B.ResourceTypeId, + B.ResourceSurrogateId + FROM (SELECT TOP (@DummyTop) * + FROM @Resources) AS A + INNER JOIN + dbo.Resource AS B WITH (ROWLOCK, HOLDLOCK) + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + WHERE B.IsHistory = 0 + AND B.ResourceId = A.ResourceId + AND B.Version = A.Version + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + IF @@rowcount = (SELECT count(*) + FROM @Resources) + SET @IsRetry = 1; + IF @IsRetry = 0 + COMMIT TRANSACTION; + END + END + SET @Mode += ' R=' + CONVERT (VARCHAR, @IsRetry); + IF @SingleTransaction = 1 + AND @@trancount = 0 + BEGIN TRANSACTION; + IF @IsRetry = 0 + BEGIN + INSERT INTO @ResourceInfos (ResourceTypeId, SurrogateId, Version, KeepHistory, PreviousVersion, PreviousSurrogateId) + SELECT A.ResourceTypeId, + A.ResourceSurrogateId, + A.Version, + A.KeepHistory, + B.Version, + B.ResourceSurrogateId + FROM (SELECT TOP (@DummyTop) * + FROM @Resources + WHERE HasVersionToCompare = 1) AS A + LEFT OUTER JOIN + dbo.Resource AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceId = A.ResourceId + AND B.IsHistory = 0 + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + IF @RaiseExceptionOnConflict = 1 + AND EXISTS (SELECT * + FROM @ResourceInfos + WHERE (PreviousVersion IS NOT NULL + AND Version <= PreviousVersion) + OR (PreviousSurrogateId IS NOT NULL + AND SurrogateId <= PreviousSurrogateId)) + THROW 50409, 'Resource has been recently updated or added, please compare the resource content in code for any duplicate updates', 1; + INSERT INTO @PreviousSurrogateIds + SELECT ResourceTypeId, + PreviousSurrogateId, + KeepHistory + FROM @ResourceInfos + WHERE PreviousSurrogateId IS NOT NULL; + IF @@rowcount > 0 + BEGIN + UPDATE dbo.Resource + SET IsHistory = 1 + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId + AND KeepHistory = 1); + SET @AffectedRows += @@rowcount; + IF @IsResourceChangeCaptureEnabled = 1 + AND NOT EXISTS (SELECT * + FROM dbo.Parameters + WHERE Id = 'InvisibleHistory.IsEnabled' + AND Number = 0) + UPDATE dbo.Resource + SET IsHistory = 1, + RawResource = 0xF, + SearchParamHash = NULL, + HistoryTransactionId = @TransactionId + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId + AND KeepHistory = 0); + ELSE + DELETE dbo.Resource + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId + AND KeepHistory = 0); + SET @AffectedRows += @@rowcount; + DELETE dbo.ResourceWriteClaim + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.ReferenceSearchParam + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.TokenSearchParam + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.TokenText + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.StringSearchParam + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.UriSearchParam + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.NumberSearchParam + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.QuantitySearchParam + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.DateTimeSearchParam + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.ReferenceTokenCompositeSearchParam + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.TokenTokenCompositeSearchParam + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.TokenDateTimeCompositeSearchParam + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.TokenQuantityCompositeSearchParam + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.TokenStringCompositeSearchParam + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.TokenNumberNumberCompositeSearchParam + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + END + INSERT INTO dbo.Resource (ResourceTypeId, ResourceId, Version, IsHistory, ResourceSurrogateId, IsDeleted, RequestMethod, RawResource, IsRawResourceMetaSet, SearchParamHash, TransactionId) + SELECT ResourceTypeId, + ResourceId, + Version, + IsHistory, + ResourceSurrogateId, + IsDeleted, + RequestMethod, + RawResource, + IsRawResourceMetaSet, + SearchParamHash, + @TransactionId + FROM @Resources; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.ResourceWriteClaim (ResourceSurrogateId, ClaimTypeId, ClaimValue) + SELECT ResourceSurrogateId, + ClaimTypeId, + ClaimValue + FROM @ResourceWriteClaims; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.ReferenceSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri, ReferenceResourceTypeId, ReferenceResourceId, ReferenceResourceVersion) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + BaseUri, + ReferenceResourceTypeId, + ReferenceResourceId, + ReferenceResourceVersion + FROM @ReferenceSearchParams; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, Code, CodeOverflow) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId, + Code, + CodeOverflow + FROM @TokenSearchParams; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenText (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + Text + FROM @TokenTexts; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.StringSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text, TextOverflow, IsMin, IsMax) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + Text, + TextOverflow, + IsMin, + IsMax + FROM @StringSearchParams; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.UriSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, Uri) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + Uri + FROM @UriSearchParams; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.NumberSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SingleValue, LowValue, HighValue) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SingleValue, + LowValue, + HighValue + FROM @NumberSearchParams; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.QuantitySearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, QuantityCodeId, SingleValue, LowValue, HighValue) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId, + QuantityCodeId, + SingleValue, + LowValue, + HighValue + FROM @QuantitySearchParams; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.DateTimeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, StartDateTime, EndDateTime, IsLongerThanADay, IsMin, IsMax) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + StartDateTime, + EndDateTime, + IsLongerThanADay, + IsMin, + IsMax + FROM @DateTimeSearchParms; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.ReferenceTokenCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri1, ReferenceResourceTypeId1, ReferenceResourceId1, ReferenceResourceVersion1, SystemId2, Code2, CodeOverflow2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + BaseUri1, + ReferenceResourceTypeId1, + ReferenceResourceId1, + ReferenceResourceVersion1, + SystemId2, + Code2, + CodeOverflow2 + FROM @ReferenceTokenCompositeSearchParams; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenTokenCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SystemId2, Code2, CodeOverflow2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SystemId2, + Code2, + CodeOverflow2 + FROM @TokenTokenCompositeSearchParams; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenDateTimeCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, StartDateTime2, EndDateTime2, IsLongerThanADay2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + StartDateTime2, + EndDateTime2, + IsLongerThanADay2 + FROM @TokenDateTimeCompositeSearchParams; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenQuantityCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, SystemId2, QuantityCodeId2, LowValue2, HighValue2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SingleValue2, + SystemId2, + QuantityCodeId2, + LowValue2, + HighValue2 + FROM @TokenQuantityCompositeSearchParams; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenStringCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, Text2, TextOverflow2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + Text2, + TextOverflow2 + FROM @TokenStringCompositeSearchParams; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenNumberNumberCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, LowValue2, HighValue2, SingleValue3, LowValue3, HighValue3, HasRange) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SingleValue2, + LowValue2, + HighValue2, + SingleValue3, + LowValue3, + HighValue3, + HasRange + FROM @TokenNumberNumberCompositeSearchParams; + SET @AffectedRows += @@rowcount; + END + ELSE + BEGIN + INSERT INTO dbo.ResourceWriteClaim (ResourceSurrogateId, ClaimTypeId, ClaimValue) + SELECT ResourceSurrogateId, + ClaimTypeId, + ClaimValue + FROM (SELECT TOP (@DummyTop) * + FROM @ResourceWriteClaims) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.ResourceWriteClaim AS C + WHERE C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.ReferenceSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri, ReferenceResourceTypeId, ReferenceResourceId, ReferenceResourceVersion) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + BaseUri, + ReferenceResourceTypeId, + ReferenceResourceId, + ReferenceResourceVersion + FROM (SELECT TOP (@DummyTop) * + FROM @ReferenceSearchParams) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.ReferenceSearchParam AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, Code, CodeOverflow) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId, + Code, + CodeOverflow + FROM (SELECT TOP (@DummyTop) * + FROM @TokenSearchParams) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.TokenSearchParam AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenText (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + Text + FROM (SELECT TOP (@DummyTop) * + FROM @TokenTexts) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.TokenText AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.StringSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text, TextOverflow, IsMin, IsMax) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + Text, + TextOverflow, + IsMin, + IsMax + FROM (SELECT TOP (@DummyTop) * + FROM @StringSearchParams) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.StringSearchParam AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.UriSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, Uri) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + Uri + FROM (SELECT TOP (@DummyTop) * + FROM @UriSearchParams) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.UriSearchParam AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.NumberSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SingleValue, LowValue, HighValue) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SingleValue, + LowValue, + HighValue + FROM (SELECT TOP (@DummyTop) * + FROM @NumberSearchParams) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.NumberSearchParam AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.QuantitySearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, QuantityCodeId, SingleValue, LowValue, HighValue) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId, + QuantityCodeId, + SingleValue, + LowValue, + HighValue + FROM (SELECT TOP (@DummyTop) * + FROM @QuantitySearchParams) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.QuantitySearchParam AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.DateTimeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, StartDateTime, EndDateTime, IsLongerThanADay, IsMin, IsMax) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + StartDateTime, + EndDateTime, + IsLongerThanADay, + IsMin, + IsMax + FROM (SELECT TOP (@DummyTop) * + FROM @DateTimeSearchParms) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.DateTimeSearchParam AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.ReferenceTokenCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri1, ReferenceResourceTypeId1, ReferenceResourceId1, ReferenceResourceVersion1, SystemId2, Code2, CodeOverflow2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + BaseUri1, + ReferenceResourceTypeId1, + ReferenceResourceId1, + ReferenceResourceVersion1, + SystemId2, + Code2, + CodeOverflow2 + FROM (SELECT TOP (@DummyTop) * + FROM @ReferenceTokenCompositeSearchParams) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.ReferenceTokenCompositeSearchParam AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenTokenCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SystemId2, Code2, CodeOverflow2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SystemId2, + Code2, + CodeOverflow2 + FROM (SELECT TOP (@DummyTop) * + FROM @TokenTokenCompositeSearchParams) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.TokenTokenCompositeSearchParam AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenDateTimeCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, StartDateTime2, EndDateTime2, IsLongerThanADay2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + StartDateTime2, + EndDateTime2, + IsLongerThanADay2 + FROM (SELECT TOP (@DummyTop) * + FROM @TokenDateTimeCompositeSearchParams) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.TokenDateTimeCompositeSearchParam AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenQuantityCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, SystemId2, QuantityCodeId2, LowValue2, HighValue2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SingleValue2, + SystemId2, + QuantityCodeId2, + LowValue2, + HighValue2 + FROM (SELECT TOP (@DummyTop) * + FROM @TokenQuantityCompositeSearchParams) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.TokenQuantityCompositeSearchParam AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenStringCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, Text2, TextOverflow2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + Text2, + TextOverflow2 + FROM (SELECT TOP (@DummyTop) * + FROM @TokenStringCompositeSearchParams) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.TokenStringCompositeSearchParam AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenNumberNumberCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, LowValue2, HighValue2, SingleValue3, LowValue3, HighValue3, HasRange) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SingleValue2, + LowValue2, + HighValue2, + SingleValue3, + LowValue3, + HighValue3, + HasRange + FROM (SELECT TOP (@DummyTop) * + FROM @TokenNumberNumberCompositeSearchParams) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.TokenNumberNumberCompositeSearchParam AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + END + IF @IsResourceChangeCaptureEnabled = 1 + EXECUTE dbo.CaptureResourceIdsForChanges @Resources; + IF @TransactionId IS NOT NULL + EXECUTE dbo.MergeResourcesCommitTransaction @TransactionId; + IF @InitialTranCount = 0 + AND @@trancount > 0 + COMMIT TRANSACTION; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @AffectedRows; +END TRY +BEGIN CATCH + IF @InitialTranCount = 0 + AND @@trancount > 0 + ROLLBACK; + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + IF @RaiseExceptionOnConflict = 1 + AND error_message() LIKE '%''dbo.Resource''%' + BEGIN + IF error_number() = 2601 + THROW 50409, 'Resource has been recently updated or added, please compare the resource content in code for any duplicate updates.', 1; + ELSE + IF error_number() = 2627 + THROW 50424, 'Cannot persit resource due to a conflict with duplicated keys. Check the volume of resource being submited for ingestion.', 1; + ELSE + THROW; + END + ELSE + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.MergeResourcesAdvanceTransactionVisibility +@AffectedRows INT=0 OUTPUT +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = object_name(@@procid), @Mode AS VARCHAR (100) = '', @st AS DATETIME = getUTCdate(), @msg AS VARCHAR (1000), @MaxTransactionId AS BIGINT, @MinTransactionId AS BIGINT, @MinNotCompletedTransactionId AS BIGINT, @CurrentTransactionId AS BIGINT; +SET @AffectedRows = 0; +BEGIN TRY + EXECUTE dbo.MergeResourcesGetTransactionVisibility @MinTransactionId OUTPUT; + SET @MinTransactionId += 1; + SET @CurrentTransactionId = (SELECT TOP 1 SurrogateIdRangeFirstValue + FROM dbo.Transactions + ORDER BY SurrogateIdRangeFirstValue DESC); + SET @MinNotCompletedTransactionId = isnull((SELECT TOP 1 SurrogateIdRangeFirstValue + FROM dbo.Transactions + WHERE IsCompleted = 0 + AND SurrogateIdRangeFirstValue BETWEEN @MinTransactionId AND @CurrentTransactionId + ORDER BY SurrogateIdRangeFirstValue), @CurrentTransactionId + 1); + SET @MaxTransactionId = (SELECT TOP 1 SurrogateIdRangeFirstValue + FROM dbo.Transactions + WHERE IsCompleted = 1 + AND SurrogateIdRangeFirstValue BETWEEN @MinTransactionId AND @CurrentTransactionId + AND SurrogateIdRangeFirstValue < @MinNotCompletedTransactionId + ORDER BY SurrogateIdRangeFirstValue DESC); + IF @MaxTransactionId >= @MinTransactionId + BEGIN + UPDATE A + SET IsVisible = 1, + VisibleDate = getUTCdate() + FROM dbo.Transactions AS A WITH (INDEX (1)) + WHERE SurrogateIdRangeFirstValue BETWEEN @MinTransactionId AND @CurrentTransactionId + AND SurrogateIdRangeFirstValue <= @MaxTransactionId; + SET @AffectedRows += @@rowcount; + END + SET @msg = 'Min=' + CONVERT (VARCHAR, @MinTransactionId) + ' C=' + CONVERT (VARCHAR, @CurrentTransactionId) + ' MinNC=' + CONVERT (VARCHAR, @MinNotCompletedTransactionId) + ' Max=' + CONVERT (VARCHAR, @MaxTransactionId); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @AffectedRows, @Text = @msg; +END TRY +BEGIN CATCH + IF @@trancount > 0 + ROLLBACK; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.MergeResourcesBeginTransaction +@Count INT, @TransactionId BIGINT OUTPUT, @SequenceRangeFirstValue INT=NULL OUTPUT, @HeartbeatDate DATETIME=NULL, @EnableThrottling BIT=0 +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'MergeResourcesBeginTransaction', @Mode AS VARCHAR (200) = 'Cnt=' + CONVERT (VARCHAR, @Count) + ' HB=' + isnull(CONVERT (VARCHAR, @HeartbeatDate, 121), 'NULL') + ' ET=' + CONVERT (VARCHAR, @EnableThrottling), @st AS DATETIME = getUTCdate(), @FirstValueVar AS SQL_VARIANT, @LastValueVar AS SQL_VARIANT, @OptimalConcurrency AS INT = isnull((SELECT Number + FROM Parameters + WHERE Id = 'MergeResources.OptimalConcurrentCalls'), 256), @CurrentConcurrency AS INT, @msg AS VARCHAR (1000); +BEGIN TRY + SET @TransactionId = NULL; + IF @@trancount > 0 + RAISERROR ('MergeResourcesBeginTransaction cannot be called inside outer transaction.', 18, 127); + IF @EnableThrottling = 1 + BEGIN + SET @CurrentConcurrency = (SELECT count(*) + FROM sys.dm_exec_sessions + WHERE status <> 'sleeping' + AND program_name = 'MergeResources'); + IF @CurrentConcurrency > @OptimalConcurrency + BEGIN + SET @msg = 'Number of concurrent MergeResources calls = ' + CONVERT (VARCHAR, @CurrentConcurrency) + ' is above optimal = ' + CONVERT (VARCHAR, @OptimalConcurrency) + '.'; + THROW 50410, @msg, 1; + END + END + SET @FirstValueVar = NULL; + WHILE @FirstValueVar IS NULL + BEGIN + EXECUTE sys.sp_sequence_get_range @sequence_name = 'dbo.ResourceSurrogateIdUniquifierSequence', @range_size = @Count, @range_first_value = @FirstValueVar OUTPUT, @range_last_value = @LastValueVar OUTPUT; + SET @SequenceRangeFirstValue = CONVERT (INT, @FirstValueVar); + IF @SequenceRangeFirstValue > CONVERT (INT, @LastValueVar) + SET @FirstValueVar = NULL; + END + SET @TransactionId = datediff_big(millisecond, '0001-01-01', sysUTCdatetime()) * 80000 + @SequenceRangeFirstValue; + INSERT INTO dbo.Transactions (SurrogateIdRangeFirstValue, SurrogateIdRangeLastValue, HeartbeatDate) + SELECT @TransactionId, + @TransactionId + @Count - 1, + isnull(@HeartbeatDate, getUTCdate()); +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + IF @@trancount > 0 + ROLLBACK; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.MergeResourcesCommitTransaction +@TransactionId BIGINT, @FailureReason VARCHAR (MAX)=NULL, @OverrideIsControlledByClientCheck BIT=0 +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'MergeResourcesCommitTransaction', @st AS DATETIME = getUTCdate(), @InitialTranCount AS INT = @@trancount, @IsCompletedBefore AS BIT, @Rows AS INT, @msg AS VARCHAR (1000); +DECLARE @Mode AS VARCHAR (200) = 'TR=' + CONVERT (VARCHAR, @TransactionId) + ' OC=' + isnull(CONVERT (VARCHAR, @OverrideIsControlledByClientCheck), 'NULL'); +BEGIN TRY + IF @InitialTranCount = 0 + BEGIN TRANSACTION; + UPDATE dbo.Transactions + SET IsCompleted = 1, + @IsCompletedBefore = IsCompleted, + EndDate = getUTCdate(), + IsSuccess = CASE WHEN @FailureReason IS NULL THEN 1 ELSE 0 END, + FailureReason = @FailureReason + WHERE SurrogateIdRangeFirstValue = @TransactionId + AND (IsControlledByClient = 1 + OR @OverrideIsControlledByClientCheck = 1); + SET @Rows = @@rowcount; + IF @Rows = 0 + BEGIN + SET @msg = 'Transaction [' + CONVERT (VARCHAR (20), @TransactionId) + '] is not controlled by client or does not exist.'; + RAISERROR (@msg, 18, 127); + END + IF @IsCompletedBefore = 1 + BEGIN + IF @InitialTranCount = 0 + ROLLBACK; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows, @Target = '@IsCompletedBefore', @Text = '=1'; + RETURN; + END + IF @InitialTranCount = 0 + COMMIT TRANSACTION; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows; +END TRY +BEGIN CATCH + IF @InitialTranCount = 0 + AND @@trancount > 0 + ROLLBACK; + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.MergeResourcesDeleteInvisibleHistory +@TransactionId BIGINT, @AffectedRows INT=NULL OUTPUT +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = object_name(@@procid), @Mode AS VARCHAR (100) = 'T=' + CONVERT (VARCHAR, @TransactionId), @st AS DATETIME = getUTCdate(), @TypeId AS SMALLINT; +SET @AffectedRows = 0; +BEGIN TRY + DECLARE @Types TABLE ( + TypeId SMALLINT PRIMARY KEY, + Name VARCHAR (100)); + INSERT INTO @Types + EXECUTE dbo.GetUsedResourceTypes ; + WHILE EXISTS (SELECT * + FROM @Types) + BEGIN + SET @TypeId = (SELECT TOP 1 TypeId + FROM @Types + ORDER BY TypeId); + DELETE dbo.Resource + WHERE ResourceTypeId = @TypeId + AND HistoryTransactionId = @TransactionId + AND RawResource = 0xF; + SET @AffectedRows += @@rowcount; + DELETE @Types + WHERE TypeId = @TypeId; + END + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @AffectedRows; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.MergeResourcesGetTimeoutTransactions +@TimeoutSec INT +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = object_name(@@procid), @Mode AS VARCHAR (100) = 'T=' + CONVERT (VARCHAR, @TimeoutSec), @st AS DATETIME = getUTCdate(), @MinTransactionId AS BIGINT; +BEGIN TRY + EXECUTE dbo.MergeResourcesGetTransactionVisibility @MinTransactionId OUTPUT; + SELECT SurrogateIdRangeFirstValue + FROM dbo.Transactions + WHERE SurrogateIdRangeFirstValue > @MinTransactionId + AND IsCompleted = 0 + AND datediff(second, HeartbeatDate, getUTCdate()) > @TimeoutSec + ORDER BY SurrogateIdRangeFirstValue; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; +END TRY +BEGIN CATCH + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.MergeResourcesGetTransactionVisibility +@TransactionId BIGINT OUTPUT +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = object_name(@@procid), @Mode AS VARCHAR (100) = '', @st AS DATETIME = getUTCdate(); +SET @TransactionId = isnull((SELECT TOP 1 SurrogateIdRangeFirstValue + FROM dbo.Transactions + WHERE IsVisible = 1 + ORDER BY SurrogateIdRangeFirstValue DESC), -1); +EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount, @Text = @TransactionId; + +GO +CREATE PROCEDURE dbo.MergeResourcesPutTransactionHeartbeat +@TransactionId BIGINT +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'MergeResourcesPutTransactionHeartbeat', @Mode AS VARCHAR (100) = 'TR=' + CONVERT (VARCHAR, @TransactionId); +BEGIN TRY + UPDATE dbo.Transactions + SET HeartbeatDate = getUTCdate() + WHERE SurrogateIdRangeFirstValue = @TransactionId + AND IsControlledByClient = 1; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.MergeResourcesPutTransactionInvisibleHistory +@TransactionId BIGINT +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = object_name(@@procid), @Mode AS VARCHAR (100) = 'TR=' + CONVERT (VARCHAR, @TransactionId), @st AS DATETIME = getUTCdate(); +BEGIN TRY + UPDATE dbo.Transactions + SET InvisibleHistoryRemovedDate = getUTCdate() + WHERE SurrogateIdRangeFirstValue = @TransactionId + AND InvisibleHistoryRemovedDate IS NULL; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.MergeSearchParams +@SearchParams dbo.SearchParamList READONLY, @IsResourceChangeCaptureEnabled BIT=0, @TransactionId BIGINT=NULL, @Resources dbo.ResourceList READONLY, @ResourceWriteClaims dbo.ResourceWriteClaimList READONLY, @ReferenceSearchParams dbo.ReferenceSearchParamList READONLY, @TokenSearchParams dbo.TokenSearchParamList READONLY, @TokenTexts dbo.TokenTextList READONLY, @StringSearchParams dbo.StringSearchParamList READONLY, @UriSearchParams dbo.UriSearchParamList READONLY, @NumberSearchParams dbo.NumberSearchParamList READONLY, @QuantitySearchParams dbo.QuantitySearchParamList READONLY, @DateTimeSearchParms dbo.DateTimeSearchParamList READONLY, @ReferenceTokenCompositeSearchParams dbo.ReferenceTokenCompositeSearchParamList READONLY, @TokenTokenCompositeSearchParams dbo.TokenTokenCompositeSearchParamList READONLY, @TokenDateTimeCompositeSearchParams dbo.TokenDateTimeCompositeSearchParamList READONLY, @TokenQuantityCompositeSearchParams dbo.TokenQuantityCompositeSearchParamList READONLY, @TokenStringCompositeSearchParams dbo.TokenStringCompositeSearchParamList READONLY, @TokenNumberNumberCompositeSearchParams dbo.TokenNumberNumberCompositeSearchParamList READONLY +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = object_name(@@procid), @Mode AS VARCHAR (200) = 'Cnt=' + CONVERT (VARCHAR, (SELECT count(*) + FROM @SearchParams)), @st AS DATETIME = getUTCdate(), @LastUpdated AS DATETIMEOFFSET (7) = sysdatetimeoffset(), @msg AS VARCHAR (4000), @Rows AS INT, @AffectedRows AS INT = 0, @Uri AS VARCHAR (4000), @Status AS VARCHAR (20); +DECLARE @SearchParamsCopy AS dbo.SearchParamList; +INSERT INTO @SearchParamsCopy +SELECT * +FROM @SearchParams; +WHILE EXISTS (SELECT * + FROM @SearchParamsCopy) + BEGIN + SELECT TOP 1 @Uri = Uri, + @Status = Status + FROM @SearchParamsCopy; + SET @msg = 'Uri=' + @Uri + ' Status=' + @Status; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start', @Text = @msg; + DELETE @SearchParamsCopy + WHERE Uri = @Uri; + END +DECLARE @SummaryOfChanges TABLE ( + Uri VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL, + Operation VARCHAR (20) NOT NULL); +BEGIN TRY + SET TRANSACTION ISOLATION LEVEL SERIALIZABLE; + BEGIN TRANSACTION; + SELECT TOP 60 @msg = string_agg(S.Uri, ', ') + FROM @SearchParams AS I + INNER JOIN + dbo.SearchParam AS S + ON S.Uri = I.Uri + WHERE I.LastUpdated != S.LastUpdated; + IF @msg IS NOT NULL + BEGIN + SET @msg = concat('Optimistic concurrency conflict detected for search parameters: ', @msg); + ROLLBACK; + THROW 50001, @msg, 1; + END + IF EXISTS (SELECT * + FROM @Resources) + BEGIN + EXECUTE dbo.MergeResources @AffectedRows = @AffectedRows OUTPUT, @RaiseExceptionOnConflict = 1, @IsResourceChangeCaptureEnabled = @IsResourceChangeCaptureEnabled, @TransactionId = @TransactionId, @SingleTransaction = 1, @Resources = @Resources, @ResourceWriteClaims = @ResourceWriteClaims, @ReferenceSearchParams = @ReferenceSearchParams, @TokenSearchParams = @TokenSearchParams, @TokenTexts = @TokenTexts, @StringSearchParams = @StringSearchParams, @UriSearchParams = @UriSearchParams, @NumberSearchParams = @NumberSearchParams, @QuantitySearchParams = @QuantitySearchParams, @DateTimeSearchParms = @DateTimeSearchParms, @ReferenceTokenCompositeSearchParams = @ReferenceTokenCompositeSearchParams, @TokenTokenCompositeSearchParams = @TokenTokenCompositeSearchParams, @TokenDateTimeCompositeSearchParams = @TokenDateTimeCompositeSearchParams, @TokenQuantityCompositeSearchParams = @TokenQuantityCompositeSearchParams, @TokenStringCompositeSearchParams = @TokenStringCompositeSearchParams, @TokenNumberNumberCompositeSearchParams = @TokenNumberNumberCompositeSearchParams; + SET @Rows = @Rows + @AffectedRows; + END + MERGE INTO dbo.SearchParam + AS S + USING @SearchParams AS I ON I.Uri = S.Uri + WHEN MATCHED THEN UPDATE + SET Status = I.Status, + LastUpdated = @LastUpdated, + IsPartiallySupported = I.IsPartiallySupported + WHEN NOT MATCHED BY TARGET THEN INSERT (Uri, Status, LastUpdated, IsPartiallySupported) VALUES (I.Uri, I.Status, @LastUpdated, I.IsPartiallySupported) + OUTPUT I.Uri, $ACTION INTO @SummaryOfChanges; + SET @Rows = @@rowcount; + SELECT S.SearchParamId, + S.Uri, + S.LastUpdated + FROM dbo.SearchParam AS S + INNER JOIN + @SummaryOfChanges AS C + ON C.Uri = S.Uri + WHERE C.Operation = 'INSERT'; + SET @msg = 'LastUpdated=' + substring(CONVERT (VARCHAR, @LastUpdated), 1, 23) + ' INSERT=' + CONVERT (VARCHAR, @@rowcount); + COMMIT TRANSACTION; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Action = 'Merge', @Rows = @Rows, @Text = @msg; +END TRY +BEGIN CATCH + IF @@trancount > 0 + ROLLBACK; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + + +GO +INSERT INTO Parameters (Id, Char) +SELECT 'MergeSearchParams', + 'LogEvent'; + +GO +CREATE PROCEDURE dbo.PutJobCancelation +@QueueType TINYINT, @GroupId BIGINT=NULL, @JobId BIGINT=NULL +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'PutJobCancelation', @Mode AS VARCHAR (100) = 'Q=' + isnull(CONVERT (VARCHAR, @QueueType), 'NULL') + ' G=' + isnull(CONVERT (VARCHAR, @GroupId), 'NULL') + ' J=' + isnull(CONVERT (VARCHAR, @JobId), 'NULL'), @st AS DATETIME = getUTCdate(), @Rows AS INT, @PartitionId AS TINYINT = @JobId % 16; +BEGIN TRY + IF @JobId IS NULL + AND @GroupId IS NULL + RAISERROR ('@JobId = NULL and @GroupId = NULL', 18, 127); + IF @JobId IS NOT NULL + BEGIN + UPDATE dbo.JobQueue + SET Status = 4, + EndDate = getUTCdate(), + Version = datediff_big(millisecond, '0001-01-01', getUTCdate()) + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND JobId = @JobId + AND Status = 0; + SET @Rows = @@rowcount; + IF @Rows = 0 + BEGIN + UPDATE dbo.JobQueue + SET CancelRequested = 1 + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND JobId = @JobId + AND Status = 1; + SET @Rows = @@rowcount; + END + END + ELSE + BEGIN + UPDATE dbo.JobQueue + SET Status = 4, + EndDate = getUTCdate(), + Version = datediff_big(millisecond, '0001-01-01', getUTCdate()) + WHERE QueueType = @QueueType + AND GroupId = @GroupId + AND Status = 0; + SET @Rows = @@rowcount; + UPDATE dbo.JobQueue + SET CancelRequested = 1 + WHERE QueueType = @QueueType + AND GroupId = @GroupId + AND Status = 1; + SET @Rows += @@rowcount; + END + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.PutJobHeartbeat +@QueueType TINYINT, @JobId BIGINT, @Version BIGINT, @CancelRequested BIT=0 OUTPUT +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'PutJobHeartbeat', @Mode AS VARCHAR (100), @st AS DATETIME = getUTCdate(), @Rows AS INT = 0, @PartitionId AS TINYINT = @JobId % 16; +SET @Mode = 'Q=' + CONVERT (VARCHAR, @QueueType) + ' J=' + CONVERT (VARCHAR, @JobId) + ' P=' + CONVERT (VARCHAR, @PartitionId) + ' V=' + CONVERT (VARCHAR, @Version); +BEGIN TRY + UPDATE dbo.JobQueue + SET @CancelRequested = CancelRequested, + HeartbeatDate = getUTCdate() + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND JobId = @JobId + AND Status = 1 + AND Version = @Version; + SET @Rows = @@rowcount; + IF @Rows = 0 + AND NOT EXISTS (SELECT * + FROM dbo.JobQueue + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND JobId = @JobId + AND Version = @Version + AND Status IN (2, 3, 4)) + BEGIN + IF EXISTS (SELECT * + FROM dbo.JobQueue + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND JobId = @JobId) + THROW 50412, 'Precondition failed', 1; + ELSE + THROW 50404, 'Job record not found', 1; + END + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.PutJobStatus +@QueueType TINYINT, @JobId BIGINT, @Version BIGINT, @Failed BIT, @Data BIGINT, @FinalResult VARCHAR (MAX), @RequestCancellationOnFailure BIT +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'PutJobStatus', @Mode AS VARCHAR (100), @st AS DATETIME = getUTCdate(), @Rows AS INT = 0, @PartitionId AS TINYINT = @JobId % 16, @GroupId AS BIGINT; +SET @Mode = 'Q=' + CONVERT (VARCHAR, @QueueType) + ' J=' + CONVERT (VARCHAR, @JobId) + ' P=' + CONVERT (VARCHAR, @PartitionId) + ' V=' + CONVERT (VARCHAR, @Version) + ' F=' + CONVERT (VARCHAR, @Failed) + ' R=' + isnull(@FinalResult, 'NULL'); +BEGIN TRY + UPDATE dbo.JobQueue + SET EndDate = getUTCdate(), + Status = CASE WHEN @Failed = 1 THEN 3 WHEN CancelRequested = 1 THEN 4 ELSE 2 END, + Data = @Data, + Result = @FinalResult, + @GroupId = GroupId + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND JobId = @JobId + AND Status = 1 + AND Version = @Version; + SET @Rows = @@rowcount; + IF @Rows = 0 + BEGIN + SET @GroupId = (SELECT GroupId + FROM dbo.JobQueue + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND JobId = @JobId + AND Version = @Version + AND Status IN (2, 3, 4)); + IF @GroupId IS NULL + IF EXISTS (SELECT * + FROM dbo.JobQueue + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND JobId = @JobId) + THROW 50412, 'Precondition failed', 1; + ELSE + THROW 50404, 'Job record not found', 1; + END + IF @Failed = 1 + AND @RequestCancellationOnFailure = 1 + EXECUTE dbo.PutJobCancelation @QueueType = @QueueType, @GroupId = @GroupId; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE OR ALTER PROCEDURE dbo.RemovePartitionFromResourceChanges_2 +@partitionNumberToSwitchOut INT, @partitionBoundaryToMerge DATETIME2 (7) +AS +BEGIN + TRUNCATE TABLE dbo.ResourceChangeDataStaging; + ALTER TABLE dbo.ResourceChangeData SWITCH PARTITION @partitionNumberToSwitchOut TO dbo.ResourceChangeDataStaging; + ALTER PARTITION FUNCTION PartitionFunction_ResourceChangeData_Timestamp( ) + MERGE RANGE (@partitionBoundaryToMerge); + TRUNCATE TABLE dbo.ResourceChangeDataStaging; +END + +GO +CREATE PROCEDURE dbo.SwitchPartitionsIn +@Tbl VARCHAR (100) +WITH EXECUTE AS 'dbo' +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'SwitchPartitionsIn', @Mode AS VARCHAR (200) = 'Tbl=' + isnull(@Tbl, 'NULL'), @st AS DATETIME = getUTCdate(), @ResourceTypeId AS SMALLINT, @Rows AS BIGINT, @Txt AS VARCHAR (1000), @TblInt AS VARCHAR (100), @Ind AS VARCHAR (200), @IndId AS INT, @DataComp AS VARCHAR (100); +DECLARE @Indexes TABLE ( + IndId INT PRIMARY KEY, + name VARCHAR (200)); +DECLARE @ResourceTypes TABLE ( + ResourceTypeId SMALLINT PRIMARY KEY); +BEGIN TRY + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start'; + IF @Tbl IS NULL + RAISERROR ('@Tbl IS NULL', 18, 127); + INSERT INTO @Indexes + SELECT index_id, + name + FROM sys.indexes + WHERE object_id = object_id(@Tbl) + AND is_disabled = 1; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@Indexes', @Action = 'Insert', @Rows = @@rowcount; + WHILE EXISTS (SELECT * + FROM @Indexes) + BEGIN + SELECT TOP 1 @IndId = IndId, + @Ind = name + FROM @Indexes + ORDER BY IndId; + SET @DataComp = CASE WHEN (SELECT PropertyValue + FROM dbo.IndexProperties + WHERE TableName = @Tbl + AND IndexName = @Ind) = 'PAGE' THEN ' PARTITION = ALL WITH (DATA_COMPRESSION = PAGE)' ELSE '' END; + SET @Txt = 'IF EXISTS (SELECT * FROM sys.indexes WHERE object_id = object_id(''' + @Tbl + ''') AND name = ''' + @Ind + ''' AND is_disabled = 1) ALTER INDEX ' + @Ind + ' ON dbo.' + @Tbl + ' REBUILD' + @DataComp; + EXECUTE (@Txt); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @Ind, @Action = 'Rebuild', @Text = @Txt; + DELETE @Indexes + WHERE IndId = @IndId; + END + INSERT INTO @ResourceTypes + SELECT CONVERT (SMALLINT, substring(name, charindex('_', name) + 1, 6)) AS ResourceTypeId + FROM sys.objects AS O + WHERE name LIKE @Tbl + '[_]%' + AND EXISTS (SELECT * + FROM sysindexes + WHERE id = O.object_id + AND indid IN (0, 1) + AND rows > 0); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '#ResourceTypes', @Action = 'Select Into', @Rows = @@rowcount; + WHILE EXISTS (SELECT * + FROM @ResourceTypes) + BEGIN + SET @ResourceTypeId = (SELECT TOP 1 ResourceTypeId + FROM @ResourceTypes); + SET @TblInt = @Tbl + '_' + CONVERT (VARCHAR, @ResourceTypeId); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @TblInt; + SET @Txt = 'ALTER TABLE dbo.' + @TblInt + ' SWITCH TO dbo.' + @Tbl + ' PARTITION $partition.PartitionFunction_ResourceTypeId(' + CONVERT (VARCHAR, @ResourceTypeId) + ')'; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @Tbl, @Action = 'Switch in start', @Text = @Txt; + EXECUTE (@Txt); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @Tbl, @Action = 'Switch in', @Text = @Txt; + IF EXISTS (SELECT * + FROM sysindexes + WHERE id = object_id(@TblInt) + AND rows > 0) + BEGIN + SET @Txt = @TblInt + ' is not empty after switch'; + RAISERROR (@Txt, 18, 127); + END + EXECUTE ('DROP TABLE dbo.' + @TblInt); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @TblInt, @Action = 'Drop'; + DELETE @ResourceTypes + WHERE ResourceTypeId = @ResourceTypeId; + END + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.SwitchPartitionsInAllTables +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'SwitchPartitionsInAllTables', @Mode AS VARCHAR (200) = 'PS=PartitionScheme_ResourceTypeId', @st AS DATETIME = getUTCdate(), @Tbl AS VARCHAR (100); +BEGIN TRY + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start'; + DECLARE @Tables TABLE ( + name VARCHAR (100) PRIMARY KEY, + supported BIT ); + INSERT INTO @Tables + EXECUTE dbo.GetPartitionedTables @IncludeNotDisabled = 1, @IncludeNotSupported = 0; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@Tables', @Action = 'Insert', @Rows = @@rowcount; + WHILE EXISTS (SELECT * + FROM @Tables) + BEGIN + SET @Tbl = (SELECT TOP 1 name + FROM @Tables + ORDER BY name); + EXECUTE dbo.SwitchPartitionsIn @Tbl = @Tbl; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = 'SwitchPartitionsIn', @Action = 'Execute', @Text = @Tbl; + DELETE @Tables + WHERE name = @Tbl; + END + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.SwitchPartitionsOut +@Tbl VARCHAR (100), @RebuildClustered BIT +WITH EXECUTE AS 'dbo' +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'SwitchPartitionsOut', @Mode AS VARCHAR (200) = 'Tbl=' + isnull(@Tbl, 'NULL') + ' ND=' + isnull(CONVERT (VARCHAR, @RebuildClustered), 'NULL'), @st AS DATETIME = getUTCdate(), @ResourceTypeId AS SMALLINT, @Rows AS BIGINT, @Txt AS VARCHAR (MAX), @TblInt AS VARCHAR (100), @IndId AS INT, @Ind AS VARCHAR (200), @Name AS VARCHAR (100), @checkName AS VARCHAR (200), @definition AS VARCHAR (200); +DECLARE @Indexes TABLE ( + IndId INT PRIMARY KEY, + name VARCHAR (200), + IsDisabled BIT ); +DECLARE @IndexesRT TABLE ( + IndId INT PRIMARY KEY, + name VARCHAR (200), + IsDisabled BIT ); +DECLARE @ResourceTypes TABLE ( + ResourceTypeId SMALLINT PRIMARY KEY, + partition_number_roundtrip INT , + partition_number INT , + row_count BIGINT ); +DECLARE @Names TABLE ( + name VARCHAR (100) PRIMARY KEY); +DECLARE @CheckConstraints TABLE ( + CheckName VARCHAR (200), + CheckDefinition VARCHAR (200)); +BEGIN TRY + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start'; + IF @Tbl IS NULL + RAISERROR ('@Tbl IS NULL', 18, 127); + IF @RebuildClustered IS NULL + RAISERROR ('@RebuildClustered IS NULL', 18, 127); + INSERT INTO @Indexes + SELECT index_id, + name, + is_disabled + FROM sys.indexes + WHERE object_id = object_id(@Tbl) + AND (is_disabled = 0 + OR @RebuildClustered = 1); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@Indexes', @Action = 'Insert', @Rows = @@rowcount; + INSERT INTO @ResourceTypes + SELECT partition_number - 1 AS ResourceTypeId, + $PARTITION.PartitionFunction_ResourceTypeId (partition_number - 1) AS partition_number_roundtrip, + partition_number, + row_count + FROM sys.dm_db_partition_stats + WHERE object_id = object_id(@Tbl) + AND index_id = 1 + AND row_count > 0; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@ResourceTypes', @Action = 'Insert', @Rows = @@rowcount, @Text = 'For partition switch'; + IF EXISTS (SELECT * + FROM @ResourceTypes + WHERE partition_number_roundtrip <> partition_number) + RAISERROR ('Partition sanity check failed', 18, 127); + WHILE EXISTS (SELECT * + FROM @ResourceTypes) + BEGIN + SELECT TOP 1 @ResourceTypeId = ResourceTypeId, + @Rows = row_count + FROM @ResourceTypes + ORDER BY ResourceTypeId; + SET @TblInt = @Tbl + '_' + CONVERT (VARCHAR, @ResourceTypeId); + SET @Txt = 'Starting @ResourceTypeId=' + CONVERT (VARCHAR, @ResourceTypeId) + ' row_count=' + CONVERT (VARCHAR, @Rows); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Text = @Txt; + IF NOT EXISTS (SELECT * + FROM sysindexes + WHERE id = object_id(@TblInt) + AND rows > 0) + BEGIN + IF object_id(@TblInt) IS NOT NULL + BEGIN + EXECUTE ('DROP TABLE dbo.' + @TblInt); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @TblInt, @Action = 'Drop'; + END + EXECUTE ('SELECT * INTO dbo.' + @TblInt + ' FROM dbo.' + @Tbl + ' WHERE 1 = 2'); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @TblInt, @Action = 'Select Into', @Rows = @@rowcount; + DELETE @CheckConstraints; + INSERT INTO @CheckConstraints + SELECT name, + definition + FROM sys.check_constraints + WHERE parent_object_id = object_id(@Tbl); + WHILE EXISTS (SELECT * + FROM @CheckConstraints) + BEGIN + SELECT TOP 1 @checkName = CheckName, + @definition = CheckDefinition + FROM @CheckConstraints; + SET @Txt = 'ALTER TABLE ' + @TblInt + ' ADD CHECK ' + @definition; + EXECUTE (@Txt); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @TblInt, @Action = 'ALTER', @Text = @Txt; + DELETE @CheckConstraints + WHERE CheckName = @checkName; + END + DELETE @Names; + INSERT INTO @Names + SELECT name + FROM sys.columns + WHERE object_id = object_id(@Tbl) + AND is_sparse = 1; + WHILE EXISTS (SELECT * + FROM @Names) + BEGIN + SET @Name = (SELECT TOP 1 name + FROM @Names + ORDER BY name); + SET @Txt = (SELECT 'ALTER TABLE dbo.' + @TblInt + ' ALTER COLUMN ' + @Name + ' ' + T.name + '(' + CONVERT (VARCHAR, C.precision) + ',' + CONVERT (VARCHAR, C.scale) + ') SPARSE NULL' + FROM sys.types AS T + INNER JOIN + sys.columns AS C + ON C.system_type_id = T.system_type_id + WHERE C.object_id = object_id(@Tbl) + AND C.name = @Name); + EXECUTE (@Txt); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @TblInt, @Action = 'ALTER', @Text = @Txt; + DELETE @Names + WHERE name = @Name; + END + END + INSERT INTO @IndexesRT + SELECT * + FROM @Indexes + WHERE IsDisabled = 0; + WHILE EXISTS (SELECT * + FROM @IndexesRT) + BEGIN + SELECT TOP 1 @IndId = IndId, + @Ind = name + FROM @IndexesRT + ORDER BY IndId; + IF NOT EXISTS (SELECT * + FROM sys.indexes + WHERE object_id = object_id(@TblInt) + AND name = @Ind) + BEGIN + EXECUTE dbo.GetIndexCommands @Tbl = @Tbl, @Ind = @Ind, @AddPartClause = 0, @IncludeClustered = 1, @Txt = @Txt OUTPUT; + SET @Txt = replace(@Txt, '[' + @Tbl + ']', @TblInt); + EXECUTE (@Txt); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @TblInt, @Action = 'Create Index', @Text = @Txt; + END + DELETE @IndexesRT + WHERE IndId = @IndId; + END + SET @Txt = 'ALTER TABLE dbo.' + @TblInt + ' ADD CHECK (ResourceTypeId >= ' + CONVERT (VARCHAR, @ResourceTypeId) + ' AND ResourceTypeId < ' + CONVERT (VARCHAR, @ResourceTypeId) + ' + 1)'; + EXECUTE (@Txt); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @Tbl, @Action = 'Add check', @Text = @Txt; + SET @Txt = 'ALTER TABLE dbo.' + @Tbl + ' SWITCH PARTITION $partition.PartitionFunction_ResourceTypeId(' + CONVERT (VARCHAR, @ResourceTypeId) + ') TO dbo.' + @TblInt; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @Tbl, @Action = 'Switch out start', @Text = @Txt; + EXECUTE (@Txt); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @Tbl, @Action = 'Switch out end', @Text = @Txt; + DELETE @ResourceTypes + WHERE ResourceTypeId = @ResourceTypeId; + END + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.SwitchPartitionsOutAllTables +@RebuildClustered BIT +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'SwitchPartitionsOutAllTables', @Mode AS VARCHAR (200) = 'PS=PartitionScheme_ResourceTypeId ND=' + isnull(CONVERT (VARCHAR, @RebuildClustered), 'NULL'), @st AS DATETIME = getUTCdate(), @Tbl AS VARCHAR (100); +BEGIN TRY + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start'; + DECLARE @Tables TABLE ( + name VARCHAR (100) PRIMARY KEY, + supported BIT ); + INSERT INTO @Tables + EXECUTE dbo.GetPartitionedTables @IncludeNotDisabled = @RebuildClustered, @IncludeNotSupported = 0; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@Tables', @Action = 'Insert', @Rows = @@rowcount; + WHILE EXISTS (SELECT * + FROM @Tables) + BEGIN + SET @Tbl = (SELECT TOP 1 name + FROM @Tables + ORDER BY name); + EXECUTE dbo.SwitchPartitionsOut @Tbl = @Tbl, @RebuildClustered = @RebuildClustered; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = 'SwitchPartitionsOut', @Action = 'Execute', @Text = @Tbl; + DELETE @Tables + WHERE name = @Tbl; + END + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +GO +CREATE OR ALTER PROCEDURE dbo.UpdateEventAgentCheckpoint +@CheckpointId VARCHAR (64), @LastProcessedDateTime DATETIMEOFFSET (7)=NULL, @LastProcessedIdentifier VARCHAR (64)=NULL +AS +BEGIN + IF EXISTS (SELECT * + FROM dbo.EventAgentCheckpoint + WHERE CheckpointId = @CheckpointId) + UPDATE dbo.EventAgentCheckpoint + SET CheckpointId = @CheckpointId, + LastProcessedDateTime = @LastProcessedDateTime, + LastProcessedIdentifier = @LastProcessedIdentifier, + UpdatedOn = sysutcdatetime() + WHERE CheckpointId = @CheckpointId; + ELSE + INSERT INTO dbo.EventAgentCheckpoint (CheckpointId, LastProcessedDateTime, LastProcessedIdentifier, UpdatedOn) + VALUES (@CheckpointId, @LastProcessedDateTime, @LastProcessedIdentifier, sysutcdatetime()); +END + +GO +CREATE PROCEDURE dbo.UpdateReindexJob +@id VARCHAR (64), @status VARCHAR (10), @rawJobRecord VARCHAR (MAX), @jobVersion BINARY (8) +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +BEGIN TRANSACTION; +DECLARE @currentJobVersion AS BINARY (8); +SELECT @currentJobVersion = JobVersion +FROM dbo.ReindexJob WITH (UPDLOCK, HOLDLOCK) +WHERE Id = @id; +IF (@currentJobVersion IS NULL) + BEGIN + THROW 50404, 'Reindex job record not found', 1; + END +IF (@jobVersion <> @currentJobVersion) + BEGIN + THROW 50412, 'Precondition failed', 1; + END +DECLARE @heartbeatDateTime AS DATETIME2 (7) = SYSUTCDATETIME(); +UPDATE dbo.ReindexJob +SET Status = @status, + HeartbeatDateTime = @heartbeatDateTime, + RawJobRecord = @rawJobRecord +WHERE Id = @id; +SELECT @@DBTS; +COMMIT TRANSACTION; + +GO +CREATE PROCEDURE dbo.UpdateResourceSearchParams +@FailedResources INT=0 OUTPUT, @Resources dbo.ResourceList READONLY, @ResourceWriteClaims dbo.ResourceWriteClaimList READONLY, @ReferenceSearchParams dbo.ReferenceSearchParamList READONLY, @TokenSearchParams dbo.TokenSearchParamList READONLY, @TokenTexts dbo.TokenTextList READONLY, @StringSearchParams dbo.StringSearchParamList READONLY, @UriSearchParams dbo.UriSearchParamList READONLY, @NumberSearchParams dbo.NumberSearchParamList READONLY, @QuantitySearchParams dbo.QuantitySearchParamList READONLY, @DateTimeSearchParams dbo.DateTimeSearchParamList READONLY, @ReferenceTokenCompositeSearchParams dbo.ReferenceTokenCompositeSearchParamList READONLY, @TokenTokenCompositeSearchParams dbo.TokenTokenCompositeSearchParamList READONLY, @TokenDateTimeCompositeSearchParams dbo.TokenDateTimeCompositeSearchParamList READONLY, @TokenQuantityCompositeSearchParams dbo.TokenQuantityCompositeSearchParamList READONLY, @TokenStringCompositeSearchParams dbo.TokenStringCompositeSearchParamList READONLY, @TokenNumberNumberCompositeSearchParams dbo.TokenNumberNumberCompositeSearchParamList READONLY +AS +SET NOCOUNT ON; +DECLARE @st AS DATETIME = getUTCdate(), @SP AS VARCHAR (100) = object_name(@@procid), @Mode AS VARCHAR (200) = isnull((SELECT 'RT=[' + CONVERT (VARCHAR, min(ResourceTypeId)) + ',' + CONVERT (VARCHAR, max(ResourceTypeId)) + '] Sur=[' + CONVERT (VARCHAR, min(ResourceSurrogateId)) + ',' + CONVERT (VARCHAR, max(ResourceSurrogateId)) + '] V=' + CONVERT (VARCHAR, max(Version)) + ' Rows=' + CONVERT (VARCHAR, count(*)) + FROM @Resources), 'Input=Empty'), @Rows AS INT, @ReferenceSearchParamsCurrent AS dbo.ReferenceSearchParamList, @ReferenceSearchParamsDelete AS dbo.ReferenceSearchParamList, @ReferenceSearchParamsInsert AS dbo.ReferenceSearchParamList, @TokenSearchParamsCurrent AS dbo.TokenSearchParamList, @TokenSearchParamsDelete AS dbo.TokenSearchParamList, @TokenSearchParamsInsert AS dbo.TokenSearchParamList, @TokenTextsCurrent AS dbo.TokenTextList, @TokenTextsDelete AS dbo.TokenTextList, @TokenTextsInsert AS dbo.TokenTextList, @StringSearchParamsCurrent AS dbo.StringSearchParamList, @StringSearchParamsDelete AS dbo.StringSearchParamList, @StringSearchParamsInsert AS dbo.StringSearchParamList, @UriSearchParamsCurrent AS dbo.UriSearchParamList, @UriSearchParamsDelete AS dbo.UriSearchParamList, @UriSearchParamsInsert AS dbo.UriSearchParamList, @NumberSearchParamsCurrent AS dbo.NumberSearchParamList, @NumberSearchParamsDelete AS dbo.NumberSearchParamList, @NumberSearchParamsInsert AS dbo.NumberSearchParamList, @QuantitySearchParamsCurrent AS dbo.QuantitySearchParamList, @QuantitySearchParamsDelete AS dbo.QuantitySearchParamList, @QuantitySearchParamsInsert AS dbo.QuantitySearchParamList, @DateTimeSearchParamsCurrent AS dbo.DateTimeSearchParamList, @DateTimeSearchParamsDelete AS dbo.DateTimeSearchParamList, @DateTimeSearchParamsInsert AS dbo.DateTimeSearchParamList, @ReferenceTokenCompositeSearchParamsCurrent AS dbo.ReferenceTokenCompositeSearchParamList, @ReferenceTokenCompositeSearchParamsDelete AS dbo.ReferenceTokenCompositeSearchParamList, @ReferenceTokenCompositeSearchParamsInsert AS dbo.ReferenceTokenCompositeSearchParamList, @TokenTokenCompositeSearchParamsCurrent AS dbo.TokenTokenCompositeSearchParamList, @TokenTokenCompositeSearchParamsDelete AS dbo.TokenTokenCompositeSearchParamList, @TokenTokenCompositeSearchParamsInsert AS dbo.TokenTokenCompositeSearchParamList, @TokenDateTimeCompositeSearchParamsCurrent AS dbo.TokenDateTimeCompositeSearchParamList, @TokenDateTimeCompositeSearchParamsDelete AS dbo.TokenDateTimeCompositeSearchParamList, @TokenDateTimeCompositeSearchParamsInsert AS dbo.TokenDateTimeCompositeSearchParamList, @TokenQuantityCompositeSearchParamsCurrent AS dbo.TokenQuantityCompositeSearchParamList, @TokenQuantityCompositeSearchParamsDelete AS dbo.TokenQuantityCompositeSearchParamList, @TokenQuantityCompositeSearchParamsInsert AS dbo.TokenQuantityCompositeSearchParamList, @TokenStringCompositeSearchParamsCurrent AS dbo.TokenStringCompositeSearchParamList, @TokenStringCompositeSearchParamsDelete AS dbo.TokenStringCompositeSearchParamList, @TokenStringCompositeSearchParamsInsert AS dbo.TokenStringCompositeSearchParamList, @TokenNumberNumberCompositeSearchParamsCurrent AS dbo.TokenNumberNumberCompositeSearchParamList, @TokenNumberNumberCompositeSearchParamsDelete AS dbo.TokenNumberNumberCompositeSearchParamList, @TokenNumberNumberCompositeSearchParamsInsert AS dbo.TokenNumberNumberCompositeSearchParamList, @ResourceWriteClaimsCurrent AS dbo.ResourceWriteClaimList, @ResourceWriteClaimsDelete AS dbo.ResourceWriteClaimList, @ResourceWriteClaimsInsert AS dbo.ResourceWriteClaimList; +BEGIN TRY + DECLARE @Ids TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL); + BEGIN TRANSACTION; + UPDATE B + SET SearchParamHash = A.SearchParamHash + OUTPUT deleted.ResourceTypeId, deleted.ResourceSurrogateId INTO @Ids + FROM @Resources AS A + INNER JOIN + dbo.Resource AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + WHERE B.IsHistory = 0; + SET @Rows = @@rowcount; + INSERT INTO @ResourceWriteClaimsCurrent (ResourceSurrogateId, ClaimTypeId, ClaimValue) + SELECT A.ResourceSurrogateId, + ClaimTypeId, + ClaimValue + FROM dbo.ResourceWriteClaim AS A + INNER JOIN + @Ids AS B + ON B.ResourceSurrogateId = A.ResourceSurrogateId; + INSERT INTO @ResourceWriteClaimsDelete (ResourceSurrogateId, ClaimTypeId, ClaimValue) + SELECT ResourceSurrogateId, + ClaimTypeId, + ClaimValue + FROM @ResourceWriteClaimsCurrent AS A + WHERE NOT EXISTS (SELECT * + FROM @ResourceWriteClaims AS B + WHERE B.ResourceSurrogateId = A.ResourceSurrogateId + AND B.ClaimTypeId = A.ClaimTypeId + AND B.ClaimValue = A.ClaimValue) + OPTION (HASH JOIN); + DELETE A + FROM dbo.ResourceWriteClaim AS A + WHERE EXISTS (SELECT * + FROM @ResourceWriteClaimsDelete AS B + WHERE B.ResourceSurrogateId = A.ResourceSurrogateId + AND B.ClaimTypeId = A.ClaimTypeId + AND B.ClaimValue = A.ClaimValue); + INSERT INTO @ResourceWriteClaimsInsert (ResourceSurrogateId, ClaimTypeId, ClaimValue) + SELECT ResourceSurrogateId, + ClaimTypeId, + ClaimValue + FROM @ResourceWriteClaims AS A + WHERE NOT EXISTS (SELECT * + FROM @ResourceWriteClaimsCurrent AS B + WHERE B.ResourceSurrogateId = A.ResourceSurrogateId + AND B.ClaimTypeId = A.ClaimTypeId + AND B.ClaimValue = A.ClaimValue) + OPTION (HASH JOIN); + INSERT INTO @ReferenceSearchParamsCurrent (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri, ReferenceResourceTypeId, ReferenceResourceId, ReferenceResourceVersion) + SELECT A.ResourceTypeId, + A.ResourceSurrogateId, + SearchParamId, + BaseUri, + ReferenceResourceTypeId, + ReferenceResourceId, + ReferenceResourceVersion + FROM dbo.ReferenceSearchParam AS A + INNER JOIN + @Ids AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + INSERT INTO @ReferenceSearchParamsDelete (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri, ReferenceResourceTypeId, ReferenceResourceId, ReferenceResourceVersion) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + BaseUri, + ReferenceResourceTypeId, + ReferenceResourceId, + ReferenceResourceVersion + FROM @ReferenceSearchParamsCurrent AS A + WHERE NOT EXISTS (SELECT * + FROM @ReferenceSearchParams AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + AND B.SearchParamId = A.SearchParamId + AND (B.BaseUri = A.BaseUri + OR B.BaseUri IS NULL + AND A.BaseUri IS NULL) + AND (B.ReferenceResourceTypeId = A.ReferenceResourceTypeId + OR B.ReferenceResourceTypeId IS NULL + AND A.ReferenceResourceTypeId IS NULL) + AND B.ReferenceResourceId = A.ReferenceResourceId) + OPTION (HASH JOIN); + DELETE A + FROM dbo.ReferenceSearchParam AS A WITH (INDEX (1)) + WHERE EXISTS (SELECT * + FROM @ReferenceSearchParamsDelete AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + AND B.SearchParamId = A.SearchParamId + AND (B.BaseUri = A.BaseUri + OR B.BaseUri IS NULL + AND A.BaseUri IS NULL) + AND (B.ReferenceResourceTypeId = A.ReferenceResourceTypeId + OR B.ReferenceResourceTypeId IS NULL + AND A.ReferenceResourceTypeId IS NULL) + AND B.ReferenceResourceId = A.ReferenceResourceId); + INSERT INTO @ReferenceSearchParamsInsert (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri, ReferenceResourceTypeId, ReferenceResourceId, ReferenceResourceVersion) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + BaseUri, + ReferenceResourceTypeId, + ReferenceResourceId, + ReferenceResourceVersion + FROM @ReferenceSearchParams AS A + WHERE NOT EXISTS (SELECT * + FROM @ReferenceSearchParamsCurrent AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + AND B.SearchParamId = A.SearchParamId + AND (B.BaseUri = A.BaseUri + OR B.BaseUri IS NULL + AND A.BaseUri IS NULL) + AND (B.ReferenceResourceTypeId = A.ReferenceResourceTypeId + OR B.ReferenceResourceTypeId IS NULL + AND A.ReferenceResourceTypeId IS NULL) + AND B.ReferenceResourceId = A.ReferenceResourceId) + OPTION (HASH JOIN); + INSERT INTO @TokenSearchParamsCurrent (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, Code, CodeOverflow) + SELECT A.ResourceTypeId, + A.ResourceSurrogateId, + SearchParamId, + SystemId, + Code, + CodeOverflow + FROM dbo.TokenSearchParam AS A + INNER JOIN + @Ids AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + INSERT INTO @TokenSearchParamsDelete (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, Code, CodeOverflow) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId, + Code, + CodeOverflow + FROM @TokenSearchParamsCurrent AS A + WHERE NOT EXISTS (SELECT * + FROM @TokenSearchParams AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + AND B.SearchParamId = A.SearchParamId + AND (B.SystemId = A.SystemId + OR B.SystemId IS NULL + AND A.SystemId IS NULL) + AND B.Code = A.Code + AND (B.CodeOverflow = A.CodeOverflow + OR B.CodeOverflow IS NULL + AND A.CodeOverflow IS NULL)) + OPTION (HASH JOIN); + DELETE A + FROM dbo.TokenSearchParam AS A WITH (INDEX (1)) + WHERE EXISTS (SELECT * + FROM @TokenSearchParamsDelete AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + AND B.SearchParamId = A.SearchParamId + AND (B.SystemId = A.SystemId + OR B.SystemId IS NULL + AND A.SystemId IS NULL) + AND B.Code = A.Code + AND (B.CodeOverflow = A.CodeOverflow + OR B.CodeOverflow IS NULL + AND A.CodeOverflow IS NULL)); + INSERT INTO @TokenSearchParamsInsert (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, Code, CodeOverflow) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId, + Code, + CodeOverflow + FROM @TokenSearchParams AS A + WHERE NOT EXISTS (SELECT * + FROM @TokenSearchParamsCurrent AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + AND B.SearchParamId = A.SearchParamId + AND (B.SystemId = A.SystemId + OR B.SystemId IS NULL + AND A.SystemId IS NULL) + AND B.Code = A.Code + AND (B.CodeOverflow = A.CodeOverflow + OR B.CodeOverflow IS NULL + AND A.CodeOverflow IS NULL)) + OPTION (HASH JOIN); + INSERT INTO @TokenStringCompositeSearchParamsCurrent (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, Text2, TextOverflow2) + SELECT A.ResourceTypeId, + A.ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + Text2, + TextOverflow2 + FROM dbo.TokenStringCompositeSearchParam AS A + INNER JOIN + @Ids AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + INSERT INTO @TokenStringCompositeSearchParamsDelete (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, Text2, TextOverflow2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + Text2, + TextOverflow2 + FROM @TokenStringCompositeSearchParamsCurrent AS A + WHERE NOT EXISTS (SELECT * + FROM @TokenStringCompositeSearchParams AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + AND B.SearchParamId = A.SearchParamId + AND (B.SystemId1 = A.SystemId1 + OR B.SystemId1 IS NULL + AND A.SystemId1 IS NULL) + AND B.Code1 = A.Code1 + AND (B.CodeOverflow1 = A.CodeOverflow1 + OR B.CodeOverflow1 IS NULL + AND A.CodeOverflow1 IS NULL) + AND B.Text2 COLLATE Latin1_General_CI_AI = A.Text2 + AND (B.TextOverflow2 COLLATE Latin1_General_CI_AI = A.TextOverflow2 + OR B.TextOverflow2 IS NULL + AND A.TextOverflow2 IS NULL)) + OPTION (HASH JOIN); + DELETE A + FROM dbo.TokenStringCompositeSearchParam AS A WITH (INDEX (1)) + WHERE EXISTS (SELECT * + FROM @TokenStringCompositeSearchParamsDelete AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + AND B.SearchParamId = A.SearchParamId + AND (B.SystemId1 = A.SystemId1 + OR B.SystemId1 IS NULL + AND A.SystemId1 IS NULL) + AND B.Code1 = A.Code1 + AND (B.CodeOverflow1 = A.CodeOverflow1 + OR B.CodeOverflow1 IS NULL + AND A.CodeOverflow1 IS NULL) + AND B.Text2 COLLATE Latin1_General_CI_AI = A.Text2 + AND (B.TextOverflow2 COLLATE Latin1_General_CI_AI = A.TextOverflow2 + OR B.TextOverflow2 IS NULL + AND A.TextOverflow2 IS NULL)); + INSERT INTO @TokenStringCompositeSearchParamsInsert (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, Text2, TextOverflow2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + Text2, + TextOverflow2 + FROM @TokenStringCompositeSearchParams AS A + WHERE NOT EXISTS (SELECT * + FROM @TokenStringCompositeSearchParamsCurrent AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + AND B.SearchParamId = A.SearchParamId + AND (B.SystemId1 = A.SystemId1 + OR B.SystemId1 IS NULL + AND A.SystemId1 IS NULL) + AND B.Code1 = A.Code1 + AND (B.CodeOverflow1 = A.CodeOverflow1 + OR B.CodeOverflow1 IS NULL + AND A.CodeOverflow1 IS NULL) + AND B.Text2 = A.Text2 + AND (B.TextOverflow2 = A.TextOverflow2 + OR B.TextOverflow2 IS NULL + AND A.TextOverflow2 IS NULL)) + OPTION (HASH JOIN); + INSERT INTO @TokenTextsCurrent (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text) + SELECT A.ResourceTypeId, + A.ResourceSurrogateId, + SearchParamId, + Text + FROM dbo.TokenText AS A + INNER JOIN + @Ids AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + INSERT INTO @TokenTextsDelete (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + Text + FROM @TokenTextsCurrent AS A + WHERE NOT EXISTS (SELECT * + FROM @TokenTexts AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + AND B.SearchParamId = A.SearchParamId + AND B.Text = A.Text) + OPTION (HASH JOIN); + DELETE A + FROM dbo.TokenText AS A WITH (INDEX (1)) + WHERE EXISTS (SELECT * + FROM @TokenTextsDelete AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + AND B.SearchParamId = A.SearchParamId + AND B.Text = A.Text); + INSERT INTO @TokenTextsInsert (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + Text + FROM @TokenTexts AS A + WHERE NOT EXISTS (SELECT * + FROM @TokenTextsCurrent AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + AND B.SearchParamId = A.SearchParamId + AND B.Text = A.Text) + OPTION (HASH JOIN); + INSERT INTO @StringSearchParamsCurrent (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text, TextOverflow, IsMin, IsMax) + SELECT A.ResourceTypeId, + A.ResourceSurrogateId, + SearchParamId, + Text, + TextOverflow, + IsMin, + IsMax + FROM dbo.StringSearchParam AS A + INNER JOIN + @Ids AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + INSERT INTO @StringSearchParamsDelete (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text, TextOverflow, IsMin, IsMax) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + Text, + TextOverflow, + IsMin, + IsMax + FROM @StringSearchParamsCurrent AS A + WHERE NOT EXISTS (SELECT * + FROM @StringSearchParams AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + AND B.SearchParamId = A.SearchParamId + AND B.Text = A.Text + AND (B.TextOverflow = A.TextOverflow + OR B.TextOverflow IS NULL + AND A.TextOverflow IS NULL) + AND B.IsMin = A.IsMin + AND B.IsMax = A.IsMax) + OPTION (HASH JOIN); + DELETE A + FROM dbo.StringSearchParam AS A WITH (INDEX (1)) + WHERE EXISTS (SELECT * + FROM @StringSearchParamsDelete AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + AND B.SearchParamId = A.SearchParamId + AND B.Text = A.Text + AND (B.TextOverflow = A.TextOverflow + OR B.TextOverflow IS NULL + AND A.TextOverflow IS NULL) + AND B.IsMin = A.IsMin + AND B.IsMax = A.IsMax); + INSERT INTO @StringSearchParamsInsert (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text, TextOverflow, IsMin, IsMax) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + Text, + TextOverflow, + IsMin, + IsMax + FROM @StringSearchParams AS A + WHERE NOT EXISTS (SELECT * + FROM @StringSearchParamsCurrent AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + AND B.SearchParamId = A.SearchParamId + AND B.Text = A.Text + AND (B.TextOverflow = A.TextOverflow + OR B.TextOverflow IS NULL + AND A.TextOverflow IS NULL) + AND B.IsMin = A.IsMin + AND B.IsMax = A.IsMax) + OPTION (HASH JOIN); + INSERT INTO @UriSearchParamsCurrent (ResourceTypeId, ResourceSurrogateId, SearchParamId, Uri) + SELECT A.ResourceTypeId, + A.ResourceSurrogateId, + SearchParamId, + Uri + FROM dbo.UriSearchParam AS A + INNER JOIN + @Ids AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + INSERT INTO @UriSearchParamsDelete (ResourceTypeId, ResourceSurrogateId, SearchParamId, Uri) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + Uri + FROM @UriSearchParamsCurrent AS A + WHERE NOT EXISTS (SELECT * + FROM @UriSearchParams AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + AND B.SearchParamId = A.SearchParamId + AND B.Uri = A.Uri) + OPTION (HASH JOIN); + DELETE A + FROM dbo.UriSearchParam AS A WITH (INDEX (1)) + WHERE EXISTS (SELECT * + FROM @UriSearchParamsDelete AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + AND B.SearchParamId = A.SearchParamId + AND B.Uri = A.Uri); + INSERT INTO @UriSearchParamsInsert (ResourceTypeId, ResourceSurrogateId, SearchParamId, Uri) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + Uri + FROM @UriSearchParams AS A + WHERE NOT EXISTS (SELECT * + FROM @UriSearchParamsCurrent AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + AND B.SearchParamId = A.SearchParamId + AND B.Uri = A.Uri) + OPTION (HASH JOIN); + INSERT INTO @NumberSearchParamsCurrent (ResourceTypeId, ResourceSurrogateId, SearchParamId, SingleValue, LowValue, HighValue) + SELECT A.ResourceTypeId, + A.ResourceSurrogateId, + SearchParamId, + SingleValue, + LowValue, + HighValue + FROM dbo.NumberSearchParam AS A + INNER JOIN + @Ids AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + INSERT INTO @NumberSearchParamsDelete (ResourceTypeId, ResourceSurrogateId, SearchParamId, SingleValue, LowValue, HighValue) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SingleValue, + LowValue, + HighValue + FROM @NumberSearchParamsCurrent AS A + WHERE NOT EXISTS (SELECT * + FROM @NumberSearchParams AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + AND B.SearchParamId = A.SearchParamId + AND (B.SingleValue = A.SingleValue + OR B.SingleValue IS NULL + AND A.SingleValue IS NULL) + AND (B.LowValue = A.LowValue + OR B.LowValue IS NULL + AND A.LowValue IS NULL) + AND (B.HighValue = A.HighValue + OR B.HighValue IS NULL + AND A.HighValue IS NULL)) + OPTION (HASH JOIN); + DELETE A + FROM dbo.NumberSearchParam AS A WITH (INDEX (1)) + WHERE EXISTS (SELECT * + FROM @NumberSearchParamsDelete AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + AND B.SearchParamId = A.SearchParamId + AND (B.SingleValue = A.SingleValue + OR B.SingleValue IS NULL + AND A.SingleValue IS NULL) + AND (B.LowValue = A.LowValue + OR B.LowValue IS NULL + AND A.LowValue IS NULL) + AND (B.HighValue = A.HighValue + OR B.HighValue IS NULL + AND A.HighValue IS NULL)); + INSERT INTO @NumberSearchParamsInsert (ResourceTypeId, ResourceSurrogateId, SearchParamId, SingleValue, LowValue, HighValue) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SingleValue, + LowValue, + HighValue + FROM @NumberSearchParams AS A + WHERE NOT EXISTS (SELECT * + FROM @NumberSearchParamsCurrent AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + AND B.SearchParamId = A.SearchParamId + AND (B.SingleValue = A.SingleValue + OR B.SingleValue IS NULL + AND A.SingleValue IS NULL) + AND (B.LowValue = A.LowValue + OR B.LowValue IS NULL + AND A.LowValue IS NULL) + AND (B.HighValue = A.HighValue + OR B.HighValue IS NULL + AND A.HighValue IS NULL)) + OPTION (HASH JOIN); + INSERT INTO @QuantitySearchParamsCurrent (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, QuantityCodeId, SingleValue, LowValue, HighValue) + SELECT A.ResourceTypeId, + A.ResourceSurrogateId, + SearchParamId, + SystemId, + QuantityCodeId, + SingleValue, + LowValue, + HighValue + FROM dbo.QuantitySearchParam AS A + INNER JOIN + @Ids AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + INSERT INTO @QuantitySearchParamsDelete (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, QuantityCodeId, SingleValue, LowValue, HighValue) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId, + QuantityCodeId, + SingleValue, + LowValue, + HighValue + FROM @QuantitySearchParamsCurrent AS A + WHERE NOT EXISTS (SELECT * + FROM @QuantitySearchParams AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + AND B.SearchParamId = A.SearchParamId + AND (B.SystemId = A.SystemId + OR B.SystemId IS NULL + AND A.SystemId IS NULL) + AND (B.QuantityCodeId = A.QuantityCodeId + OR B.QuantityCodeId IS NULL + AND A.QuantityCodeId IS NULL) + AND (B.SingleValue = A.SingleValue + OR B.SingleValue IS NULL + AND A.SingleValue IS NULL) + AND (B.LowValue = A.LowValue + OR B.LowValue IS NULL + AND A.LowValue IS NULL) + AND (B.HighValue = A.HighValue + OR B.HighValue IS NULL + AND A.HighValue IS NULL)) + OPTION (HASH JOIN); + DELETE A + FROM dbo.QuantitySearchParam AS A WITH (INDEX (1)) + WHERE EXISTS (SELECT * + FROM @QuantitySearchParamsDelete AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + AND B.SearchParamId = A.SearchParamId + AND (B.SystemId = A.SystemId + OR B.SystemId IS NULL + AND A.SystemId IS NULL) + AND (B.QuantityCodeId = A.QuantityCodeId + OR B.QuantityCodeId IS NULL + AND A.QuantityCodeId IS NULL) + AND (B.SingleValue = A.SingleValue + OR B.SingleValue IS NULL + AND A.SingleValue IS NULL) + AND (B.LowValue = A.LowValue + OR B.LowValue IS NULL + AND A.LowValue IS NULL) + AND (B.HighValue = A.HighValue + OR B.HighValue IS NULL + AND A.HighValue IS NULL)); + INSERT INTO @QuantitySearchParamsInsert (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, QuantityCodeId, SingleValue, LowValue, HighValue) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId, + QuantityCodeId, + SingleValue, + LowValue, + HighValue + FROM @QuantitySearchParams AS A + WHERE NOT EXISTS (SELECT * + FROM @QuantitySearchParamsCurrent AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + AND B.SearchParamId = A.SearchParamId + AND (B.SystemId = A.SystemId + OR B.SystemId IS NULL + AND A.SystemId IS NULL) + AND (B.QuantityCodeId = A.QuantityCodeId + OR B.QuantityCodeId IS NULL + AND A.QuantityCodeId IS NULL) + AND (B.SingleValue = A.SingleValue + OR B.SingleValue IS NULL + AND A.SingleValue IS NULL) + AND (B.LowValue = A.LowValue + OR B.LowValue IS NULL + AND A.LowValue IS NULL) + AND (B.HighValue = A.HighValue + OR B.HighValue IS NULL + AND A.HighValue IS NULL)) + OPTION (HASH JOIN); + INSERT INTO @DateTimeSearchParamsCurrent (ResourceTypeId, ResourceSurrogateId, SearchParamId, StartDateTime, EndDateTime, IsLongerThanADay, IsMin, IsMax) + SELECT A.ResourceTypeId, + A.ResourceSurrogateId, + SearchParamId, + StartDateTime, + EndDateTime, + IsLongerThanADay, + IsMin, + IsMax + FROM dbo.DateTimeSearchParam AS A + INNER JOIN + @Ids AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + INSERT INTO @DateTimeSearchParamsDelete (ResourceTypeId, ResourceSurrogateId, SearchParamId, StartDateTime, EndDateTime, IsLongerThanADay, IsMin, IsMax) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + StartDateTime, + EndDateTime, + IsLongerThanADay, + IsMin, + IsMax + FROM @DateTimeSearchParamsCurrent AS A + WHERE NOT EXISTS (SELECT * + FROM @DateTimeSearchParams AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + AND B.SearchParamId = A.SearchParamId + AND B.StartDateTime = A.StartDateTime + AND B.EndDateTime = A.EndDateTime + AND B.IsLongerThanADay = A.IsLongerThanADay + AND B.IsMin = A.IsMin + AND B.IsMax = A.IsMax) + OPTION (HASH JOIN); + DELETE A + FROM dbo.DateTimeSearchParam AS A WITH (INDEX (1)) + WHERE EXISTS (SELECT * + FROM @DateTimeSearchParamsDelete AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + AND B.SearchParamId = A.SearchParamId + AND B.StartDateTime = A.StartDateTime + AND B.EndDateTime = A.EndDateTime + AND B.IsLongerThanADay = A.IsLongerThanADay + AND B.IsMin = A.IsMin + AND B.IsMax = A.IsMax); + INSERT INTO @DateTimeSearchParamsInsert (ResourceTypeId, ResourceSurrogateId, SearchParamId, StartDateTime, EndDateTime, IsLongerThanADay, IsMin, IsMax) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + StartDateTime, + EndDateTime, + IsLongerThanADay, + IsMin, + IsMax + FROM @DateTimeSearchParams AS A + WHERE NOT EXISTS (SELECT * + FROM @DateTimeSearchParamsCurrent AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + AND B.SearchParamId = A.SearchParamId + AND B.StartDateTime = A.StartDateTime + AND B.EndDateTime = A.EndDateTime + AND B.IsLongerThanADay = A.IsLongerThanADay + AND B.IsMin = A.IsMin + AND B.IsMax = A.IsMax) + OPTION (HASH JOIN); + INSERT INTO @ReferenceTokenCompositeSearchParamsCurrent (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri1, ReferenceResourceTypeId1, ReferenceResourceId1, ReferenceResourceVersion1, SystemId2, Code2, CodeOverflow2) + SELECT A.ResourceTypeId, + A.ResourceSurrogateId, + SearchParamId, + BaseUri1, + ReferenceResourceTypeId1, + ReferenceResourceId1, + ReferenceResourceVersion1, + SystemId2, + Code2, + CodeOverflow2 + FROM dbo.ReferenceTokenCompositeSearchParam AS A + INNER JOIN + @Ids AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + INSERT INTO @ReferenceTokenCompositeSearchParamsDelete (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri1, ReferenceResourceTypeId1, ReferenceResourceId1, ReferenceResourceVersion1, SystemId2, Code2, CodeOverflow2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + BaseUri1, + ReferenceResourceTypeId1, + ReferenceResourceId1, + ReferenceResourceVersion1, + SystemId2, + Code2, + CodeOverflow2 + FROM @ReferenceTokenCompositeSearchParamsCurrent AS A + WHERE NOT EXISTS (SELECT * + FROM @ReferenceTokenCompositeSearchParams AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + AND B.SearchParamId = A.SearchParamId + AND (B.BaseUri1 = A.BaseUri1 + OR B.BaseUri1 IS NULL + AND A.BaseUri1 IS NULL) + AND (B.ReferenceResourceTypeId1 = A.ReferenceResourceTypeId1 + OR B.ReferenceResourceTypeId1 IS NULL + AND A.ReferenceResourceTypeId1 IS NULL) + AND B.ReferenceResourceId1 = A.ReferenceResourceId1 + AND (B.SystemId2 = A.SystemId2 + OR B.SystemId2 IS NULL + AND A.SystemId2 IS NULL) + AND B.Code2 = A.Code2 + AND (B.CodeOverflow2 = A.CodeOverflow2 + OR B.CodeOverflow2 IS NULL + AND A.CodeOverflow2 IS NULL)) + OPTION (HASH JOIN); + DELETE A + FROM dbo.ReferenceTokenCompositeSearchParam AS A WITH (INDEX (1)) + WHERE EXISTS (SELECT * + FROM @ReferenceTokenCompositeSearchParamsDelete AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + AND B.SearchParamId = A.SearchParamId + AND (B.BaseUri1 = A.BaseUri1 + OR B.BaseUri1 IS NULL + AND A.BaseUri1 IS NULL) + AND (B.ReferenceResourceTypeId1 = A.ReferenceResourceTypeId1 + OR B.ReferenceResourceTypeId1 IS NULL + AND A.ReferenceResourceTypeId1 IS NULL) + AND B.ReferenceResourceId1 = A.ReferenceResourceId1 + AND (B.SystemId2 = A.SystemId2 + OR B.SystemId2 IS NULL + AND A.SystemId2 IS NULL) + AND B.Code2 = A.Code2 + AND (B.CodeOverflow2 = A.CodeOverflow2 + OR B.CodeOverflow2 IS NULL + AND A.CodeOverflow2 IS NULL)); + INSERT INTO @ReferenceTokenCompositeSearchParamsInsert (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri1, ReferenceResourceTypeId1, ReferenceResourceId1, ReferenceResourceVersion1, SystemId2, Code2, CodeOverflow2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + BaseUri1, + ReferenceResourceTypeId1, + ReferenceResourceId1, + ReferenceResourceVersion1, + SystemId2, + Code2, + CodeOverflow2 + FROM @ReferenceTokenCompositeSearchParams AS A + WHERE NOT EXISTS (SELECT * + FROM @ReferenceTokenCompositeSearchParamsCurrent AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + AND B.SearchParamId = A.SearchParamId + AND (B.BaseUri1 = A.BaseUri1 + OR B.BaseUri1 IS NULL + AND A.BaseUri1 IS NULL) + AND (B.ReferenceResourceTypeId1 = A.ReferenceResourceTypeId1 + OR B.ReferenceResourceTypeId1 IS NULL + AND A.ReferenceResourceTypeId1 IS NULL) + AND B.ReferenceResourceId1 = A.ReferenceResourceId1 + AND (B.SystemId2 = A.SystemId2 + OR B.SystemId2 IS NULL + AND A.SystemId2 IS NULL) + AND B.Code2 = A.Code2 + AND (B.CodeOverflow2 = A.CodeOverflow2 + OR B.CodeOverflow2 IS NULL + AND A.CodeOverflow2 IS NULL)) + OPTION (HASH JOIN); + INSERT INTO @TokenTokenCompositeSearchParamsCurrent (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SystemId2, Code2, CodeOverflow2) + SELECT A.ResourceTypeId, + A.ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SystemId2, + Code2, + CodeOverflow2 + FROM dbo.TokenTokenCompositeSearchParam AS A + INNER JOIN + @Ids AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + INSERT INTO @TokenTokenCompositeSearchParamsDelete (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SystemId2, Code2, CodeOverflow2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SystemId2, + Code2, + CodeOverflow2 + FROM @TokenTokenCompositeSearchParamsCurrent AS A + WHERE NOT EXISTS (SELECT * + FROM @TokenTokenCompositeSearchParams AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + AND B.SearchParamId = A.SearchParamId + AND (B.SystemId1 = A.SystemId1 + OR B.SystemId1 IS NULL + AND A.SystemId1 IS NULL) + AND B.Code1 = A.Code1 + AND (B.CodeOverflow1 = A.CodeOverflow1 + OR B.CodeOverflow1 IS NULL + AND A.CodeOverflow1 IS NULL) + AND (B.SystemId2 = A.SystemId2 + OR B.SystemId2 IS NULL + AND A.SystemId2 IS NULL) + AND B.Code2 = A.Code2 + AND (B.CodeOverflow2 = A.CodeOverflow2 + OR B.CodeOverflow2 IS NULL + AND A.CodeOverflow2 IS NULL)) + OPTION (HASH JOIN); + DELETE A + FROM dbo.TokenTokenCompositeSearchParam AS A WITH (INDEX (1)) + WHERE EXISTS (SELECT * + FROM @TokenTokenCompositeSearchParamsDelete AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + AND B.SearchParamId = A.SearchParamId + AND (B.SystemId1 = A.SystemId1 + OR B.SystemId1 IS NULL + AND A.SystemId1 IS NULL) + AND B.Code1 = A.Code1 + AND (B.CodeOverflow1 = A.CodeOverflow1 + OR B.CodeOverflow1 IS NULL + AND A.CodeOverflow1 IS NULL) + AND (B.SystemId2 = A.SystemId2 + OR B.SystemId2 IS NULL + AND A.SystemId2 IS NULL) + AND B.Code2 = A.Code2 + AND (B.CodeOverflow2 = A.CodeOverflow2 + OR B.CodeOverflow2 IS NULL + AND A.CodeOverflow2 IS NULL)); + INSERT INTO @TokenTokenCompositeSearchParamsInsert (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SystemId2, Code2, CodeOverflow2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SystemId2, + Code2, + CodeOverflow2 + FROM @TokenTokenCompositeSearchParams AS A + WHERE NOT EXISTS (SELECT * + FROM @TokenTokenCompositeSearchParamsCurrent AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + AND B.SearchParamId = A.SearchParamId + AND (B.SystemId1 = A.SystemId1 + OR B.SystemId1 IS NULL + AND A.SystemId1 IS NULL) + AND B.Code1 = A.Code1 + AND (B.CodeOverflow1 = A.CodeOverflow1 + OR B.CodeOverflow1 IS NULL + AND A.CodeOverflow1 IS NULL) + AND (B.SystemId2 = A.SystemId2 + OR B.SystemId2 IS NULL + AND A.SystemId2 IS NULL) + AND B.Code2 = A.Code2 + AND (B.CodeOverflow2 = A.CodeOverflow2 + OR B.CodeOverflow2 IS NULL + AND A.CodeOverflow2 IS NULL)) + OPTION (HASH JOIN); + INSERT INTO @TokenDateTimeCompositeSearchParamsCurrent (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, StartDateTime2, EndDateTime2, IsLongerThanADay2) + SELECT A.ResourceTypeId, + A.ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + StartDateTime2, + EndDateTime2, + IsLongerThanADay2 + FROM dbo.TokenDateTimeCompositeSearchParam AS A + INNER JOIN + @Ids AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + INSERT INTO @TokenDateTimeCompositeSearchParamsDelete (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, StartDateTime2, EndDateTime2, IsLongerThanADay2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + StartDateTime2, + EndDateTime2, + IsLongerThanADay2 + FROM @TokenDateTimeCompositeSearchParamsCurrent AS A + WHERE NOT EXISTS (SELECT * + FROM @TokenDateTimeCompositeSearchParams AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + AND B.SearchParamId = A.SearchParamId + AND (B.SystemId1 = A.SystemId1 + OR B.SystemId1 IS NULL + AND A.SystemId1 IS NULL) + AND B.Code1 = A.Code1 + AND (B.CodeOverflow1 = A.CodeOverflow1 + OR B.CodeOverflow1 IS NULL + AND A.CodeOverflow1 IS NULL) + AND B.StartDateTime2 = A.StartDateTime2 + AND B.EndDateTime2 = A.EndDateTime2 + AND B.IsLongerThanADay2 = A.IsLongerThanADay2) + OPTION (HASH JOIN); + DELETE A + FROM dbo.TokenDateTimeCompositeSearchParam AS A WITH (INDEX (1)) + WHERE EXISTS (SELECT * + FROM @TokenDateTimeCompositeSearchParamsDelete AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + AND B.SearchParamId = A.SearchParamId + AND (B.SystemId1 = A.SystemId1 + OR B.SystemId1 IS NULL + AND A.SystemId1 IS NULL) + AND B.Code1 = A.Code1 + AND (B.CodeOverflow1 = A.CodeOverflow1 + OR B.CodeOverflow1 IS NULL + AND A.CodeOverflow1 IS NULL) + AND B.StartDateTime2 = A.StartDateTime2 + AND B.EndDateTime2 = A.EndDateTime2 + AND B.IsLongerThanADay2 = A.IsLongerThanADay2); + INSERT INTO @TokenDateTimeCompositeSearchParamsInsert (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, StartDateTime2, EndDateTime2, IsLongerThanADay2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + StartDateTime2, + EndDateTime2, + IsLongerThanADay2 + FROM @TokenDateTimeCompositeSearchParams AS A + WHERE NOT EXISTS (SELECT * + FROM @TokenDateTimeCompositeSearchParamsCurrent AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + AND B.SearchParamId = A.SearchParamId + AND (B.SystemId1 = A.SystemId1 + OR B.SystemId1 IS NULL + AND A.SystemId1 IS NULL) + AND B.Code1 = A.Code1 + AND (B.CodeOverflow1 = A.CodeOverflow1 + OR B.CodeOverflow1 IS NULL + AND A.CodeOverflow1 IS NULL) + AND B.StartDateTime2 = A.StartDateTime2 + AND B.EndDateTime2 = A.EndDateTime2 + AND B.IsLongerThanADay2 = A.IsLongerThanADay2) + OPTION (HASH JOIN); + INSERT INTO @TokenQuantityCompositeSearchParamsCurrent (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, SystemId2, QuantityCodeId2, LowValue2, HighValue2) + SELECT A.ResourceTypeId, + A.ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SingleValue2, + SystemId2, + QuantityCodeId2, + LowValue2, + HighValue2 + FROM dbo.TokenQuantityCompositeSearchParam AS A + INNER JOIN + @Ids AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + INSERT INTO @TokenQuantityCompositeSearchParamsDelete (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, SystemId2, QuantityCodeId2, LowValue2, HighValue2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SingleValue2, + SystemId2, + QuantityCodeId2, + LowValue2, + HighValue2 + FROM @TokenQuantityCompositeSearchParamsCurrent AS A + WHERE NOT EXISTS (SELECT * + FROM @TokenQuantityCompositeSearchParams AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + AND B.SearchParamId = A.SearchParamId + AND (B.SystemId1 = A.SystemId1 + OR B.SystemId1 IS NULL + AND A.SystemId1 IS NULL) + AND B.Code1 = A.Code1 + AND (B.CodeOverflow1 = A.CodeOverflow1 + OR B.CodeOverflow1 IS NULL + AND A.CodeOverflow1 IS NULL) + AND (B.SingleValue2 = A.SingleValue2 + OR B.SingleValue2 IS NULL + AND A.SingleValue2 IS NULL) + AND (B.SystemId2 = A.SystemId2 + OR B.SystemId2 IS NULL + AND A.SystemId2 IS NULL) + AND (B.QuantityCodeId2 = A.QuantityCodeId2 + OR B.QuantityCodeId2 IS NULL + AND A.QuantityCodeId2 IS NULL) + AND (B.LowValue2 = A.LowValue2 + OR B.LowValue2 IS NULL + AND A.LowValue2 IS NULL) + AND (B.HighValue2 = A.HighValue2 + OR B.HighValue2 IS NULL + AND A.HighValue2 IS NULL)) + OPTION (HASH JOIN); + DELETE A + FROM dbo.TokenQuantityCompositeSearchParam AS A WITH (INDEX (1)) + WHERE EXISTS (SELECT * + FROM @TokenQuantityCompositeSearchParamsDelete AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + AND B.SearchParamId = A.SearchParamId + AND (B.SystemId1 = A.SystemId1 + OR B.SystemId1 IS NULL + AND A.SystemId1 IS NULL) + AND B.Code1 = A.Code1 + AND (B.CodeOverflow1 = A.CodeOverflow1 + OR B.CodeOverflow1 IS NULL + AND A.CodeOverflow1 IS NULL) + AND (B.SingleValue2 = A.SingleValue2 + OR B.SingleValue2 IS NULL + AND A.SingleValue2 IS NULL) + AND (B.SystemId2 = A.SystemId2 + OR B.SystemId2 IS NULL + AND A.SystemId2 IS NULL) + AND (B.QuantityCodeId2 = A.QuantityCodeId2 + OR B.QuantityCodeId2 IS NULL + AND A.QuantityCodeId2 IS NULL) + AND (B.LowValue2 = A.LowValue2 + OR B.LowValue2 IS NULL + AND A.LowValue2 IS NULL) + AND (B.HighValue2 = A.HighValue2 + OR B.HighValue2 IS NULL + AND A.HighValue2 IS NULL)); + INSERT INTO @TokenQuantityCompositeSearchParamsInsert (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, SystemId2, QuantityCodeId2, LowValue2, HighValue2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SingleValue2, + SystemId2, + QuantityCodeId2, + LowValue2, + HighValue2 + FROM @TokenQuantityCompositeSearchParams AS A + WHERE NOT EXISTS (SELECT * + FROM @TokenQuantityCompositeSearchParamsCurrent AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + AND B.SearchParamId = A.SearchParamId + AND (B.SystemId1 = A.SystemId1 + OR B.SystemId1 IS NULL + AND A.SystemId1 IS NULL) + AND B.Code1 = A.Code1 + AND (B.CodeOverflow1 = A.CodeOverflow1 + OR B.CodeOverflow1 IS NULL + AND A.CodeOverflow1 IS NULL) + AND (B.SingleValue2 = A.SingleValue2 + OR B.SingleValue2 IS NULL + AND A.SingleValue2 IS NULL) + AND (B.SystemId2 = A.SystemId2 + OR B.SystemId2 IS NULL + AND A.SystemId2 IS NULL) + AND (B.QuantityCodeId2 = A.QuantityCodeId2 + OR B.QuantityCodeId2 IS NULL + AND A.QuantityCodeId2 IS NULL) + AND (B.LowValue2 = A.LowValue2 + OR B.LowValue2 IS NULL + AND A.LowValue2 IS NULL) + AND (B.HighValue2 = A.HighValue2 + OR B.HighValue2 IS NULL + AND A.HighValue2 IS NULL)) + OPTION (HASH JOIN); + INSERT INTO @TokenNumberNumberCompositeSearchParamsCurrent (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, LowValue2, HighValue2, SingleValue3, LowValue3, HighValue3, HasRange) + SELECT A.ResourceTypeId, + A.ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SingleValue2, + LowValue2, + HighValue2, + SingleValue3, + LowValue3, + HighValue3, + HasRange + FROM dbo.TokenNumberNumberCompositeSearchParam AS A + INNER JOIN + @Ids AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + INSERT INTO @TokenNumberNumberCompositeSearchParamsDelete (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, LowValue2, HighValue2, SingleValue3, LowValue3, HighValue3, HasRange) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SingleValue2, + LowValue2, + HighValue2, + SingleValue3, + LowValue3, + HighValue3, + HasRange + FROM @TokenNumberNumberCompositeSearchParamsCurrent AS A + WHERE NOT EXISTS (SELECT * + FROM @TokenNumberNumberCompositeSearchParams AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + AND B.SearchParamId = A.SearchParamId + AND (B.SystemId1 = A.SystemId1 + OR B.SystemId1 IS NULL + AND A.SystemId1 IS NULL) + AND B.Code1 = A.Code1 + AND (B.CodeOverflow1 = A.CodeOverflow1 + OR B.CodeOverflow1 IS NULL + AND A.CodeOverflow1 IS NULL) + AND (B.SingleValue2 = A.SingleValue2 + OR B.SingleValue2 IS NULL + AND A.SingleValue2 IS NULL) + AND (B.LowValue2 = A.LowValue2 + OR B.LowValue2 IS NULL + AND A.LowValue2 IS NULL) + AND (B.HighValue2 = A.HighValue2 + OR B.HighValue2 IS NULL + AND A.HighValue2 IS NULL) + AND (B.SingleValue3 = A.SingleValue3 + OR B.SingleValue3 IS NULL + AND A.SingleValue3 IS NULL) + AND (B.LowValue3 = A.LowValue3 + OR B.LowValue3 IS NULL + AND A.LowValue3 IS NULL) + AND (B.HighValue3 = A.HighValue3 + OR B.HighValue3 IS NULL + AND A.HighValue3 IS NULL) + AND B.HasRange = A.HasRange) + OPTION (HASH JOIN); + DELETE A + FROM dbo.TokenNumberNumberCompositeSearchParam AS A WITH (INDEX (1)) + WHERE EXISTS (SELECT * + FROM @TokenNumberNumberCompositeSearchParamsDelete AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + AND B.SearchParamId = A.SearchParamId + AND (B.SystemId1 = A.SystemId1 + OR B.SystemId1 IS NULL + AND A.SystemId1 IS NULL) + AND B.Code1 = A.Code1 + AND (B.CodeOverflow1 = A.CodeOverflow1 + OR B.CodeOverflow1 IS NULL + AND A.CodeOverflow1 IS NULL) + AND (B.SingleValue2 = A.SingleValue2 + OR B.SingleValue2 IS NULL + AND A.SingleValue2 IS NULL) + AND (B.LowValue2 = A.LowValue2 + OR B.LowValue2 IS NULL + AND A.LowValue2 IS NULL) + AND (B.HighValue2 = A.HighValue2 + OR B.HighValue2 IS NULL + AND A.HighValue2 IS NULL) + AND (B.SingleValue3 = A.SingleValue3 + OR B.SingleValue3 IS NULL + AND A.SingleValue3 IS NULL) + AND (B.LowValue3 = A.LowValue3 + OR B.LowValue3 IS NULL + AND A.LowValue3 IS NULL) + AND (B.HighValue3 = A.HighValue3 + OR B.HighValue3 IS NULL + AND A.HighValue3 IS NULL) + AND B.HasRange = A.HasRange); + INSERT INTO @TokenNumberNumberCompositeSearchParamsInsert (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, LowValue2, HighValue2, SingleValue3, LowValue3, HighValue3, HasRange) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SingleValue2, + LowValue2, + HighValue2, + SingleValue3, + LowValue3, + HighValue3, + HasRange + FROM @TokenNumberNumberCompositeSearchParams AS A + WHERE NOT EXISTS (SELECT * + FROM @TokenNumberNumberCompositeSearchParamsCurrent AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + AND B.SearchParamId = A.SearchParamId + AND (B.SystemId1 = A.SystemId1 + OR B.SystemId1 IS NULL + AND A.SystemId1 IS NULL) + AND B.Code1 = A.Code1 + AND (B.CodeOverflow1 = A.CodeOverflow1 + OR B.CodeOverflow1 IS NULL + AND A.CodeOverflow1 IS NULL) + AND (B.SingleValue2 = A.SingleValue2 + OR B.SingleValue2 IS NULL + AND A.SingleValue2 IS NULL) + AND (B.LowValue2 = A.LowValue2 + OR B.LowValue2 IS NULL + AND A.LowValue2 IS NULL) + AND (B.HighValue2 = A.HighValue2 + OR B.HighValue2 IS NULL + AND A.HighValue2 IS NULL) + AND (B.SingleValue3 = A.SingleValue3 + OR B.SingleValue3 IS NULL + AND A.SingleValue3 IS NULL) + AND (B.LowValue3 = A.LowValue3 + OR B.LowValue3 IS NULL + AND A.LowValue3 IS NULL) + AND (B.HighValue3 = A.HighValue3 + OR B.HighValue3 IS NULL + AND A.HighValue3 IS NULL) + AND B.HasRange = A.HasRange) + OPTION (HASH JOIN); + INSERT INTO dbo.ResourceWriteClaim (ResourceSurrogateId, ClaimTypeId, ClaimValue) + SELECT ResourceSurrogateId, + ClaimTypeId, + ClaimValue + FROM @ResourceWriteClaimsInsert; + INSERT INTO dbo.ReferenceSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri, ReferenceResourceTypeId, ReferenceResourceId, ReferenceResourceVersion) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + BaseUri, + ReferenceResourceTypeId, + ReferenceResourceId, + ReferenceResourceVersion + FROM @ReferenceSearchParamsInsert; + INSERT INTO dbo.TokenSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, Code, CodeOverflow) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId, + Code, + CodeOverflow + FROM @TokenSearchParamsInsert; + INSERT INTO dbo.TokenStringCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, Text2, TextOverflow2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + Text2, + TextOverflow2 + FROM @TokenStringCompositeSearchParamsInsert; + INSERT INTO dbo.TokenText (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + Text + FROM @TokenTextsInsert; + INSERT INTO dbo.StringSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text, TextOverflow, IsMin, IsMax) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + Text, + TextOverflow, + IsMin, + IsMax + FROM @StringSearchParamsInsert; + INSERT INTO dbo.UriSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, Uri) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + Uri + FROM @UriSearchParamsInsert; + INSERT INTO dbo.NumberSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SingleValue, LowValue, HighValue) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SingleValue, + LowValue, + HighValue + FROM @NumberSearchParamsInsert; + INSERT INTO dbo.QuantitySearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, QuantityCodeId, SingleValue, LowValue, HighValue) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId, + QuantityCodeId, + SingleValue, + LowValue, + HighValue + FROM @QuantitySearchParamsInsert; + INSERT INTO dbo.DateTimeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, StartDateTime, EndDateTime, IsLongerThanADay, IsMin, IsMax) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + StartDateTime, + EndDateTime, + IsLongerThanADay, + IsMin, + IsMax + FROM @DateTimeSearchParamsInsert; + INSERT INTO dbo.ReferenceTokenCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri1, ReferenceResourceTypeId1, ReferenceResourceId1, ReferenceResourceVersion1, SystemId2, Code2, CodeOverflow2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + BaseUri1, + ReferenceResourceTypeId1, + ReferenceResourceId1, + ReferenceResourceVersion1, + SystemId2, + Code2, + CodeOverflow2 + FROM @ReferenceTokenCompositeSearchParamsInsert; + INSERT INTO dbo.TokenTokenCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SystemId2, Code2, CodeOverflow2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SystemId2, + Code2, + CodeOverflow2 + FROM @TokenTokenCompositeSearchParamsInsert; + INSERT INTO dbo.TokenDateTimeCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, StartDateTime2, EndDateTime2, IsLongerThanADay2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + StartDateTime2, + EndDateTime2, + IsLongerThanADay2 + FROM @TokenDateTimeCompositeSearchParamsInsert; + INSERT INTO dbo.TokenQuantityCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, SystemId2, QuantityCodeId2, LowValue2, HighValue2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SingleValue2, + SystemId2, + QuantityCodeId2, + LowValue2, + HighValue2 + FROM @TokenQuantityCompositeSearchParamsInsert; + INSERT INTO dbo.TokenNumberNumberCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, LowValue2, HighValue2, SingleValue3, LowValue3, HighValue3, HasRange) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SingleValue2, + LowValue2, + HighValue2, + SingleValue3, + LowValue3, + HighValue3, + HasRange + FROM @TokenNumberNumberCompositeSearchParamsInsert; + COMMIT TRANSACTION; + SET @FailedResources = (SELECT count(*) + FROM @Resources) - @Rows; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows; +END TRY +BEGIN CATCH + IF @@trancount > 0 + ROLLBACK; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE VIEW dbo.CurrentResource +AS +SELECT * +FROM dbo.Resource +WHERE IsHistory = 0; + +GO diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersion.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersion.cs index 0c9e4bcc28..b0e9e4be93 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersion.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersion.cs @@ -118,5 +118,6 @@ public enum SchemaVersion V106 = 106, V107 = 107, V108 = 108, + V109 = 109, } } diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersionConstants.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersionConstants.cs index 48291dae8a..731bd533b2 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersionConstants.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersionConstants.cs @@ -8,7 +8,7 @@ namespace Microsoft.Health.Fhir.SqlServer.Features.Schema public static class SchemaVersionConstants { public const int Min = (int)SchemaVersion.V103; - public const int Max = (int)SchemaVersion.V108; + public const int Max = (int)SchemaVersion.V109; public const int MinForUpgrade = (int)SchemaVersion.V103; // this is used for upgrade tests only public const int SearchParameterStatusSchemaVersion = (int)SchemaVersion.V6; public const int SupportForReferencesWithMissingTypeVersion = (int)SchemaVersion.V7; diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/CheckSearchParamCacheConsistency.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/CheckSearchParamCacheConsistency.sql new file mode 100644 index 0000000000..c635d3837f --- /dev/null +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/CheckSearchParamCacheConsistency.sql @@ -0,0 +1,32 @@ +--DROP PROCEDURE dbo.CheckSearchParamCacheConsistency +GO +CREATE OR ALTER PROCEDURE dbo.CheckSearchParamCacheConsistency + @TargetSearchParamLastUpdated varchar(100) + ,@SyncStartDate datetime2(7) + ,@ActiveHostsSince datetime2(7) + ,@StalenessThresholdMinutes int = 10 +AS +set nocount on +SELECT HostName + ,CAST(NULL AS datetime2(7)) AS SyncEventDate + ,CAST(NULL AS nvarchar(3500)) AS EventText + FROM dbo.EventLog + WHERE EventDate >= @ActiveHostsSince + AND HostName IS NOT NULL + AND Process = 'DequeueJob' + +UNION ALL + +SELECT HostName + ,EventDate + ,EventText + FROM dbo.EventLog + WHERE EventDate >= @SyncStartDate + AND HostName IS NOT NULL + AND Process = 'SearchParameterCacheRefresh' + AND Status = 'End' +GO +INSERT INTO dbo.Parameters (Id, Char) SELECT 'DequeueJob', 'LogEvent' WHERE NOT EXISTS (SELECT * FROM dbo.Parameters WHERE Id = 'DequeueJob') +GO +INSERT INTO Parameters (Id,Char) SELECT 'SearchParameterCacheRefresh','LogEvent' +GO diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/DequeueJob.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/DequeueJob.sql index b7c3c46453..ebbfe6b175 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/DequeueJob.sql +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/DequeueJob.sql @@ -155,3 +155,5 @@ BEGIN CATCH THROW END CATCH GO +INSERT INTO Parameters (Id,Char) SELECT 'DequeueJob','LogEvent' +GO diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/MergeSearchParams.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/MergeSearchParams.sql index 5347f2e50b..9bc4ddfbf6 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/MergeSearchParams.sql +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/MergeSearchParams.sql @@ -1,4 +1,22 @@ CREATE PROCEDURE dbo.MergeSearchParams @SearchParams dbo.SearchParamList READONLY + ,@IsResourceChangeCaptureEnabled bit = 0 + ,@TransactionId bigint = NULL + ,@Resources dbo.ResourceList READONLY + ,@ResourceWriteClaims dbo.ResourceWriteClaimList READONLY + ,@ReferenceSearchParams dbo.ReferenceSearchParamList READONLY + ,@TokenSearchParams dbo.TokenSearchParamList READONLY + ,@TokenTexts dbo.TokenTextList READONLY + ,@StringSearchParams dbo.StringSearchParamList READONLY + ,@UriSearchParams dbo.UriSearchParamList READONLY + ,@NumberSearchParams dbo.NumberSearchParamList READONLY + ,@QuantitySearchParams dbo.QuantitySearchParamList READONLY + ,@DateTimeSearchParms dbo.DateTimeSearchParamList READONLY + ,@ReferenceTokenCompositeSearchParams dbo.ReferenceTokenCompositeSearchParamList READONLY + ,@TokenTokenCompositeSearchParams dbo.TokenTokenCompositeSearchParamList READONLY + ,@TokenDateTimeCompositeSearchParams dbo.TokenDateTimeCompositeSearchParamList READONLY + ,@TokenQuantityCompositeSearchParams dbo.TokenQuantityCompositeSearchParamList READONLY + ,@TokenStringCompositeSearchParams dbo.TokenStringCompositeSearchParamList READONLY + ,@TokenNumberNumberCompositeSearchParams dbo.TokenNumberNumberCompositeSearchParamList READONLY AS set nocount on DECLARE @SP varchar(100) = object_name(@@procid) @@ -7,6 +25,7 @@ DECLARE @SP varchar(100) = object_name(@@procid) ,@LastUpdated datetimeoffset(7) = sysdatetimeoffset() ,@msg varchar(4000) ,@Rows int + ,@AffectedRows int = 0 ,@Uri varchar(4000) ,@Status varchar(20) @@ -39,6 +58,34 @@ BEGIN TRY THROW 50001, @msg, 1 END + IF EXISTS (SELECT * FROM @Resources) + BEGIN + EXECUTE dbo.MergeResources + @AffectedRows = @AffectedRows OUTPUT + ,@RaiseExceptionOnConflict = 1 + ,@IsResourceChangeCaptureEnabled = @IsResourceChangeCaptureEnabled + ,@TransactionId = @TransactionId + ,@SingleTransaction = 1 + ,@Resources = @Resources + ,@ResourceWriteClaims = @ResourceWriteClaims + ,@ReferenceSearchParams = @ReferenceSearchParams + ,@TokenSearchParams = @TokenSearchParams + ,@TokenTexts = @TokenTexts + ,@StringSearchParams = @StringSearchParams + ,@UriSearchParams = @UriSearchParams + ,@NumberSearchParams = @NumberSearchParams + ,@QuantitySearchParams = @QuantitySearchParams + ,@DateTimeSearchParms = @DateTimeSearchParms + ,@ReferenceTokenCompositeSearchParams = @ReferenceTokenCompositeSearchParams + ,@TokenTokenCompositeSearchParams = @TokenTokenCompositeSearchParams + ,@TokenDateTimeCompositeSearchParams = @TokenDateTimeCompositeSearchParams + ,@TokenQuantityCompositeSearchParams = @TokenQuantityCompositeSearchParams + ,@TokenStringCompositeSearchParams = @TokenStringCompositeSearchParams + ,@TokenNumberNumberCompositeSearchParams = @TokenNumberNumberCompositeSearchParams; + + SET @Rows = @Rows + @AffectedRows; + END + MERGE INTO dbo.SearchParam S USING @SearchParams I ON I.Uri = S.Uri WHEN MATCHED THEN diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/UpsertSearchParamsWithOptimisticConcurrency.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/UpsertSearchParamsWithOptimisticConcurrency.sql deleted file mode 100644 index e8546494fd..0000000000 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/UpsertSearchParamsWithOptimisticConcurrency.sql +++ /dev/null @@ -1,88 +0,0 @@ -/************************************************************* - Stored procedures - UpsertSearchParamsWithOptimisticConcurrency -**************************************************************/ --- --- STORED PROCEDURE --- UpsertSearchParams_2 --- --- DESCRIPTION --- Given a set of search parameters, creates or updates the parameters. --- Implements optimistic concurrency control using LastUpdated. --- --- PARAMETERS --- @searchParams --- * The updated existing search parameters or the new search parameters --- --- RETURN VALUE --- The IDs, URIs and LastUpdated of the search parameters that were inserted or updated. --- -CREATE PROCEDURE dbo.UpsertSearchParamsWithOptimisticConcurrency - @searchParams dbo.SearchParamList READONLY -AS -SET NOCOUNT ON; -DECLARE @SP varchar(100) = object_name(@@procid) - ,@Mode varchar(200) = null - ,@st datetime = getUTCdate() - -BEGIN TRANSACTION; - -DECLARE @lastUpdated AS DATETIMEOFFSET (7) = SYSDATETIMEOFFSET(); -DECLARE @summaryOfChanges TABLE ( - Uri VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL, - Action VARCHAR (20) NOT NULL); - --- Declare table to collect concurrency conflicts -DECLARE @conflictedRows TABLE ( - Uri VARCHAR(128) COLLATE Latin1_General_100_CS_AS NOT NULL -); - -BEGIN TRY --- Check for concurrency conflicts first using LastUpdated -INSERT INTO @conflictedRows (Uri) -SELECT sp.Uri -FROM @searchParams sp -INNER JOIN dbo.SearchParam existing WITH (TABLOCKX) - ON sp.Uri = existing.Uri -WHERE sp.LastUpdated != existing.LastUpdated; - --- If we have conflicts, raise an error -IF EXISTS (SELECT 1 FROM @conflictedRows) -BEGIN - DECLARE @conflictMessage NVARCHAR(4000); - SELECT @conflictMessage = CONCAT('Optimistic concurrency conflict detected for search parameters: ', - STRING_AGG(Uri, ', ')) - FROM @conflictedRows; - - ROLLBACK TRANSACTION; - THROW 50001, @conflictMessage, 1; -END - --- Acquire and hold an exclusive table lock for the entire transaction to prevent parameters from being added or modified during upsert. -MERGE INTO dbo.SearchParam - AS target -USING @searchParams AS source ON target.Uri = source.Uri -WHEN MATCHED THEN UPDATE -SET Status = source.Status, - LastUpdated = @lastUpdated, - IsPartiallySupported = source.IsPartiallySupported -WHEN NOT MATCHED BY TARGET THEN INSERT (Uri, Status, LastUpdated, IsPartiallySupported) VALUES (source.Uri, source.Status, @lastUpdated, source.IsPartiallySupported) -OUTPUT source.Uri, $ACTION INTO @summaryOfChanges; - -SELECT SearchParamId, - SearchParam.Uri, - SearchParam.LastUpdated -FROM dbo.SearchParam AS searchParam - INNER JOIN - @summaryOfChanges AS upsertedSearchParam - ON searchParam.Uri = upsertedSearchParam.Uri -WHERE upsertedSearchParam.Action = 'INSERT'; - -EXECUTE dbo.LogEvent @Process=@SP,@Mode=@Mode,@Status='End',@Start=@st -COMMIT TRANSACTION; -END TRY -BEGIN CATCH - IF @@trancount > 0 ROLLBACK TRANSACTION; - EXECUTE dbo.LogEvent @Process=@SP,@Mode=@Mode,@Status='Error',@Start=@st; - THROW -END CATCH -GO diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/Registry/SqlServerSearchParameterStatusDataStore.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/Registry/SqlServerSearchParameterStatusDataStore.cs index c49b847a22..8fb763aba0 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/Registry/SqlServerSearchParameterStatusDataStore.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/Registry/SqlServerSearchParameterStatusDataStore.cs @@ -31,6 +31,8 @@ namespace Microsoft.Health.Fhir.SqlServer.Features.Storage.Registry { internal class SqlServerSearchParameterStatusDataStore : ISearchParameterStatusDataStore { + private const string SearchParamLastUpdatedPrefix = "SearchParamLastUpdated="; + private readonly ISqlRetryService _sqlRetryService; private readonly SchemaInformation _schemaInformation; private readonly ISqlServerFhirModel _fhirModel; @@ -188,6 +190,29 @@ private async Task UpsertStatusesInternal(IReadOnlyCollection= 109) + { + cmd.Parameters.AddWithValue("@IsResourceChangeCaptureEnabled", false); + cmd.Parameters.Add(new SqlParameter("@TransactionId", SqlDbType.BigInt) { Value = DBNull.Value }); + + new ResourceListTableValuedParameterDefinition("@Resources").AddParameter(cmd.Parameters, Array.Empty()); + new ResourceWriteClaimListTableValuedParameterDefinition("@ResourceWriteClaims").AddParameter(cmd.Parameters, Array.Empty()); + new ReferenceSearchParamListTableValuedParameterDefinition("@ReferenceSearchParams").AddParameter(cmd.Parameters, Array.Empty()); + new TokenSearchParamListTableValuedParameterDefinition("@TokenSearchParams").AddParameter(cmd.Parameters, Array.Empty()); + new TokenTextListTableValuedParameterDefinition("@TokenTexts").AddParameter(cmd.Parameters, Array.Empty()); + new StringSearchParamListTableValuedParameterDefinition("@StringSearchParams").AddParameter(cmd.Parameters, Array.Empty()); + new UriSearchParamListTableValuedParameterDefinition("@UriSearchParams").AddParameter(cmd.Parameters, Array.Empty()); + new NumberSearchParamListTableValuedParameterDefinition("@NumberSearchParams").AddParameter(cmd.Parameters, Array.Empty()); + new QuantitySearchParamListTableValuedParameterDefinition("@QuantitySearchParams").AddParameter(cmd.Parameters, Array.Empty()); + new DateTimeSearchParamListTableValuedParameterDefinition("@DateTimeSearchParms").AddParameter(cmd.Parameters, Array.Empty()); + new ReferenceTokenCompositeSearchParamListTableValuedParameterDefinition("@ReferenceTokenCompositeSearchParams").AddParameter(cmd.Parameters, Array.Empty()); + new TokenTokenCompositeSearchParamListTableValuedParameterDefinition("@TokenTokenCompositeSearchParams").AddParameter(cmd.Parameters, Array.Empty()); + new TokenDateTimeCompositeSearchParamListTableValuedParameterDefinition("@TokenDateTimeCompositeSearchParams").AddParameter(cmd.Parameters, Array.Empty()); + new TokenQuantityCompositeSearchParamListTableValuedParameterDefinition("@TokenQuantityCompositeSearchParams").AddParameter(cmd.Parameters, Array.Empty()); + new TokenStringCompositeSearchParamListTableValuedParameterDefinition("@TokenStringCompositeSearchParams").AddParameter(cmd.Parameters, Array.Empty()); + new TokenNumberNumberCompositeSearchParamListTableValuedParameterDefinition("@TokenNumberNumberCompositeSearchParams").AddParameter(cmd.Parameters, Array.Empty()); + } + var results = await cmd.ExecuteReaderAsync( _sqlRetryService, (reader) => { return reader.ReadRow(VLatest.SearchParam.SearchParamId, VLatest.SearchParam.Uri, VLatest.SearchParam.LastUpdated); }, @@ -221,5 +246,85 @@ public void SyncStatuses(IReadOnlyCollection stat _fhirModel.TryAddSearchParamIdToUriMapping(status.Uri.OriginalString, status.Id); } } + + public async Task CheckCacheConsistencyAsync(string targetSearchParamLastUpdated, DateTime syncStartDate, DateTime activeHostsSince, CancellationToken cancellationToken) + { + EnsureArg.IsNotNullOrWhiteSpace(targetSearchParamLastUpdated, nameof(targetSearchParamLastUpdated)); + + if (_schemaInformation.Current < (int)SchemaVersion.V108) + { + // Pre-V108 schemas don't have the sproc; assume consistent + return new CacheConsistencyResult { IsConsistent = true, TotalActiveHosts = 1, ConvergedHosts = 1 }; + } + + using var cmd = new SqlCommand(); + cmd.CommandType = CommandType.StoredProcedure; + cmd.CommandText = "dbo.CheckSearchParamCacheConsistency"; + cmd.Parameters.AddWithValue("@TargetSearchParamLastUpdated", targetSearchParamLastUpdated); + cmd.Parameters.AddWithValue("@SyncStartDate", syncStartDate); + cmd.Parameters.AddWithValue("@ActiveHostsSince", activeHostsSince); + + var results = await cmd.ExecuteReaderAsync( + _sqlRetryService, + (reader) => + { + var hostName = reader.GetString(0); + var syncEventDate = reader.IsDBNull(1) ? (DateTime?)null : reader.GetDateTime(1); + var eventText = reader.IsDBNull(2) ? null : reader.GetString(2); + return (hostName, syncEventDate, eventText); + }, + _logger, + cancellationToken); + + if (results.Count == 0) + { + // If no results, assume consistent (could be a fresh database) + return new CacheConsistencyResult { IsConsistent = true, TotalActiveHosts = 0, ConvergedHosts = 0 }; + } + + var activeHosts = new HashSet(StringComparer.OrdinalIgnoreCase); + var latestSyncByHost = new Dictionary(StringComparer.OrdinalIgnoreCase); + + foreach (var (hostName, syncEventDate, eventText) in results) + { + activeHosts.Add(hostName); + + if (syncEventDate.HasValue && !string.IsNullOrEmpty(eventText)) + { + if (!latestSyncByHost.TryGetValue(hostName, out var existingSync) + || syncEventDate.Value > existingSync.SyncEventDate) + { + latestSyncByHost[hostName] = (syncEventDate.Value, eventText); + } + } + } + + int totalActiveHosts = activeHosts.Count; + int totalConvergedHosts = activeHosts.Count(hostName => + latestSyncByHost.TryGetValue(hostName, out var latestSync) + && TryGetLoggedSearchParamLastUpdated(latestSync.EventText, out string loggedSearchParamLastUpdated) + && StringComparer.Ordinal.Compare(loggedSearchParamLastUpdated, targetSearchParamLastUpdated) >= 0); + + return new CacheConsistencyResult + { + IsConsistent = totalActiveHosts > 0 && totalConvergedHosts >= totalActiveHosts, + TotalActiveHosts = totalActiveHosts, + ConvergedHosts = totalConvergedHosts, + }; + } + + private static bool TryGetLoggedSearchParamLastUpdated(string eventText, out string loggedSearchParamLastUpdated) + { + if (!string.IsNullOrEmpty(eventText) + && eventText.StartsWith(SearchParamLastUpdatedPrefix, StringComparison.Ordinal) + && eventText.Length > SearchParamLastUpdatedPrefix.Length) + { + loggedSearchParamLastUpdated = eventText.Substring(SearchParamLastUpdatedPrefix.Length); + return true; + } + + loggedSearchParamLastUpdated = null; + return false; + } } } diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlServerFhirDataStore.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlServerFhirDataStore.cs index d5cf1f0193..622c378060 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlServerFhirDataStore.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlServerFhirDataStore.cs @@ -30,6 +30,8 @@ using Microsoft.Health.Fhir.Core.Features.Persistence; using Microsoft.Health.Fhir.Core.Features.Persistence.Orchestration; using Microsoft.Health.Fhir.Core.Features.Search; +using Microsoft.Health.Fhir.Core.Features.Search.Parameters; +using Microsoft.Health.Fhir.Core.Features.Search.Registry; using Microsoft.Health.Fhir.Core.Models; using Microsoft.Health.Fhir.SqlServer.Features.Schema.Model; using Microsoft.Health.Fhir.SqlServer.Features.Storage.TvpRowGeneration; @@ -420,6 +422,11 @@ private async Task MergeInternalAsync(IReadOnlyList r.PendingSearchParameterStatuses?.Count > 0) + .SelectMany(r => r.PendingSearchParameterStatuses) + .ToList(); + if (mergeWrappersWithVersions.Count > 0) // Do not call DB with empty input { await using (new Timer(async _ => await _sqlStoreClient.MergeResourcesPutTransactionHeartbeatAsync(transactionId, MergeResourcesTransactionHeartbeatPeriod, cancellationToken), null, TimeSpan.FromSeconds(RandomNumberGenerator.GetInt32(100) / 100.0 * MergeResourcesTransactionHeartbeatPeriod.TotalSeconds), MergeResourcesTransactionHeartbeatPeriod)) @@ -430,7 +437,7 @@ private async Task MergeInternalAsync(IReadOnlyList _.Wrapper).ToList(), enlistInTransaction, timeoutRetries, cancellationToken); + await MergeResourcesWrapperAsync(transactionId, singleTransaction, mergeWrappersWithVersions.Select(_ => _.Wrapper).ToList(), enlistInTransaction, timeoutRetries, allSearchParameterStatuses, cancellationToken); break; } catch (Exception e) @@ -786,16 +793,28 @@ await MergeInternalAsync( } } - internal async Task MergeResourcesWrapperAsync(long transactionId, bool singleTransaction, IReadOnlyList mergeWrappers, bool enlistInTransaction, int timeoutRetries, CancellationToken cancellationToken) + internal async Task MergeResourcesWrapperAsync(long transactionId, bool singleTransaction, IReadOnlyList mergeWrappers, bool enlistInTransaction, int timeoutRetries, IReadOnlyList pendingStatuses, CancellationToken cancellationToken) { var sw = Stopwatch.StartNew(); using var cmd = new SqlCommand(); //// Do not use auto generated tvp generator as it does not allow to skip compartment tvp and paramters with default values cmd.CommandType = CommandType.StoredProcedure; - cmd.CommandText = "dbo.MergeResources"; + bool hasPendingStatuses = pendingStatuses?.Count > 0; + + if (hasPendingStatuses) + { + cmd.CommandText = "dbo.MergeSearchParams"; + new SearchParamListTableValuedParameterDefinition("@SearchParams").AddParameter(cmd.Parameters, new SearchParamListRowGenerator().GenerateRows(pendingStatuses.ToList())); + } + else + { + cmd.CommandText = "dbo.MergeResources"; + cmd.Parameters.AddWithValue("@SingleTransaction", singleTransaction); + } + cmd.Parameters.AddWithValue("@IsResourceChangeCaptureEnabled", _coreFeatures.SupportsResourceChangeCapture); cmd.Parameters.AddWithValue("@TransactionId", transactionId); - cmd.Parameters.AddWithValue("@SingleTransaction", singleTransaction); + new ResourceListTableValuedParameterDefinition("@Resources").AddParameter(cmd.Parameters, new ResourceListRowGenerator(_model, _compressedRawResourceConverter).GenerateRows(mergeWrappers)); new ResourceWriteClaimListTableValuedParameterDefinition("@ResourceWriteClaims").AddParameter(cmd.Parameters, new ResourceWriteClaimListRowGenerator(_model, _searchParameterTypeMap).GenerateRows(mergeWrappers)); new ReferenceSearchParamListTableValuedParameterDefinition("@ReferenceSearchParams").AddParameter(cmd.Parameters, new ReferenceSearchParamListRowGenerator(_model, _searchParameterTypeMap).GenerateRows(mergeWrappers)); @@ -830,6 +849,42 @@ internal async Task MergeResourcesWrapperAsync(long transactionId, bool singleTr _logger.LogInformation($"MergeResourcesWrapperAsync: transactionId={transactionId}, singleTransaction={singleTransaction}, resources={mergeWrappers.Count}, enlistInTran={enlistInTransaction}, commandTimeout={commandTimeout}, elapsed={sw.Elapsed.TotalMilliseconds} ms."); } + private bool TryGetPendingSearchParameterStatusUpdates(out List pendingStatuses) + { + pendingStatuses = null; + + var context = _requestContextAccessor?.RequestContext; + if (context?.Properties == null) + { + return false; + } + + if (!context.Properties.TryGetValue(SearchParameterRequestContextPropertyNames.PendingStatusUpdates, out object value) || + value is not List statuses || + statuses.Count == 0) + { + return false; + } + + lock (statuses) + { + if (statuses.Count == 0) + { + return false; + } + + pendingStatuses = statuses.ToList(); + } + + return true; + } + + private void ClearPendingSearchParameterStatusUpdates() + { + var context = _requestContextAccessor?.RequestContext; + context?.Properties?.Remove(SearchParameterRequestContextPropertyNames.PendingStatusUpdates); + } + public async Task UpsertAsync(ResourceWrapperOperation resource, CancellationToken cancellationToken) { bool isBundleParallelOperation = @@ -843,10 +898,29 @@ public async Task UpsertAsync(ResourceWrapperOperation resource, if (isBundleParallelOperation) { IBundleOrchestratorOperation bundleOperation = _bundleOrchestrator.GetOperation(resource.BundleResourceContext.BundleOperationId); + TryGetPendingSearchParameterStatusUpdates(out var pendingStatuses); + if (pendingStatuses?.Count > 0) + { + resource.PendingSearchParameterStatuses = pendingStatuses; + ClearPendingSearchParameterStatusUpdates(); + } + return await bundleOperation.AppendResourceAsync(resource, this, cancellationToken).ConfigureAwait(false); } else { + // For non-transaction operations, extract pending statuses now so they are merged with the resource. + // Transaction bundles skip this; statuses are flushed at the end of the bundle by BundleHandler. + if (!isBundleTransaction) + { + TryGetPendingSearchParameterStatusUpdates(out var pendingStatuses); + if (pendingStatuses?.Count > 0) + { + resource.PendingSearchParameterStatuses = pendingStatuses; + ClearPendingSearchParameterStatusUpdates(); + } + } + // For regular upserts and sequential bundle operations, enlistTransaction is set to true. MergeOptions mergeOptions = new MergeOptions(enlistTransaction: true, ensureAtomicOperations: isBundleTransaction); var mergeOutcome = await MergeAsync(new[] { resource }, mergeOptions, cancellationToken); diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/TransactionWatchdog.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/TransactionWatchdog.cs index 83f36b972f..0b0e17b1e1 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/TransactionWatchdog.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/TransactionWatchdog.cs @@ -83,7 +83,7 @@ protected override async Task RunWorkAsync(CancellationToken cancellationToken) _factory.Update(resource); } - await _store.MergeResourcesWrapperAsync(tranId, false, resources.Select(x => new MergeResourceWrapper(x, true, true)).ToList(), false, 0, cancellationToken); + await _store.MergeResourcesWrapperAsync(tranId, false, resources.Select(x => new MergeResourceWrapper(x, true, true)).ToList(), false, 0, null, cancellationToken); await _store.StoreClient.MergeResourcesCommitTransactionAsync(tranId, null, cancellationToken); _logger.LogWarning("TransactionWatchdog committed transaction={Transaction}, resources={Resources}", tranId, resources.Count); await _store.StoreClient.TryLogEvent("TransactionWatchdog", "Warn", $"committed transaction={tranId}, resources={resources.Count}", st, cancellationToken); diff --git a/src/Microsoft.Health.Fhir.SqlServer/Microsoft.Health.Fhir.SqlServer.csproj b/src/Microsoft.Health.Fhir.SqlServer/Microsoft.Health.Fhir.SqlServer.csproj index e970a27ce6..dda4835481 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Microsoft.Health.Fhir.SqlServer.csproj +++ b/src/Microsoft.Health.Fhir.SqlServer/Microsoft.Health.Fhir.SqlServer.csproj @@ -1,7 +1,7 @@  - 108 + 109 Features\Schema\Migrations\$(LatestSchemaVersion).sql LatestSchemaVersion-$(LatestSchemaVersion) diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs index 44735d6a4b..d871cef997 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs @@ -18,7 +18,11 @@ using Microsoft.Health.Fhir.Tests.Common; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; using Microsoft.Health.Test.Utilities; +using Microsoft.VisualStudio.TestPlatform.Utilities; +using Newtonsoft.Json; using Xunit; +using Xunit.Abstractions; +using static Hl7.Fhir.Model.Bundle; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Reindex @@ -31,11 +35,13 @@ public class ReindexTests : IClassFixture { private readonly HttpIntegrationTestFixture _fixture; private readonly bool _isSql; + private readonly ITestOutputHelper _output; - public ReindexTests(HttpIntegrationTestFixture fixture) + public ReindexTests(HttpIntegrationTestFixture fixture, ITestOutputHelper output) { _fixture = fixture; _isSql = _fixture.DataStore == DataStore.SqlServer; + _output = output; } [Fact] @@ -59,18 +65,20 @@ public async Task GivenReindexJobWithConcurrentUpdates_ThenReportedCountsAreLess { Parameter = [ - new Parameters.ParameterComponent { Name = "maximumNumberOfResourcesPerQuery", Value = new Integer(1) }, + new Parameters.ParameterComponent { Name = "maximumNumberOfResourcesPerQuery", Value = new Integer(2) }, new Parameters.ParameterComponent { Name = "maximumNumberOfResourcesPerWrite", Value = new Integer(1) }, ], }; + await Task.Delay(5000); + value = await _fixture.TestFhirClient.PostReindexJobAsync(parameters); Assert.Equal(HttpStatusCode.Created, value.response.Response.StatusCode); var tasks = new[] { WaitForJobCompletionAsync(value.jobUri, TimeSpan.FromSeconds(300)), - RandomPersonUpdate(testResources), + RandomPersonUpdate(testResources.Take(6).ToList()), }; await Task.WhenAll(tasks); @@ -517,7 +525,7 @@ public async Task GivenSearchParameterAddedAndReindexed_WhenSearchParameterIsDel if (specimenResources.Count > 0) { specimenId = specimenResources[0].resourceId; - System.Diagnostics.Debug.WriteLine($"Created specimen with ID {specimenId} and type {specimenType}"); + _output.WriteLine($"Created specimen with ID {specimenId} and type {specimenType}"); } // Step 2: Create a custom search parameter for Specimen.type @@ -527,7 +535,7 @@ public async Task GivenSearchParameterAddedAndReindexed_WhenSearchParameterIsDel "Specimen.type", SearchParamType.Token); Assert.NotNull(searchParam); - System.Diagnostics.Debug.WriteLine($"Created search parameter with code {searchParam.Code} and ID {searchParam.Id}"); + _output.WriteLine($"Created search parameter with code {searchParam.Code} and ID {searchParam.Id}"); // Step 3: Reindex to index the newly created search parameter var reindexParams = new Parameters @@ -550,13 +558,13 @@ public async Task GivenSearchParameterAddedAndReindexed_WhenSearchParameterIsDel reindexRequest1 = await _fixture.TestFhirClient.PostReindexJobAsync(reindexParams); Assert.Equal(HttpStatusCode.Created, reindexRequest1.response.Response.StatusCode); Assert.NotNull(reindexRequest1.jobUri); - System.Diagnostics.Debug.WriteLine("Started first reindex job to index the new search parameter"); + _output.WriteLine("Started first reindex job to index the new search parameter"); var jobStatus1 = await WaitForJobCompletionAsync(reindexRequest1.jobUri, TimeSpan.FromSeconds(240)); Assert.True( jobStatus1 == OperationStatus.Completed, $"First reindex job should complete successfully, but got {jobStatus1}"); - System.Diagnostics.Debug.WriteLine("First reindex job completed successfully"); + _output.WriteLine("First reindex job completed successfully"); await CheckReportedCounts(reindexRequest1.jobUri, testResources.Count, false); @@ -570,24 +578,24 @@ await VerifySearchParameterIsWorkingAsync( // Step 5: Delete the search parameter await _fixture.TestFhirClient.DeleteAsync($"SearchParameter/{searchParam.Id}"); - System.Diagnostics.Debug.WriteLine($"Deleted search parameter {searchParam.Code} (ID: {searchParam.Id})"); + _output.WriteLine($"Deleted search parameter {searchParam.Code} (ID: {searchParam.Id})"); // Step 6: Reindex to remove the search parameter from the index reindexRequest2 = await _fixture.TestFhirClient.PostReindexJobAsync(reindexParams); Assert.Equal(HttpStatusCode.Created, reindexRequest2.response.Response.StatusCode); Assert.NotNull(reindexRequest2.jobUri); - System.Diagnostics.Debug.WriteLine("Started second reindex job to remove the deleted search parameter"); + _output.WriteLine("Started second reindex job to remove the deleted search parameter"); var jobStatus2 = await WaitForJobCompletionAsync(reindexRequest2.jobUri, TimeSpan.FromSeconds(240)); Assert.True( jobStatus2 == OperationStatus.Completed, $"Second reindex job should complete successfully, but got {jobStatus2}"); - System.Diagnostics.Debug.WriteLine("Second reindex job completed successfully"); + _output.WriteLine("Second reindex job completed successfully"); // Step 7: Verify the search parameter is no longer supported var postDeleteSearchResponse = await _fixture.TestFhirClient.SearchAsync(searchQuery); Assert.NotNull(postDeleteSearchResponse); - System.Diagnostics.Debug.WriteLine($"Executed search query after deletion: {searchQuery}"); + _output.WriteLine($"Executed search query after deletion: {searchQuery}"); // Verify that a "NotSupported" error is now present var hasNotSupportedErrorAfterDelete = HasNotSupportedError(postDeleteSearchResponse.Resource); @@ -595,7 +603,7 @@ await VerifySearchParameterIsWorkingAsync( Assert.True( hasNotSupportedErrorAfterDelete, $"Search parameter {searchParam.Code} should NOT be supported after deletion and reindex. Got 'NotSupported' error in response."); - System.Diagnostics.Debug.WriteLine($"Search parameter {searchParam.Code} correctly returns 'NotSupported' error after deletion"); + _output.WriteLine($"Search parameter {searchParam.Code} correctly returns 'NotSupported' error after deletion"); } finally { @@ -631,16 +639,16 @@ private Person CreatePersonResource(string id, string name) if (response?.Resource != null && !string.IsNullOrEmpty(response.Resource.Id)) { - System.Diagnostics.Debug.WriteLine($"Successfully created {typeof(T).Name}/{response.Resource.Id}"); + _output.WriteLine($"Successfully created {typeof(T).Name}/{response.Resource.Id}"); return (true, response.Resource, response.Resource.Id); } - System.Diagnostics.Debug.WriteLine($"Failed to create resource {id}: Response was null or had no ID"); + _output.WriteLine($"Failed to create resource {id}: Response was null or had no ID"); return (false, resource, id); } catch (Exception ex) { - System.Diagnostics.Debug.WriteLine($"Failed to create resource {id}: {ex.Message}"); + _output.WriteLine($"Failed to create resource {id}: {ex.Message}"); return (false, null, id); } } @@ -661,7 +669,7 @@ private async Task CreateAndPostResourceAsync(string id, string name, Func } catch (Exception ex) { - System.Diagnostics.Debug.WriteLine($"Failed to create resource {id}: {ex.Message}"); + _output.WriteLine($"Failed to create resource {id}: {ex.Message}"); return resource; // Return the original resource even on failure so ID can be tracked } } @@ -710,7 +718,7 @@ private async Task CreateCustomSearchParameterAsync(string code catch (Exception ex) { // Log the exception for debugging - System.Diagnostics.Debug.WriteLine($"Failed to create search parameter: {ex.Message}"); + _output.WriteLine($"Failed to create search parameter: {ex.Message}"); throw; } } @@ -751,7 +759,7 @@ private async Task CleanupTestDataAsync(List<(string resourceType, string resour // Delete test data resources (Person, Observation, etc.) in parallel batches if (testResources != null && testResources.Count > 0) { - System.Diagnostics.Debug.WriteLine($"Starting cleanup of {testResources.Count} test resources..."); + _output.WriteLine($"Starting cleanup of {testResources.Count} test resources..."); const int batchSize = 500; // Process 500 resources at a time in parallel int totalDeleted = 0; @@ -779,18 +787,18 @@ private async Task CleanupTestDataAsync(List<(string resourceType, string resour await Task.WhenAll(batchTasks); totalDeleted += currentBatchSize; - System.Diagnostics.Debug.WriteLine($"Deleted batch {(batchStart / batchSize) + 1}: {currentBatchSize} resources (total: {totalDeleted}/{testResources.Count})"); + _output.WriteLine($"Deleted batch {(batchStart / batchSize) + 1}: {currentBatchSize} resources (total: {totalDeleted}/{testResources.Count})"); } catch (Exception ex) { totalFailed += currentBatchSize; - System.Diagnostics.Debug.WriteLine($"Failed to delete batch at offset {batchStart}: {ex.Message}"); + _output.WriteLine($"Failed to delete batch at offset {batchStart}: {ex.Message}"); // Continue with next batch instead of failing completely } } - System.Diagnostics.Debug.WriteLine($"Cleanup completed: {totalDeleted} deleted successfully, {totalFailed} failed"); + _output.WriteLine($"Cleanup completed: {totalDeleted} deleted successfully, {totalFailed} failed"); } await Task.Delay(TimeSpan.FromSeconds(3)); @@ -807,11 +815,11 @@ private async Task DeleteResourceAsync(string resourceType, string resourceId) try { await _fixture.TestFhirClient.DeleteAsync($"{resourceType}/{resourceId}"); - System.Diagnostics.Debug.WriteLine($"Deleted {resourceType}/{resourceId}"); + _output.WriteLine($"Deleted {resourceType}/{resourceId}"); } catch (Exception ex) { - System.Diagnostics.Debug.WriteLine($"Failed to delete {resourceType}/{resourceId}: {ex.Message}"); + _output.WriteLine($"Failed to delete {resourceType}/{resourceId}: {ex.Message}"); // Don't throw - allow other deletions to continue } @@ -840,7 +848,7 @@ private async Task CleanupSearchParametersAsync(params SearchParameter[] searchP catch (Exception ex) { // Log but continue cleanup even if delete fails - System.Diagnostics.Debug.WriteLine($"Failed to delete SearchParameter/{param?.Id}: {ex.Message}"); + _output.WriteLine($"Failed to delete SearchParameter/{param?.Id}: {ex.Message}"); } } @@ -891,6 +899,18 @@ private async Task WaitForJobCompletionAsync(Uri jobUri, TimeSp status == OperationStatus.Failed || status == OperationStatus.Canceled) { + if (status == OperationStatus.Failed || status == OperationStatus.Canceled) + { + var failureParam = jobResponse.Resource.Parameter.FirstOrDefault(p => p.Name == "failureDetails"); + var failureReason = failureParam?.Value is FhirString fs ? fs.Value : failureParam?.Value?.ToString(); + _output.WriteLine($"Reindex job {jobUri} reached terminal status: {status}. Failure reason: {failureReason ?? "N/A"}"); + + foreach (var param in jobResponse.Resource.Parameter) + { + _output.WriteLine($" Parameter: {param.Name} = {param.Value}"); + } + } + return status; } } @@ -961,14 +981,14 @@ private async Task VerifySearchParameterIsWorkingAsync( int pageCount = 0; int totalEntriesRetrieved = 0; - System.Diagnostics.Debug.WriteLine( + _output.WriteLine( $"Search parameter {searchParameterCode} - attempt {attempt} of {maxRetries}, starting search with page size {pageSize}."); // Follow pagination until we get all results while (!string.IsNullOrEmpty(nextLink)) { pageCount++; - System.Diagnostics.Debug.WriteLine($"Search parameter {searchParameterCode} - fetching page {pageCount}"); + _output.WriteLine($"Search parameter {searchParameterCode} - fetching page {pageCount}"); var searchResponse = await _fixture.TestFhirClient.SearchAsync(nextLink); Assert.NotNull(searchResponse); @@ -989,11 +1009,11 @@ private async Task VerifySearchParameterIsWorkingAsync( // Check if there's a next link for pagination nextLink = searchResponse.Resource?.NextLink?.OriginalString; - System.Diagnostics.Debug.WriteLine( + _output.WriteLine( $"Search parameter {searchParameterCode} - page {pageCount} returned {searchResponse.Resource?.Entry?.Count ?? 0} entries (total so far: {totalEntriesRetrieved}). Next link: {(string.IsNullOrEmpty(nextLink) ? "none" : "present")}"); } - System.Diagnostics.Debug.WriteLine( + _output.WriteLine( $"Search parameter {searchParameterCode} is working - search executed successfully without 'NotSupported' error on attempt {attempt}. Total pages fetched: {pageCount}, Total entries retrieved: {totalEntriesRetrieved}"); // Validate record expectations if specified @@ -1010,7 +1030,7 @@ private async Task VerifySearchParameterIsWorkingAsync( resourcesFound.Count > 0, $"Expected to find {expectedResourceType} records for search parameter {searchParameterCode}, but none were returned."); - System.Diagnostics.Debug.WriteLine( + _output.WriteLine( $"Search parameter {searchParameterCode} correctly returned {resourcesFound.Count} {expectedResourceType} record(s) across {pageCount} page(s)"); } else @@ -1019,13 +1039,13 @@ private async Task VerifySearchParameterIsWorkingAsync( Assert.True( resourcesFound.Count == 0, $"Expected no {expectedResourceType} records for search parameter {searchParameterCode}, but found {resourcesFound.Count}."); - System.Diagnostics.Debug.WriteLine( + _output.WriteLine( $"Search parameter {searchParameterCode} correctly returned no {expectedResourceType} records"); } } // Log the successful search query for reference - System.Diagnostics.Debug.WriteLine($"Search query: {searchQuery} executed successfully on attempt {attempt} across {pageCount} page(s)."); + _output.WriteLine($"Search query: {searchQuery} executed successfully on attempt {attempt} across {pageCount} page(s)."); // Success - return without retrying return; @@ -1036,7 +1056,7 @@ private async Task VerifySearchParameterIsWorkingAsync( if (attempt < maxRetries) { - System.Diagnostics.Debug.WriteLine( + _output.WriteLine( $"Search parameter {searchParameterCode} verification failed on attempt {attempt} of {maxRetries}. " + $"Retrying in {retryDelayMs}ms. Error: {ex.Message}"); @@ -1070,7 +1090,7 @@ private async Task VerifySearchParameterIsWorkingAsync( { var createdResources = new List<(string resourceType, string resourceId)>(); - System.Diagnostics.Debug.WriteLine($"Creating {desiredCount} new {resourceType} resources..."); + _output.WriteLine($"Creating {desiredCount} new {resourceType} resources..."); // Create resources in batches using parallel individual creates for better performance const int batchSize = 500; // Process 500 resources at a time in parallel @@ -1098,7 +1118,7 @@ private async Task VerifySearchParameterIsWorkingAsync( { // Exponential backoff for retries var delayMs = 1000 * (int)Math.Pow(2, retryAttempt - 1); // 1s, 2s, 4s - System.Diagnostics.Debug.WriteLine($"Retrying {resourcesToCreateInBatch.Count} failed resources after {delayMs}ms delay (attempt {retryAttempt + 1}/{maxCreateRetries})"); + _output.WriteLine($"Retrying {resourcesToCreateInBatch.Count} failed resources after {delayMs}ms delay (attempt {retryAttempt + 1}/{maxCreateRetries})"); await Task.Delay(delayMs); } @@ -1139,14 +1159,14 @@ private async Task VerifySearchParameterIsWorkingAsync( resourcesToCreateInBatch = nextRetryBatch; - System.Diagnostics.Debug.WriteLine( + _output.WriteLine( $"Batch {(batchStart / batchSize) + 1} attempt {retryAttempt + 1}: " + $"{createdResources.Count} total created, {resourcesToCreateInBatch.Count} pending retry, " + $"{failedIds.Count} permanently failed"); } catch (Exception ex) { - System.Diagnostics.Debug.WriteLine($"Failed to create batch at offset {batchStart}: {ex.Message}"); + _output.WriteLine($"Failed to create batch at offset {batchStart}: {ex.Message}"); if (retryAttempt == maxCreateRetries - 1) { // Add all remaining to failed list @@ -1165,8 +1185,8 @@ private async Task VerifySearchParameterIsWorkingAsync( // Report on any failures if (failedIds.Any()) { - System.Diagnostics.Debug.WriteLine($"WARNING: {failedIds.Count} resources failed to create after {maxCreateRetries} retries"); - System.Diagnostics.Debug.WriteLine($"Failed IDs (first 10): {string.Join(", ", failedIds.Take(10))}"); + _output.WriteLine($"WARNING: {failedIds.Count} resources failed to create after {maxCreateRetries} retries"); + _output.WriteLine($"Failed IDs (first 10): {string.Join(", ", failedIds.Take(10))}"); } // Calculate acceptable threshold (allow 5% failure rate for transient issues) @@ -1178,18 +1198,18 @@ private async Task VerifySearchParameterIsWorkingAsync( var errorMsg = $"CRITICAL: Failed to create sufficient {resourceType} resources. " + $"Desired: {desiredCount}, Acceptable minimum: {acceptableMinimum}, " + $"Successfully created: {createdResources.Count}, Failed: {failedIds.Count}"; - System.Diagnostics.Debug.WriteLine(errorMsg); + _output.WriteLine(errorMsg); Assert.Fail(errorMsg); } else if (createdResources.Count < desiredCount) { // Log warning but don't fail - System.Diagnostics.Debug.WriteLine( + _output.WriteLine( $"WARNING: Created {createdResources.Count}/{desiredCount} {resourceType} resources " + $"(within acceptable threshold of {acceptableMinimum})"); } - System.Diagnostics.Debug.WriteLine($"Successfully created {createdResources.Count} new {resourceType} resources."); + _output.WriteLine($"Successfully created {createdResources.Count} new {resourceType} resources."); // Return the ACTUAL count of resources we created and have IDs for return createdResources; @@ -1203,10 +1223,11 @@ private async Task SearchCreatedResources(string resourceType, int expectedCount private async Task RandomPersonUpdate(IList<(string resourceType, string resourceId)> resources) { - foreach (var resource in resources.OrderBy(_ => RandomNumberGenerator.GetInt32((int)1e6))) - { - await _fixture.TestFhirClient.UpdateAsync(CreatePersonResource(resource.resourceId, Guid.NewGuid().ToString())); - } + var tasks = resources + .OrderBy(_ => RandomNumberGenerator.GetInt32((int)1e6)) + .Select(resource => _fixture.TestFhirClient.UpdateAsync(CreatePersonResource(resource.resourceId, Guid.NewGuid().ToString()))); + + await Task.WhenAll(tasks); } private async Task CheckReportedCounts(Uri jobUri, long expected, bool lessThan) @@ -1220,6 +1241,10 @@ private async Task CheckReportedCounts(Uri jobUri, long expected, bool lessThan) var content = await response.Content.ReadAsStringAsync(); var parameters = new Hl7.Fhir.Serialization.FhirJsonParser().Parse(content); var total = (long)((FhirDecimal)parameters.Parameter.FirstOrDefault(p => p.Name == "totalResourcesToReindex").Value).Value; + var successes = (long)((FhirDecimal)parameters.Parameter.FirstOrDefault(p => p.Name == "resourcesSuccessfullyReindexed").Value).Value; + + _output.WriteLine($"CheckReportedCounts: totalResourcesToReindex={total}, resourcesSuccessfullyReindexed={successes}, expected={expected}, lessThan={lessThan}"); + if (lessThan) { Assert.True(total < expected, $"total={total} < expected={expected}"); @@ -1229,7 +1254,6 @@ private async Task CheckReportedCounts(Uri jobUri, long expected, bool lessThan) Assert.True(total >= expected, $"total={total} >= expected={expected}"); // some of resources might come for not completed retries } - var successes = (long)((FhirDecimal)parameters.Parameter.FirstOrDefault(p => p.Name == "resourcesSuccessfullyReindexed").Value).Value; Assert.True(total == successes, $"total={total} == successes={successes}"); } @@ -1243,14 +1267,14 @@ private async Task CancelAnyRunningReindexJobsAsync(CancellationToken cancellati { try { - System.Diagnostics.Debug.WriteLine("Checking for any running reindex jobs via GET $reindex..."); + _output.WriteLine("Checking for any running reindex jobs via GET $reindex..."); // Use GET to $reindex to check for active jobs var response = await _fixture.TestFhirClient.HttpClient.GetAsync("$reindex", cancellationToken); if (!response.IsSuccessStatusCode) { - System.Diagnostics.Debug.WriteLine($"GET $reindex returned non-success status: {response.StatusCode}. No active jobs to cancel."); + _output.WriteLine($"GET $reindex returned non-success status: {response.StatusCode}. No active jobs to cancel."); return; } @@ -1267,13 +1291,13 @@ private async Task CancelAnyRunningReindexJobsAsync(CancellationToken cancellati } catch (Exception parseEx) { - System.Diagnostics.Debug.WriteLine($"Failed to parse $reindex response as Parameters: {parseEx.Message}"); + _output.WriteLine($"Failed to parse $reindex response as Parameters: {parseEx.Message}"); return; } if (parameters?.Parameter == null || !parameters.Parameter.Any()) { - System.Diagnostics.Debug.WriteLine("No parameters found in $reindex response. No active jobs to cancel."); + _output.WriteLine("No parameters found in $reindex response. No active jobs to cancel."); return; } @@ -1283,7 +1307,7 @@ private async Task CancelAnyRunningReindexJobsAsync(CancellationToken cancellati if (idParam?.Value == null) { - System.Diagnostics.Debug.WriteLine("No job ID found in $reindex response. No active jobs to cancel."); + _output.WriteLine("No job ID found in $reindex response. No active jobs to cancel."); return; } @@ -1304,12 +1328,12 @@ private async Task CancelAnyRunningReindexJobsAsync(CancellationToken cancellati if (string.IsNullOrEmpty(jobId)) { - System.Diagnostics.Debug.WriteLine("Job ID is empty. No active jobs to cancel."); + _output.WriteLine("Job ID is empty. No active jobs to cancel."); return; } // Job is active (Running or Queued), cancel it - System.Diagnostics.Debug.WriteLine($"Job {jobId} is active. Attempting to cancel..."); + _output.WriteLine($"Job {jobId} is active. Attempting to cancel..."); // Use the correct URI format: /_operations/reindex/{jobId} var jobUri = new Uri($"{_fixture.TestFhirClient.HttpClient.BaseAddress}_operations/reindex/{jobId}"); @@ -1322,25 +1346,25 @@ private async Task CancelAnyRunningReindexJobsAsync(CancellationToken cancellati }; var cancelResponse = await _fixture.TestFhirClient.HttpClient.SendAsync(deleteRequest, cancellationToken); - System.Diagnostics.Debug.WriteLine($"Cancel request for job {jobId} completed with status: {cancelResponse.StatusCode}"); + _output.WriteLine($"Cancel request for job {jobId} completed with status: {cancelResponse.StatusCode}"); // Wait for the job to reach a terminal status if (cancelResponse.IsSuccessStatusCode) { - System.Diagnostics.Debug.WriteLine($"Waiting for job {jobId} to reach terminal state..."); + _output.WriteLine($"Waiting for job {jobId} to reach terminal state..."); var finalStatus = await WaitForJobCompletionAsync(jobUri, TimeSpan.FromSeconds(120)); - System.Diagnostics.Debug.WriteLine($"Job {jobId} reached final status: {finalStatus}"); + _output.WriteLine($"Job {jobId} reached final status: {finalStatus}"); // Add a small delay to ensure system is ready await Task.Delay(2000, cancellationToken); } - System.Diagnostics.Debug.WriteLine("Completed checking and canceling running reindex jobs"); + _output.WriteLine("Completed checking and canceling running reindex jobs"); } catch (Exception ex) { // Log but don't fail - this is a cleanup/safety check - System.Diagnostics.Debug.WriteLine($"Error in CancelAnyRunningReindexJobsAsync: {ex.Message}"); + _output.WriteLine($"Error in CancelAnyRunningReindexJobsAsync: {ex.Message}"); } } } diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/Reindex/ReindexJobTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/Reindex/ReindexJobTests.cs index 0891d47794..fabbe5b77f 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/Reindex/ReindexJobTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/Reindex/ReindexJobTests.cs @@ -37,6 +37,7 @@ using Microsoft.Health.Fhir.Core.Features.Search.SearchValues; using Microsoft.Health.Fhir.Core.Features.Security.Authorization; using Microsoft.Health.Fhir.Core.Messages.Reindex; +using Microsoft.Health.Fhir.Core.Messages.Search; using Microsoft.Health.Fhir.Core.Models; using Microsoft.Health.Fhir.Core.Registration; using Microsoft.Health.Fhir.Core.UnitTests.Extensions; @@ -94,6 +95,7 @@ public class ReindexJobTests : IClassFixture, IAsyncLif private readonly IDataStoreSearchParameterValidator _dataStoreSearchParameterValidator = Substitute.For(); private readonly IOptions _optionsReindexConfig = Substitute.For>(); private readonly IOptions _coreFeatureConfig = Substitute.For>(); + private SearchParameterCacheRefreshBackgroundService _cacheRefreshBackgroundService; public ReindexJobTests(FhirStorageTestsFixture fixture, ITestOutputHelper output) { @@ -144,9 +146,25 @@ public async Task InitializeAsync() ModelInfoProvider.Instance, _searchParameterSupportResolver, _dataStoreSearchParameterValidator, + () => _fhirOperationDataStore.CreateMockScope(), () => _searchService, NullLogger.Instance); + // Start background service so it triggers GetAndApplySearchParameterUpdates which signals the TCS. + _coreFeatureConfig.Value.Returns(new CoreFeatureConfiguration + { + SearchParameterCacheRefreshIntervalSeconds = 1, + }); + _cacheRefreshBackgroundService = new SearchParameterCacheRefreshBackgroundService( + _searchParameterStatusManager, + (SearchParameterOperations)_searchParameterOperations, + _coreFeatureConfig, + NullLogger.Instance); + + // Start the primary background service and trigger initialization so it begins refreshing immediately. + await _cacheRefreshBackgroundService.StartAsync(CancellationToken.None); + await _cacheRefreshBackgroundService.Handle(new SearchParametersInitializedNotification(), CancellationToken.None); + _createReindexRequestHandler = new CreateReindexRequestHandler( _fhirOperationDataStore, DisabledFhirAuthorizationService.Instance, @@ -164,7 +182,7 @@ public async Task InitializeAsync() await _fhirStorageTestHelper.DeleteAllReindexJobRecordsAsync(CancellationToken.None); - // Initialize second FHIR service + // Initialize second FHIR service. await InitializeSecondFHIRService(); await InitializeJobHosting(); @@ -172,6 +190,11 @@ public async Task InitializeAsync() public async Task DisposeAsync() { + if (_cacheRefreshBackgroundService != null) + { + await _cacheRefreshBackgroundService.StopAsync(CancellationToken.None); + } + // Clean up resources before finishing test class await DeleteTestResources(); @@ -223,11 +246,6 @@ private async Task InitializeJobHosting() IJob job = null; - _coreFeatureConfig.Value.Returns(new CoreFeatureConfiguration - { - SearchParameterCacheRefreshIntervalSeconds = 1, // Use a short interval for tests - }); - if (typeId == (int)JobType.ReindexOrchestrator) { // Create a mock OperationsConfiguration for the test @@ -783,8 +801,6 @@ public async Task GivenSecondFHIRServiceSynced_WhenReindexJobCompleted_ThenSecon var searchParamWrapper = CreateSearchParamResourceWrapper(searchParam); - await _scopedDataStore.Value.UpsertAsync(new ResourceWrapperOperation(searchParamWrapper, true, true, null, false, false, bundleResourceContext: null), CancellationToken.None); - // Create the query /Patient?foo=searchIndicesPatient1 var queryParams = new List> { new(searchParamCode, sampleName1) }; SearchResult searchResults = await _searchService.Value.SearchAsync("Patient", queryParams, CancellationToken.None); @@ -808,12 +824,24 @@ public async Task GivenSecondFHIRServiceSynced_WhenReindexJobCompleted_ThenSecon { await PerformReindexingOperation(response, OperationStatus.Completed, cancellationTokenSource); - var queryParams2 = new List>(); + ResourceSearchParameterStatus syncedStatus = null; + bool hasPrimaryDefinition = false; + for (int attempt = 0; attempt < 50; attempt++) + { + var statuses = await _searchParameterStatusManager.GetAllSearchParameterStatus(CancellationToken.None); + syncedStatus = statuses.FirstOrDefault(s => string.Equals(s.Uri?.OriginalString, searchParam.Url, StringComparison.Ordinal)); + hasPrimaryDefinition = _searchParameterDefinitionManager.TryGetSearchParameter(searchParam.Url, out _); - // make sure we can search for, and find, the newly created search parameter - queryParams2.Add(new Tuple("url", searchParam.Url)); - var result = await _searchService.Value.SearchAsync(KnownResourceTypes.SearchParameter, queryParams2, CancellationToken.None); - Assert.NotEmpty(result.Results); + if (syncedStatus != null && hasPrimaryDefinition) + { + break; + } + + await Task.Delay(250, CancellationToken.None); + } + + Assert.NotNull(syncedStatus); + Assert.True(hasPrimaryDefinition); // Rerun the same search as above searchResults = await _searchService.Value.SearchAsync("Patient", queryParams, CancellationToken.None); @@ -832,19 +860,23 @@ public async Task GivenSecondFHIRServiceSynced_WhenReindexJobCompleted_ThenSecon bool tryGetSearchParamResult = _searchParameterDefinitionManager2.TryGetSearchParameter(searchParam.Url, out var searchParamInfo); Assert.False(tryGetSearchParamResult); - await _searchParameterOperations2.GetAndApplySearchParameterUpdates(CancellationToken.None); + for (int attempt = 0; attempt < 30 && !tryGetSearchParamResult; attempt++) + { + await _searchParameterOperations2.GetAndApplySearchParameterUpdates(CancellationToken.None, true); + tryGetSearchParamResult = _searchParameterDefinitionManager2.TryGetSearchParameter(searchParam.Url, out searchParamInfo); + if (!tryGetSearchParamResult) + { + await Task.Delay(200, CancellationToken.None); + } + } // now we should have sync'd the search parameter - tryGetSearchParamResult = _searchParameterDefinitionManager2.TryGetSearchParameter(searchParam.Url, out searchParamInfo); Assert.True(tryGetSearchParamResult); } finally { cancellationTokenSource.Cancel(); - _searchParameterDefinitionManager.DeleteSearchParameter(searchParam.ToTypedElement()); - await _searchParameterStatusManager2.DeleteSearchParameterStatusAsync(searchParam.Url, CancellationToken.None); - _searchParameterDefinitionManager2.DeleteSearchParameter(searchParam.ToTypedElement()); await _testHelper.DeleteSearchParameterStatusAsync(searchParam.Url, CancellationToken.None); await _fixture.DataStore.HardDeleteAsync(sample1.Wrapper.ToResourceKey(), false, false, CancellationToken.None); @@ -864,41 +896,32 @@ public async Task GivenSecondFHIRServiceSynced_WhenSyncParametersOccursDuringDel var searchParamWrapper = CreateSearchParamResourceWrapper(searchParam); - await _scopedDataStore.Value.UpsertAsync(new ResourceWrapperOperation(searchParamWrapper, true, true, null, false, false, bundleResourceContext: null), CancellationToken.None); - using var cancellationTokenSource = new CancellationTokenSource(); try { - var queryParams2 = new List>(); - - // make sure we can search for, and find, the newly created search parameter - queryParams2.Add(new Tuple("url", searchParam.Url)); - var result = await _searchService.Value.SearchAsync(KnownResourceTypes.SearchParameter, queryParams2, CancellationToken.None); - Assert.NotEmpty(result.Results); - // first service should have knowledge of new Searchparameter bool tryGetSearchParamResult1 = _searchParameterDefinitionManager.TryGetSearchParameter(searchParam.Url, out var searchParamInfo); Assert.True(tryGetSearchParamResult1); - // second service should not have knowledge of new Searchparameter + // Sync service2 to match the scenario and ensure deterministic pre-delete state. + await _searchParameterOperations2.GetAndApplySearchParameterUpdates(CancellationToken.None, true); bool tryGetSearchParamResult2 = _searchParameterDefinitionManager2.TryGetSearchParameter(searchParam.Url, out searchParamInfo); - Assert.False(tryGetSearchParamResult2); + Assert.True(tryGetSearchParamResult2); ResourceWrapper deletedWrapper = CreateSearchParamResourceWrapper(searchParam, deleted: true); - // As per DeleteSearchParameterBehavior.Handle, first step of the delete process would be to delete from the in-memory datastore - // then delete the search parameter resource from data base - await _searchParameterOperations2.DeleteSearchParameterAsync(deletedWrapper.RawResource, CancellationToken.None); - + // Simulate the delete: upsert a deleted wrapper so GetAndApplySearchParameterUpdates can pick up the deletion. UpsertOutcome deleteResult = await _fixture.DataStore.UpsertAsync(new ResourceWrapperOperation(deletedWrapper, true, true, null, false, false, bundleResourceContext: null), CancellationToken.None); - // After trying to sync the new "supported" status, but finding the resource missing, we should have it listed as PendingDelete - var tryGetSearchParamResult = _searchParameterDefinitionManager2.TryGetSearchParameter(searchParam.Url, out searchParamInfo); - Assert.True(tryGetSearchParamResult); + await _searchParameterOperations2.GetAndApplySearchParameterUpdates(CancellationToken.None, true); + + // If the SearchParameter resource is missing at sync time, service2 should handle refresh without throwing. + _searchParameterDefinitionManager2.TryGetSearchParameter(searchParam.Url, out searchParamInfo); var statuses = await _searchParameterStatusManager2.GetAllSearchParameterStatus(CancellationToken.None); - Assert.True(statuses.Where(sp => sp.Uri.OriginalString.Equals(searchParamInfo.Url.OriginalString)).First().Status == SearchParameterStatus.PendingDelete); + var status = statuses.Single(sp => sp.Uri.OriginalString.Equals(searchParam.Url, StringComparison.Ordinal)).Status; + Assert.Contains(status, new[] { SearchParameterStatus.Supported, SearchParameterStatus.PendingDelete }); } finally { @@ -946,7 +969,7 @@ public async Task GivenNewSearchParamWithResourceBaseType_WhenReindexJobComplete // CRITICAL: Force the search parameter definition manager to refresh/sync // This is the missing piece - the search service needs to know about status changes - await _searchParameterOperations.GetAndApplySearchParameterUpdates(CancellationToken.None); + await _searchParameterOperations.GetAndApplySearchParameterUpdates(CancellationToken.None, true); // Now test the actual search functionality // Rerun the same search as above @@ -1352,7 +1375,8 @@ private async Task PerformReindexingOperation( if (operationStatus == OperationStatus.Completed && (reindexJobWrapper.JobRecord.Status == OperationStatus.Failed || reindexJobWrapper.JobRecord.Status == OperationStatus.Canceled)) { - Assert.Fail($"Fail-fast. Current job status '{reindexJobWrapper.JobRecord.Status}'. Expected job status '{operationStatus}'. Number of attempts: {MaxNumberOfAttempts}. Time elapsed: {stopwatch.Elapsed}."); + var failureReason = reindexJobWrapper.JobRecord.FailureDetails?.FailureReason; + Assert.Fail($"Fail-fast. Current job status '{reindexJobWrapper.JobRecord.Status}'. Expected job status '{operationStatus}'. Number of attempts: {MaxNumberOfAttempts}. Time elapsed: {stopwatch.Elapsed}. FailureDetails: '{failureReason}'."); } } @@ -1391,10 +1415,18 @@ private void MockSearchIndexExtraction(IEnumerable<(string id, ISearchValue sear { SearchParameterInfo searchParamInfo = searchParam.ToInfo(); + // Default extraction result for resources that are not explicitly configured below. + // This is important for Resource-base reindex tests where many resource types are processed. + _searchIndexer + .Extract(Arg.Any()) + .Returns(_ => new List()); + foreach ((string id, ISearchValue searchValue) in searchValues) { - var searchIndexValues = new List(); - searchIndexValues.Add(new SearchIndexEntry(searchParamInfo, searchValue)); + var searchIndexValues = new List + { + new SearchIndexEntry(searchParamInfo, searchValue), + }; // Add null check for ResourceElement _searchIndexer.Extract(Arg.Is(r => r != null && r.Id != null && r.Id.Equals(id))).Returns(searchIndexValues); @@ -1419,11 +1451,42 @@ private async Task CreateSearchParam(string searchParamName, Se searchParam.Base = new List() { Enum.Parse(baseType) }; #endif - await _searchParameterOperations.AddSearchParameterAsync(searchParam.ToTypedElement(), CancellationToken.None); + await _fixture.Mediator.UpsertResourceAsync(searchParam.ToResourceElement()); + + if (!_searchParameterDefinitionManager.TryGetSearchParameter(searchParam.Url, out _)) + { + _searchParameterDefinitionManager.AddNewSearchParameters(new List { searchParam.ToTypedElement() }); + } + + await _searchParameterStatusManager.UpdateSearchParameterStatusAsync( + new List { searchParam.Url }, + SearchParameterStatus.Supported, + CancellationToken.None); + + await _searchParameterOperations.GetAndApplySearchParameterUpdates(CancellationToken.None, true); return searchParam; } + private async Task SearchForSearchParameterByUrlAsync(string searchParamUrl, CancellationToken cancellationToken, int maxAttempts = 10, int delayMilliseconds = 200) + { + SearchResult result = null; + var queryParams = new List> { new("url", searchParamUrl) }; + + for (int attempt = 0; attempt < maxAttempts; attempt++) + { + result = await _searchService.Value.SearchAsync(KnownResourceTypes.SearchParameter, queryParams, cancellationToken); + if (result.Results.Any()) + { + return result; + } + + await Task.Delay(delayMilliseconds, cancellationToken); + } + + return result; + } + private ResourceWrapper CreatePatientResourceWrapper(string patientName, string patientId) { Patient patientResource = Samples.GetDefaultPatient().ToPoco(); @@ -1517,16 +1580,14 @@ private async Task InitializeSecondFHIRService() var mediator = new Mediator(services); var searchParameterComparer = Substitute.For>(); - var statusDataStore = Substitute.For(); - var fhirDataStore = Substitute.For(); _searchParameterDefinitionManager2 = new SearchParameterDefinitionManager( ModelInfoProvider.Instance, mediator, - _searchService.CreateMockScopeProviderFromScoped(), + _fixture.SearchService.CreateMockScopeProvider(), searchParameterComparer, - statusDataStore.CreateMockScopeProvider(), - fhirDataStore.CreateMockScopeProvider(), + _fixture.SearchParameterStatusDataStore.CreateMockScopeProvider(), + _fixture.DataStore.CreateMockScopeProvider(), NullLogger.Instance); await _searchParameterDefinitionManager2.EnsureInitializedAsync(CancellationToken.None); _supportedSearchParameterDefinitionManager2 = new SupportedSearchParameterDefinitionManager(_searchParameterDefinitionManager2); @@ -1540,6 +1601,7 @@ private async Task InitializeSecondFHIRService() ModelInfoProvider.Instance, _searchParameterSupportResolver, _dataStoreSearchParameterValidator, + () => _fhirOperationDataStore.CreateMockScope(), () => _searchService, NullLogger.Instance); } diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/CosmosDbFhirStorageTestsFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/CosmosDbFhirStorageTestsFixture.cs index 6f5a43f1c7..185ae6f9f8 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/CosmosDbFhirStorageTestsFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/CosmosDbFhirStorageTestsFixture.cs @@ -228,6 +228,16 @@ public virtual async Task InitializeAsync() IOptions options = Options.Create(new CoreFeatureConfiguration()); + ISearchParameterSupportResolver searchParameterSupportResolver = Substitute.For(); + searchParameterSupportResolver.IsSearchParameterSupported(Arg.Any()).Returns((true, false)); + + _searchParameterStatusManager = new SearchParameterStatusManager( + _searchParameterStatusDataStore, + _searchParameterDefinitionManager, + searchParameterSupportResolver, + mediator, + NullLogger.Instance); + _fhirDataStore = new CosmosFhirDataStore( documentClient, _cosmosDataStoreConfiguration, @@ -238,7 +248,9 @@ public virtual async Task InitializeAsync() options, bundleOrchestrator, new Lazy(_supportedSearchParameterDefinitionManager), - ModelInfoProvider.Instance); + ModelInfoProvider.Instance, + _searchParameterStatusDataStore, + _fhirRequestContextAccessor); _queueClient = new CosmosQueueClient( () => _container.CreateMockScope(), @@ -284,16 +296,6 @@ public virtual async Task InitializeAsync() await _searchParameterDefinitionManager.EnsureInitializedAsync(CancellationToken.None); - ISearchParameterSupportResolver searchParameterSupportResolver = Substitute.For(); - searchParameterSupportResolver.IsSearchParameterSupported(Arg.Any()).Returns((true, false)); - - _searchParameterStatusManager = new SearchParameterStatusManager( - _searchParameterStatusDataStore, - _searchParameterDefinitionManager, - searchParameterSupportResolver, - mediator, - NullLogger.Instance); - var queueClient = new TestQueueClient(); _fhirOperationDataStore = new CosmosFhirOperationDataStore( queueClient, diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerWatchdogTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerWatchdogTests.cs index 76a32680f7..269216fd7c 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerWatchdogTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerWatchdogTests.cs @@ -330,7 +330,7 @@ WHILE @i < 10500 var wrapper = factory.Create(patient.ToResourceElement(), false, true); wrapper.ResourceSurrogateId = tran.TransactionId; var mergeWrapper = new MergeResourceWrapper(wrapper, true, true); - await _fixture.SqlServerFhirDataStore.MergeResourcesWrapperAsync(tran.TransactionId, false, [mergeWrapper], false, 0, cts.Token); + await _fixture.SqlServerFhirDataStore.MergeResourcesWrapperAsync(tran.TransactionId, false, [mergeWrapper], false, 0, null, cts.Token); var typeId = _fixture.SqlServerFhirModel.GetResourceTypeId("Patient"); ExecuteSql($"IF NOT EXISTS (SELECT * FROM dbo.Resource WHERE ResourceTypeId = {typeId} AND ResourceId = '{patient.Id}') RAISERROR('Resource is not created',18,127)"); @@ -397,7 +397,7 @@ CREATE TRIGGER dbo.tmp_NumberSearchParam ON dbo.NumberSearchParam FOR INSERT try { // no eventual consistency - await _fixture.SqlServerFhirDataStore.MergeResourcesWrapperAsync(tran.TransactionId, true, [mergeWrapper], false, 0, cts.Token); + await _fixture.SqlServerFhirDataStore.MergeResourcesWrapperAsync(tran.TransactionId, true, [mergeWrapper], false, 0, null, cts.Token); } catch (SqlException e) { @@ -409,7 +409,7 @@ CREATE TRIGGER dbo.tmp_NumberSearchParam ON dbo.NumberSearchParam FOR INSERT try { // eventual consistency - await _fixture.SqlServerFhirDataStore.MergeResourcesWrapperAsync(tran.TransactionId, false, [mergeWrapper], false, 0, cts.Token); + await _fixture.SqlServerFhirDataStore.MergeResourcesWrapperAsync(tran.TransactionId, false, [mergeWrapper], false, 0, null, cts.Token); } catch (SqlException e) {