Skip to content

Commit ffb2148

Browse files
jestradaMSjestradaMS
authored
Reindex updates for zeroCountParams and PendingDelete (#5205)
* Refactored and enhanced logic in SearchParameterDefinitionManager to improve search parameter initialization, deletion. * Updated ReindexOrchestratorJob to optimize job creation, resource counting, and search parameter readiness checks for reindex operations. * Improved error handling, logging, and resource type derivation in both files.
1 parent 996f23a commit ffb2148

File tree

11 files changed

+429
-140
lines changed

11 files changed

+429
-140
lines changed

‎src/Microsoft.Health.Fhir.Core/Features/Definition/SearchParameterDefinitionManager.cs‎

Lines changed: 159 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
using Microsoft.Health.Fhir.Core.Features.Definition.BundleWrappers;
2222
using Microsoft.Health.Fhir.Core.Features.Health;
2323
using Microsoft.Health.Fhir.Core.Features.Operations;
24+
using Microsoft.Health.Fhir.Core.Features.Persistence;
2425
using Microsoft.Health.Fhir.Core.Features.Search;
2526
using Microsoft.Health.Fhir.Core.Features.Search.Expressions;
2627
using Microsoft.Health.Fhir.Core.Features.Search.Parameters;
@@ -42,6 +43,8 @@ public class SearchParameterDefinitionManager : ISearchParameterDefinitionManage
4243
private readonly ConcurrentDictionary<string, string> _resourceTypeSearchParameterHashMap;
4344
private readonly IScopeProvider<ISearchService> _searchServiceFactory;
4445
private readonly ISearchParameterComparer<SearchParameterInfo> _searchParameterComparer;
46+
private readonly IScopeProvider<ISearchParameterStatusDataStore> _searchParameterStatusDataStoreFactory;
47+
private readonly IScopeProvider<IFhirDataStore> _fhirDataStoreFactory;
4548
private readonly ILogger _logger;
4649

4750
private bool _initialized = false;
@@ -51,12 +54,16 @@ public SearchParameterDefinitionManager(
5154
IMediator mediator,
5255
IScopeProvider<ISearchService> searchServiceFactory,
5356
ISearchParameterComparer<SearchParameterInfo> searchParameterComparer,
57+
IScopeProvider<ISearchParameterStatusDataStore> searchParameterStatusDataStoreFactory,
58+
IScopeProvider<IFhirDataStore> fhirDataStoreFactory,
5459
ILogger<SearchParameterDefinitionManager> logger)
5560
{
5661
EnsureArg.IsNotNull(modelInfoProvider, nameof(modelInfoProvider));
5762
EnsureArg.IsNotNull(mediator, nameof(mediator));
5863
EnsureArg.IsNotNull(searchServiceFactory, nameof(searchServiceFactory));
5964
EnsureArg.IsNotNull(searchParameterComparer, nameof(searchParameterComparer));
65+
EnsureArg.IsNotNull(searchParameterStatusDataStoreFactory, nameof(searchParameterStatusDataStoreFactory));
66+
EnsureArg.IsNotNull(fhirDataStoreFactory, nameof(fhirDataStoreFactory));
6067
EnsureArg.IsNotNull(logger, nameof(logger));
6168

6269
_modelInfoProvider = modelInfoProvider;
@@ -66,6 +73,8 @@ public SearchParameterDefinitionManager(
6673
UrlLookup = new ConcurrentDictionary<string, SearchParameterInfo>();
6774
_searchServiceFactory = searchServiceFactory;
6875
_searchParameterComparer = searchParameterComparer;
76+
_searchParameterStatusDataStoreFactory = searchParameterStatusDataStoreFactory;
77+
_fhirDataStoreFactory = fhirDataStoreFactory;
6978
_logger = logger;
7079

7180
var bundle = SearchParameterDefinitionBuilder.ReadEmbeddedSearchParameters("search-parameters.json", _modelInfoProvider);
@@ -370,14 +379,38 @@ private async Task LoadSearchParamsFromDataStore(CancellationToken cancellationT
370379
{
371380
// now read in any previously POST'd SearchParameter resources
372381
using IScoped<ISearchService> search = _searchServiceFactory.Invoke();
382+
using IScoped<ISearchParameterStatusDataStore> statusDataStore = _searchParameterStatusDataStoreFactory.Invoke();
383+
using IScoped<IFhirDataStore> fhirDataStore = _fhirDataStoreFactory.Invoke();
384+
373385
string continuationToken = null;
386+
int totalLoaded = 0;
387+
int totalPendingDelete = 0;
388+
389+
// Get all PendingDelete search parameters from the status store
390+
var allStatuses = await statusDataStore.Value.GetSearchParameterStatuses(cancellationToken);
391+
var pendingDeleteUrls = new HashSet<string>(
392+
allStatuses
393+
.Where(s => s.Status == SearchParameterStatus.PendingDelete)
394+
.Select(s => s.Uri.OriginalString),
395+
StringComparer.OrdinalIgnoreCase);
396+
397+
_logger.LogInformation(
398+
"Found {PendingDeleteCount} search parameters with PendingDelete status in the status store",
399+
pendingDeleteUrls.Count);
400+
374401
do
375402
{
376403
var searchOptions = new SearchOptions();
377404
searchOptions.Sort = new List<(SearchParameterInfo, SortOrder)>();
378405
searchOptions.UnsupportedSearchParams = new List<Tuple<string, string>>();
379-
searchOptions.Expression = Expression.SearchParameter(SearchParameterInfo.ResourceTypeSearchParameter, Expression.StringEquals(FieldName.TokenCode, null, KnownResourceTypes.SearchParameter, false));
406+
searchOptions.Expression = Expression.SearchParameter(
407+
SearchParameterInfo.ResourceTypeSearchParameter,
408+
Expression.StringEquals(FieldName.TokenCode, null, KnownResourceTypes.SearchParameter, false));
380409
searchOptions.MaxItemCount = 10;
410+
411+
// ✅ Include soft-deleted resources to find PendingDelete search parameters
412+
searchOptions.ResourceVersionTypes = ResourceVersionType.Latest | ResourceVersionType.SoftDeleted;
413+
381414
if (continuationToken != null)
382415
{
383416
searchOptions.ContinuationToken = continuationToken;
@@ -388,40 +421,142 @@ private async Task LoadSearchParamsFromDataStore(CancellationToken cancellationT
388421

389422
if (result?.Results != null && result.Results.Any())
390423
{
391-
var searchParams = result.Results.Select(r => r.Resource.RawResource.ToITypedElement(_modelInfoProvider)).ToList();
392-
393-
_logger.LogInformation("There are {CustomSearchParameters} custom Search Parameters", result.Results.Count().ToString());
394-
395-
foreach (var searchParam in searchParams)
424+
foreach (var searchResult in result.Results)
396425
{
397-
try
398-
{
399-
SearchParameterDefinitionBuilder.Build(
400-
new List<ITypedElement>() { searchParam },
401-
UrlLookup,
402-
TypeLookup,
403-
_modelInfoProvider,
404-
_searchParameterComparer,
405-
_logger);
406-
}
407-
catch (FhirException ex)
426+
var isDeleted = searchResult.Resource.IsDeleted;
427+
428+
// For soft-deleted resources, check if they are in PendingDelete status
429+
if (isDeleted)
408430
{
409-
StringBuilder issueDetails = new StringBuilder();
410-
foreach (OperationOutcomeIssue issue in ex.Issues)
431+
try
411432
{
412-
issueDetails.Append(issue.Diagnostics).Append("; ");
433+
// Get the resource ID to fetch its last version before deletion
434+
var resourceId = searchResult.Resource.ResourceId;
435+
436+
// Parse the current version and calculate the previous version
437+
if (int.TryParse(searchResult.Resource.Version, out int currentVersion) && currentVersion > 1)
438+
{
439+
var previousVersion = (currentVersion - 1).ToString();
440+
var resourceKey = new ResourceKey(KnownResourceTypes.SearchParameter, resourceId, previousVersion);
441+
var lastVersion = await fhirDataStore.Value.GetAsync(resourceKey, cancellationToken);
442+
443+
if (lastVersion?.RawResource != null)
444+
{
445+
var searchParam = lastVersion.RawResource.ToITypedElement(_modelInfoProvider);
446+
var urlScalar = searchParam.GetStringScalar("url");
447+
448+
// Only load if this URL is marked as PendingDelete in the status store
449+
if (!string.IsNullOrEmpty(urlScalar) && pendingDeleteUrls.Contains(urlScalar))
450+
{
451+
// Build the search parameter using the last version before deletion
452+
SearchParameterDefinitionBuilder.Build(
453+
new List<ITypedElement>() { searchParam },
454+
UrlLookup,
455+
TypeLookup,
456+
_modelInfoProvider,
457+
_searchParameterComparer,
458+
_logger);
459+
460+
totalLoaded++;
461+
462+
// Update the status to PendingDelete since the resource is soft-deleted
463+
if (UrlLookup.TryGetValue(urlScalar, out var loadedParam))
464+
{
465+
loadedParam.SearchParameterStatus = SearchParameterStatus.PendingDelete;
466+
totalPendingDelete++;
467+
_logger.LogInformation(
468+
"Loaded PendingDelete search parameter from last version before deletion: {Url}",
469+
urlScalar);
470+
}
471+
}
472+
else if (!string.IsNullOrEmpty(urlScalar))
473+
{
474+
_logger.LogDebug(
475+
"Skipping soft-deleted SearchParameter {ResourceId} with URL {Url} - not in PendingDelete status",
476+
resourceId,
477+
urlScalar);
478+
}
479+
else
480+
{
481+
_logger.LogWarning(
482+
"Could not retrieve valid URL for soft-deleted SearchParameter {ResourceId}",
483+
resourceId);
484+
}
485+
}
486+
else
487+
{
488+
_logger.LogWarning(
489+
"Could not retrieve last version for soft-deleted SearchParameter {ResourceId}",
490+
resourceId);
491+
}
492+
}
493+
else
494+
{
495+
_logger.LogWarning(
496+
"Could not parse version or version is 1 for soft-deleted SearchParameter {ResourceId}, version: {Version}",
497+
resourceId,
498+
searchResult.Resource.Version);
499+
}
500+
}
501+
catch (Exception ex)
502+
{
503+
_logger.LogError(
504+
ex,
505+
"Error loading last version of soft-deleted SearchParameter {ResourceId}",
506+
searchResult.Resource.ResourceId);
413507
}
414-
415-
_logger.LogWarning(ex, "Error loading search parameter {Url} from data store. Issues: {Issues}", searchParam.GetStringScalar("url"), issueDetails.ToString());
416508
}
417-
catch (Exception ex)
509+
else
418510
{
419-
_logger.LogError(ex, "Error loading search parameter {Url} from data store.", searchParam.GetStringScalar("url"));
511+
// Normal processing for active resources
512+
var searchParam = searchResult.Resource.RawResource.ToITypedElement(_modelInfoProvider);
513+
514+
try
515+
{
516+
SearchParameterDefinitionBuilder.Build(
517+
new List<ITypedElement>() { searchParam },
518+
UrlLookup,
519+
TypeLookup,
520+
_modelInfoProvider,
521+
_searchParameterComparer,
522+
_logger);
523+
524+
totalLoaded++;
525+
}
526+
catch (FhirException ex)
527+
{
528+
StringBuilder issueDetails = new StringBuilder();
529+
foreach (OperationOutcomeIssue issue in ex.Issues)
530+
{
531+
issueDetails.Append(issue.Diagnostics).Append("; ");
532+
}
533+
534+
_logger.LogWarning(
535+
ex,
536+
"Error loading search parameter {Url} from data store. Issues: {Issues}",
537+
searchParam.GetStringScalar("url"),
538+
issueDetails.ToString());
539+
}
540+
catch (Exception ex) when (
541+
!(ex is OutOfMemoryException
542+
|| ex is StackOverflowException
543+
|| ex is ThreadAbortException))
544+
{
545+
_logger.LogError(
546+
ex,
547+
"Error loading search parameter {Url} from data store.",
548+
searchParam.GetStringScalar("url"));
549+
}
420550
}
421551
}
422552
}
423553
}
424554
while (continuationToken != null);
555+
556+
_logger.LogInformation(
557+
"Loaded {TotalLoaded} active and {TotalPendingDelete} PendingDelete search parameters from data store",
558+
totalLoaded,
559+
totalPendingDelete);
425560
}
426561

427562
[System.Diagnostics.CodeAnalysis.SuppressMessage("Performance", "CA1859:Use concrete types when possible for improved performance", Justification = "Collection defined on model")]

0 commit comments

Comments
 (0)