From 23e9df12ba5ec3f8a7edbb164e39bd1971a189e7 Mon Sep 17 00:00:00 2001 From: viacheslav_kolesnyk Date: Wed, 4 Mar 2026 17:29:30 +0100 Subject: [PATCH 1/9] fix(indexing): Improve memory management for resource indexing --- .../search/service/converter/SearchDocumentConverter.java | 7 +++++-- src/main/java/org/folio/search/utils/JsonConverter.java | 7 ++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/folio/search/service/converter/SearchDocumentConverter.java b/src/main/java/org/folio/search/service/converter/SearchDocumentConverter.java index 30382c677..33d324e31 100644 --- a/src/main/java/org/folio/search/service/converter/SearchDocumentConverter.java +++ b/src/main/java/org/folio/search/service/converter/SearchDocumentConverter.java @@ -82,8 +82,11 @@ private SearchDocumentBody convert(ConversionContext context) { var baseFields = convertMapUsingResourceFields(getNewAsMap(resourceEvent), resourceDescriptionFields, context); var searchFields = searchFieldsProcessor.getSearchFields(context); var resultDocument = mergeSafely(baseFields, searchFields); - return SearchDocumentBody.of(searchDocumentBodyConverter.apply(resultDocument), - indexingDataFormat, resourceEvent, INDEX); + // Release the large _new Map before serialization — resultDocument is already a fully independent copy. + // ResourceEvent is still referenced by SearchDocumentBody for id/tenant/resource metadata only. + resourceEvent.setNew(null); + var documentBody = searchDocumentBodyConverter.apply(resultDocument); + return SearchDocumentBody.of(documentBody, indexingDataFormat, resourceEvent, INDEX); } private List getResourceLanguages(List languageSource, Map resourceData) { diff --git a/src/main/java/org/folio/search/utils/JsonConverter.java b/src/main/java/org/folio/search/utils/JsonConverter.java index 719049d1f..0ba324bd1 100644 --- a/src/main/java/org/folio/search/utils/JsonConverter.java +++ b/src/main/java/org/folio/search/utils/JsonConverter.java @@ -220,7 +220,12 @@ public BytesArray toJsonBytes(Object value) { if (value == null) { return null; } - return new BytesArray(toJson(value)); + try { + return new BytesArray(objectMapper.writeValueAsBytes(value)); + } catch (JacksonException e) { + throw new SerializationException(String.format( + SERIALIZATION_ERROR_MSG_TEMPLATE, e.getMessage()), e); + } } /** From 1fd5f93e9ec77fb936a77e8f6967dc18c41bfff0 Mon Sep 17 00:00:00 2001 From: viacheslav_kolesnyk Date: Wed, 4 Mar 2026 18:55:53 +0100 Subject: [PATCH 2/9] Fix test --- src/test/java/org/folio/search/utils/JsonConverterTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/folio/search/utils/JsonConverterTest.java b/src/test/java/org/folio/search/utils/JsonConverterTest.java index 90b8d2b11..1922c324c 100644 --- a/src/test/java/org/folio/search/utils/JsonConverterTest.java +++ b/src/test/java/org/folio/search/utils/JsonConverterTest.java @@ -67,7 +67,7 @@ void toJsonBytes_positive() throws JacksonException { var actual = jsonConverter.toJsonBytes(TestClass.of(FIELD_VALUE)); assertThat(actual).isEqualTo(JSON_BYTES_BODY); - verify(objectMapper).writeValueAsString(TestClass.of(FIELD_VALUE)); + verify(objectMapper).writeValueAsBytes(TestClass.of(FIELD_VALUE)); } @Test From 4d14de1f0151d527f80b42bae70f9626cfa19c0a Mon Sep 17 00:00:00 2001 From: viacheslav_kolesnyk Date: Mon, 9 Mar 2026 19:40:30 +0100 Subject: [PATCH 3/9] Fix instance sharing logic being used on reindexing Improve memory management --- .../search/model/entity/CallNumberEntity.java | 97 ------------------- .../entity/InstanceCallNumberEntity.java | 17 ---- .../InstanceChildrenResourceService.java | 12 ++- .../MultiTenantSearchDocumentConverter.java | 23 +++++ .../impl/CallNumberResourceExtractor.java | 67 ++++++------- .../reindex/ReindexOrchestrationService.java | 3 +- .../reindex/jdbc/CallNumberRepository.java | 7 +- .../jdbc/ClassificationRepository.java | 39 +++----- .../reindex/jdbc/ContributorRepository.java | 42 ++++---- .../service/reindex/jdbc/ItemRepository.java | 6 +- .../reindex/jdbc/MergeInstanceRepository.java | 6 +- .../reindex/jdbc/SubjectRepository.java | 45 ++++----- .../reindex/jdbc/UploadRangeRepository.java | 13 --- ...ndexingInstanceCallNumberConsortiumIT.java | 2 + .../IndexingInstanceCallNumberIT.java | 6 +- .../impl/CallNumberResourceExtractorTest.java | 18 ++-- .../ReindexOrchestrationServiceTest.java | 23 +++-- .../reindex/jdbc/CallNumberRepositoryIT.java | 10 +- 18 files changed, 148 insertions(+), 288 deletions(-) delete mode 100644 src/main/java/org/folio/search/model/entity/CallNumberEntity.java delete mode 100644 src/main/java/org/folio/search/model/entity/InstanceCallNumberEntity.java diff --git a/src/main/java/org/folio/search/model/entity/CallNumberEntity.java b/src/main/java/org/folio/search/model/entity/CallNumberEntity.java deleted file mode 100644 index 4d103fd3f..000000000 --- a/src/main/java/org/folio/search/model/entity/CallNumberEntity.java +++ /dev/null @@ -1,97 +0,0 @@ -package org.folio.search.model.entity; - -import static org.apache.commons.lang3.StringUtils.truncate; - -import java.util.Objects; -import lombok.Getter; -import org.folio.search.utils.ShaUtils; -import org.jspecify.annotations.NonNull; - -@Getter -public class CallNumberEntity implements Comparable { - - public static final int CALL_NUMBER_MAX_LENGTH = 50; - private static final int CALL_NUMBER_PREFIX_MAX_LENGTH = 20; - private static final int CALL_NUMBER_SUFFIX_MAX_LENGTH = 25; - private static final int CALL_NUMBER_TYPE_MAX_LENGTH = 40; - - private final String id; - private final String callNumber; - private final String callNumberPrefix; - private final String callNumberSuffix; - private final String callNumberTypeId; - - CallNumberEntity(String id, String callNumber, String callNumberPrefix, String callNumberSuffix, - String callNumberTypeId) { - this.id = id; - this.callNumber = callNumber; - this.callNumberPrefix = callNumberPrefix; - this.callNumberSuffix = callNumberSuffix; - this.callNumberTypeId = callNumberTypeId; - } - - public static CallNumberEntityBuilder builder() { - return new CallNumberEntityBuilder(); - } - - @Override - public int hashCode() { - return Objects.hashCode(id); - } - - @Override - public boolean equals(Object o) { - if (!(o instanceof CallNumberEntity that)) { - return false; - } - return Objects.equals(id, that.id); - } - - @Override - public int compareTo(@NonNull CallNumberEntity o) { - return id.compareTo(o.id); - } - - public static class CallNumberEntityBuilder { - private String id; - private String callNumber; - private String callNumberPrefix; - private String callNumberSuffix; - private String callNumberTypeId; - - CallNumberEntityBuilder() { } - - public CallNumberEntityBuilder id(String id) { - this.id = id; - return this; - } - - public CallNumberEntityBuilder callNumber(String callNumber) { - this.callNumber = truncate(callNumber, CALL_NUMBER_MAX_LENGTH); - return this; - } - - public CallNumberEntityBuilder callNumberPrefix(String callNumberPrefix) { - this.callNumberPrefix = truncate(callNumberPrefix, CALL_NUMBER_PREFIX_MAX_LENGTH); - return this; - } - - public CallNumberEntityBuilder callNumberSuffix(String callNumberSuffix) { - this.callNumberSuffix = truncate(callNumberSuffix, CALL_NUMBER_SUFFIX_MAX_LENGTH); - return this; - } - - public CallNumberEntityBuilder callNumberTypeId(String callNumberTypeId) { - this.callNumberTypeId = truncate(callNumberTypeId, CALL_NUMBER_TYPE_MAX_LENGTH); - return this; - } - - public CallNumberEntity build() { - if (id == null) { - this.id = ShaUtils.sha(callNumber, callNumberPrefix, callNumberSuffix, callNumberTypeId); - } - return new CallNumberEntity(this.id, this.callNumber, this.callNumberPrefix, this.callNumberSuffix, - this.callNumberTypeId); - } - } -} diff --git a/src/main/java/org/folio/search/model/entity/InstanceCallNumberEntity.java b/src/main/java/org/folio/search/model/entity/InstanceCallNumberEntity.java deleted file mode 100644 index a50f3484e..000000000 --- a/src/main/java/org/folio/search/model/entity/InstanceCallNumberEntity.java +++ /dev/null @@ -1,17 +0,0 @@ -package org.folio.search.model.entity; - -import lombok.Builder; -import lombok.EqualsAndHashCode; -import lombok.Getter; - -@Getter -@Builder -@EqualsAndHashCode -public class InstanceCallNumberEntity { - - private String callNumberId; - private String itemId; - private String instanceId; - private String locationId; - private String tenantId; -} diff --git a/src/main/java/org/folio/search/service/InstanceChildrenResourceService.java b/src/main/java/org/folio/search/service/InstanceChildrenResourceService.java index bb48a4b21..e8b3cb0a6 100644 --- a/src/main/java/org/folio/search/service/InstanceChildrenResourceService.java +++ b/src/main/java/org/folio/search/service/InstanceChildrenResourceService.java @@ -93,6 +93,16 @@ public void persistChildrenOnReindex(String tenantId, ResourceType resourceType, .tenant(tenantId) ._new(instance)) .toList(); - persistChildren(tenantId, resourceType, events); + + var extractors = resourceExtractors.get(resourceType); + if (extractors == null) { + return; + } + + var shared = consortiumTenantProvider.isCentralTenant(tenantId); + + // Process child resources normally + extractors.forEach(resourceExtractor -> + resourceExtractor.persistChildren(tenantId, shared, events)); } } diff --git a/src/main/java/org/folio/search/service/converter/MultiTenantSearchDocumentConverter.java b/src/main/java/org/folio/search/service/converter/MultiTenantSearchDocumentConverter.java index dd66f81e2..62d140185 100644 --- a/src/main/java/org/folio/search/service/converter/MultiTenantSearchDocumentConverter.java +++ b/src/main/java/org/folio/search/service/converter/MultiTenantSearchDocumentConverter.java @@ -57,6 +57,29 @@ public Map> convert(Collection r .collect(groupingBy(SearchDocumentBody::getResource)); } + /** + * Converts {@link ResourceEvent} objects to a flat list of {@link SearchDocumentBody} objects. + * All events are expected to carry the same tenant ID that matches the current execution context — + * per-tenant grouping and context switching are therefore skipped entirely. + * + * @param resourceEvents list with {@link ResourceEvent} objects, all belonging to the current tenant + * @return flat {@link List} of {@link SearchDocumentBody} objects + */ + public List convertForReindex(Collection resourceEvents) { + log.debug("convertForReindex:: by [resourceEvents.size: {}]", collectionToLogMsg(resourceEvents, true)); + + if (CollectionUtils.isEmpty(resourceEvents)) { + return List.of(); + } + + return resourceEvents.stream() + .flatMap(this::populateResourceEvents) + .map(event -> event.getId() != null ? event : event.id(getResourceEventId(event))) + .map(searchDocumentConverter::convert) + .flatMap(Optional::stream) + .toList(); + } + private List convertForTenant(Entry> entry) { var convert = (Supplier>) () -> entry.getValue().stream() diff --git a/src/main/java/org/folio/search/service/converter/preprocessor/extractor/impl/CallNumberResourceExtractor.java b/src/main/java/org/folio/search/service/converter/preprocessor/extractor/impl/CallNumberResourceExtractor.java index b37dfbe43..dfe493627 100644 --- a/src/main/java/org/folio/search/service/converter/preprocessor/extractor/impl/CallNumberResourceExtractor.java +++ b/src/main/java/org/folio/search/service/converter/preprocessor/extractor/impl/CallNumberResourceExtractor.java @@ -2,26 +2,24 @@ import static org.apache.commons.collections4.MapUtils.getMap; import static org.apache.commons.collections4.MapUtils.getString; -import static org.folio.search.model.entity.CallNumberEntity.CALL_NUMBER_MAX_LENGTH; +import static org.apache.commons.lang3.StringUtils.truncate; import static org.folio.search.utils.SearchConverterUtils.getNewAsMap; import static org.folio.search.utils.SearchUtils.prepareForExpectedFormat; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Set; import lombok.extern.log4j.Log4j2; import org.apache.commons.lang3.StringUtils; import org.folio.search.domain.dto.ResourceEvent; import org.folio.search.domain.dto.TenantConfiguredFeature; -import org.folio.search.model.entity.CallNumberEntity; -import org.folio.search.model.entity.InstanceCallNumberEntity; import org.folio.search.model.types.ResourceType; import org.folio.search.service.FeatureConfigService; import org.folio.search.service.converter.preprocessor.extractor.ChildResourceExtractor; import org.folio.search.service.reindex.jdbc.CallNumberRepository; -import org.folio.search.utils.JsonConverter; +import org.folio.search.utils.ShaUtils; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; @@ -36,14 +34,16 @@ public class CallNumberResourceExtractor extends ChildResourceExtractor { public static final String SUFFIX_FIELD = "suffix"; public static final String TYPE_ID_FIELD = "typeId"; - private final JsonConverter jsonConverter; + private static final int CALL_NUMBER_MAX_LENGTH = 50; + private static final int CALL_NUMBER_PREFIX_MAX_LENGTH = 20; + private static final int CALL_NUMBER_SUFFIX_MAX_LENGTH = 25; + private static final int CALL_NUMBER_TYPE_MAX_LENGTH = 40; + private final FeatureConfigService featureConfigService; public CallNumberResourceExtractor(CallNumberRepository repository, - JsonConverter jsonConverter, FeatureConfigService featureConfigService) { super(repository); - this.jsonConverter = jsonConverter; this.featureConfigService = featureConfigService; } @@ -57,14 +57,12 @@ protected List> constructRelations(boolean shared, ResourceE List> entities) { var resourceMap = getNewAsMap(event); return entities.stream() - .map(entity -> InstanceCallNumberEntity.builder() - .callNumberId(getString(entity, "id")) - .itemId(getString(resourceMap, "id")) - .instanceId(getString(resourceMap, "instanceId")) - .locationId(getString(resourceMap, "effectiveLocationId")) - .tenantId(event.getTenant()) - .build()) - .map(jsonConverter::convertToMap) + .map(entity -> Map.of( + "callNumberId", getString(entity, "id"), + "itemId", getString(resourceMap, "id"), + "instanceId", getString(resourceMap, "instanceId"), + "locationId", getString(resourceMap, "effectiveLocationId"), + "tenantId", event.getTenant())) .toList(); } @@ -73,9 +71,24 @@ protected Map constructEntity(Map entityProperti if (entityProperties == null || !featureConfigService.isEnabled(TenantConfiguredFeature.BROWSE_CALL_NUMBERS)) { return Collections.emptyMap(); } - return toCallNumberEntity(entityProperties) - .map(jsonConverter::convertToMap) - .orElse(Collections.emptyMap()); + var callNumberComponents = getCallNumberComponents(entityProperties); + var callNumber = prepareForExpectedFormat( + getString(callNumberComponents, CALL_NUMBER_FIELD), CALL_NUMBER_MAX_LENGTH); + if (StringUtils.isBlank(callNumber)) { + return Collections.emptyMap(); + } + var callNumberPrefix = truncate(getString(callNumberComponents, PREFIX_FIELD), CALL_NUMBER_PREFIX_MAX_LENGTH); + var callNumberSuffix = truncate(getString(callNumberComponents, SUFFIX_FIELD), CALL_NUMBER_SUFFIX_MAX_LENGTH); + var callNumberTypeId = truncate(getString(callNumberComponents, TYPE_ID_FIELD), CALL_NUMBER_TYPE_MAX_LENGTH); + var id = ShaUtils.sha(callNumber, callNumberPrefix, callNumberSuffix, callNumberTypeId); + + var entity = new HashMap(); + entity.put("id", id); + entity.put("callNumber", callNumber); + entity.put("callNumberPrefix", callNumberPrefix); + entity.put("callNumberSuffix", callNumberSuffix); + entity.put("callNumberTypeId", callNumberTypeId); + return entity; } @Override @@ -88,22 +101,6 @@ protected Set> getChildResources(Map event) return Set.of(event); } - private Optional toCallNumberEntity(Map entityProperties) { - var callNumberComponents = getCallNumberComponents(entityProperties); - var callNumber = prepareForExpectedFormat(getString(callNumberComponents, CALL_NUMBER_FIELD), - CALL_NUMBER_MAX_LENGTH); - if (StringUtils.isNotBlank(callNumber)) { - var callNumberEntity = CallNumberEntity.builder() - .callNumber(callNumber) - .callNumberPrefix(getString(callNumberComponents, PREFIX_FIELD)) - .callNumberSuffix(getString(callNumberComponents, SUFFIX_FIELD)) - .callNumberTypeId(getString(callNumberComponents, TYPE_ID_FIELD)) - .build(); - return Optional.of(callNumberEntity); - } - return Optional.empty(); - } - @SuppressWarnings("unchecked") private Map getCallNumberComponents(Map entityProperties) { return (Map) getMap(entityProperties, EFFECTIVE_CALL_NUMBER_COMPONENTS_FIELD, diff --git a/src/main/java/org/folio/search/service/reindex/ReindexOrchestrationService.java b/src/main/java/org/folio/search/service/reindex/ReindexOrchestrationService.java index 1390c3b17..911bfb978 100644 --- a/src/main/java/org/folio/search/service/reindex/ReindexOrchestrationService.java +++ b/src/main/java/org/folio/search/service/reindex/ReindexOrchestrationService.java @@ -1,6 +1,5 @@ package org.folio.search.service.reindex; -import java.util.Collection; import lombok.RequiredArgsConstructor; import lombok.extern.log4j.Log4j2; import org.apache.logging.log4j.message.FormattedMessage; @@ -75,7 +74,7 @@ public boolean process(ReindexRecordsEvent event) { private FolioIndexOperationResponse fetchRecordsAndIndexForUploadRange(ReindexRangeIndexEvent event) { try { var resourceEvents = uploadRangeService.fetchRecordRange(event); - var documents = documentConverter.convert(resourceEvents).values().stream().flatMap(Collection::stream).toList(); + var documents = documentConverter.convertForReindex(resourceEvents); return elasticRepository.indexResources(documents); } catch (Exception ex) { throw handleReindexUploadFailure(event, ex.getMessage()); diff --git a/src/main/java/org/folio/search/service/reindex/jdbc/CallNumberRepository.java b/src/main/java/org/folio/search/service/reindex/jdbc/CallNumberRepository.java index f09d08f92..a6bb6d8dc 100644 --- a/src/main/java/org/folio/search/service/reindex/jdbc/CallNumberRepository.java +++ b/src/main/java/org/folio/search/service/reindex/jdbc/CallNumberRepository.java @@ -258,10 +258,6 @@ protected RowMapper> rowToMapMapper() { protected RowMapper> rowToMapMapper2() { return (rs, rowNum) -> { var callNumberMap = getCallNumberMap(rs); - var maps = jsonConverter.fromJsonToListOfMaps(getInstances(rs)).stream().filter(Objects::nonNull).toList(); - if (!maps.isEmpty()) { - callNumberMap.put(SUB_RESOURCE_INSTANCES_FIELD, maps); - } callNumberMap.put(LAST_UPDATED_DATE_FIELD, rs.getTimestamp("last_updated_date")); return callNumberMap; }; @@ -278,8 +274,7 @@ private Map getCallNumberMap(ResultSet rs) throws SQLException { callNumberMap.put(CALL_NUMBER_PREFIX_FIELD, getCallNumberPrefix(rs)); callNumberMap.put(CALL_NUMBER_SUFFIX_FIELD, callNumberSuffix); callNumberMap.put(CALL_NUMBER_TYPE_ID_FIELD, getCallNumberTypeId(rs)); - var subResources = jsonConverter.toJson(parseInstanceSubResources(getInstances(rs))); - var maps = jsonConverter.fromJsonToListOfMaps(subResources).stream().filter(Objects::nonNull).toList(); + var maps = jsonConverter.fromJsonToListOfMaps(getInstances(rs)).stream().filter(Objects::nonNull).toList(); if (!maps.isEmpty()) { callNumberMap.put(SUB_RESOURCE_INSTANCES_FIELD, maps); } diff --git a/src/main/java/org/folio/search/service/reindex/jdbc/ClassificationRepository.java b/src/main/java/org/folio/search/service/reindex/jdbc/ClassificationRepository.java index 0b6d31b4c..0989b268e 100644 --- a/src/main/java/org/folio/search/service/reindex/jdbc/ClassificationRepository.java +++ b/src/main/java/org/folio/search/service/reindex/jdbc/ClassificationRepository.java @@ -179,19 +179,7 @@ protected String getFetchBySql() { @Override protected RowMapper> rowToMapMapper() { - return (rs, rowNum) -> { - Map classification = new HashMap<>(); - classification.put("id", getId(rs)); - classification.put(CLASSIFICATION_NUMBER_ENTITY_FIELD, getNumber(rs)); - classification.put("typeId", getTypeId(rs)); - - var maps = jsonConverter.fromJsonToListOfMaps(getInstances(rs)).stream().filter(Objects::nonNull).toList(); - if (!maps.isEmpty()) { - classification.put(SUB_RESOURCE_INSTANCES_FIELD, maps); - } - - return classification; - }; + return (rs, rowNum) -> buildClassificationMap(rs); } @Override @@ -238,21 +226,26 @@ public void saveAll(ChildResourceEntityBatch entityBatch) { protected RowMapper> rowToMapMapper2() { return (rs, rowNum) -> { - Map classification = new HashMap<>(); - classification.put("id", getId(rs)); - classification.put(CLASSIFICATION_NUMBER_ENTITY_FIELD, getNumber(rs)); - classification.put("typeId", getTypeId(rs)); + var classification = buildClassificationMap(rs); classification.put(LAST_UPDATED_DATE_FIELD, rs.getTimestamp("last_updated_date")); - - var maps = jsonConverter.fromJsonToListOfMaps(getInstances(rs)).stream().filter(Objects::nonNull).toList(); - if (!maps.isEmpty()) { - classification.put(SUB_RESOURCE_INSTANCES_FIELD, maps); - } - return classification; }; } + private Map buildClassificationMap(ResultSet rs) throws SQLException { + Map classification = new HashMap<>(); + classification.put("id", getId(rs)); + classification.put(CLASSIFICATION_NUMBER_ENTITY_FIELD, getNumber(rs)); + classification.put("typeId", getTypeId(rs)); + + var maps = jsonConverter.fromJsonToListOfMaps(getInstances(rs)).stream().filter(Objects::nonNull).toList(); + if (!maps.isEmpty()) { + classification.put(SUB_RESOURCE_INSTANCES_FIELD, maps); + } + + return classification; + } + private String getId(ResultSet rs) throws SQLException { return rs.getString("id"); } diff --git a/src/main/java/org/folio/search/service/reindex/jdbc/ContributorRepository.java b/src/main/java/org/folio/search/service/reindex/jdbc/ContributorRepository.java index 372ffa2a1..9e3ee79df 100644 --- a/src/main/java/org/folio/search/service/reindex/jdbc/ContributorRepository.java +++ b/src/main/java/org/folio/search/service/reindex/jdbc/ContributorRepository.java @@ -158,20 +158,7 @@ protected String getFetchBySql() { @Override protected RowMapper> rowToMapMapper() { - return (rs, rowNum) -> { - Map contributor = new HashMap<>(); - contributor.put("id", getId(rs)); - contributor.put("name", getName(rs)); - contributor.put("contributorNameTypeId", getNameTypeId(rs)); - contributor.put(AUTHORITY_ID_FIELD, getAuthorityId(rs)); - - var maps = jsonConverter.fromJsonToListOfMaps(getInstances(rs)).stream().filter(Objects::nonNull).toList(); - if (!maps.isEmpty()) { - contributor.put(SUB_RESOURCE_INSTANCES_FIELD, maps); - } - - return contributor; - }; + return (rs, rowNum) -> buildContributorMap(rs); } @Override @@ -245,22 +232,27 @@ public void saveAll(ChildResourceEntityBatch entityBatch) { protected RowMapper> rowToMapMapper2() { return (rs, rowNum) -> { - Map contributor = new HashMap<>(); - contributor.put("id", getId(rs)); - contributor.put("name", getName(rs)); - contributor.put("contributorNameTypeId", getNameTypeId(rs)); + var contributor = buildContributorMap(rs); contributor.put(LAST_UPDATED_DATE_FIELD, rs.getTimestamp("last_updated_date")); - contributor.put(AUTHORITY_ID_FIELD, getAuthorityId(rs)); - - var maps = jsonConverter.fromJsonToListOfMaps(getInstances(rs)).stream().filter(Objects::nonNull).toList(); - if (!maps.isEmpty()) { - contributor.put(SUB_RESOURCE_INSTANCES_FIELD, maps); - } - return contributor; }; } + private Map buildContributorMap(ResultSet rs) throws SQLException { + Map contributor = new HashMap<>(); + contributor.put("id", getId(rs)); + contributor.put("name", getName(rs)); + contributor.put("contributorNameTypeId", getNameTypeId(rs)); + contributor.put(AUTHORITY_ID_FIELD, getAuthorityId(rs)); + + var maps = jsonConverter.fromJsonToListOfMaps(getInstances(rs)).stream().filter(Objects::nonNull).toList(); + if (!maps.isEmpty()) { + contributor.put(SUB_RESOURCE_INSTANCES_FIELD, maps); + } + + return contributor; + } + private String getId(ResultSet rs) throws SQLException { return rs.getString("id"); } diff --git a/src/main/java/org/folio/search/service/reindex/jdbc/ItemRepository.java b/src/main/java/org/folio/search/service/reindex/jdbc/ItemRepository.java index a89a6f47b..62f8690c4 100644 --- a/src/main/java/org/folio/search/service/reindex/jdbc/ItemRepository.java +++ b/src/main/java/org/folio/search/service/reindex/jdbc/ItemRepository.java @@ -3,7 +3,6 @@ import static org.folio.search.utils.JdbcUtils.getFullTableName; import java.sql.Timestamp; -import java.util.HashMap; import java.util.List; import java.util.Map; import org.folio.search.configuration.properties.SearchConfigurationProperties; @@ -80,7 +79,7 @@ public SubResourceResult fetchByTimestamp(String tenant, Timestamp timestamp, St private RowMapper> itemRowMapper() { return (rs, rowNum) -> { - Map item = new HashMap<>(); + var item = jsonConverter.fromJsonToMap(rs.getString("json")); item.put("id", rs.getString("id")); item.put("tenantId", rs.getString("tenant_id")); item.put("instanceId", rs.getString("instance_id")); @@ -88,9 +87,6 @@ private RowMapper> itemRowMapper() { item.put("isDeleted", rs.getBoolean("is_deleted")); item.put(LAST_UPDATED_DATE_FIELD, rs.getTimestamp("last_updated_date")); - var jsonContent = jsonConverter.fromJsonToMap(rs.getString("json")); - item.putAll(jsonContent); - return item; }; } diff --git a/src/main/java/org/folio/search/service/reindex/jdbc/MergeInstanceRepository.java b/src/main/java/org/folio/search/service/reindex/jdbc/MergeInstanceRepository.java index 928a09000..67e3d0c4e 100644 --- a/src/main/java/org/folio/search/service/reindex/jdbc/MergeInstanceRepository.java +++ b/src/main/java/org/folio/search/service/reindex/jdbc/MergeInstanceRepository.java @@ -3,7 +3,6 @@ import static org.folio.search.utils.JdbcUtils.getFullTableName; import java.sql.Timestamp; -import java.util.HashMap; import java.util.List; import java.util.Map; import lombok.extern.log4j.Log4j2; @@ -110,7 +109,7 @@ public SubResourceResult fetchByTimestamp(String tenant, Timestamp timestamp, St private RowMapper> instanceRowMapper() { return (rs, rowNum) -> { - Map instance = new HashMap<>(); + var instance = jsonConverter.fromJsonToMap(rs.getString("json")); instance.put("id", rs.getString("id")); instance.put("tenantId", rs.getString("tenant_id")); instance.put("shared", rs.getBoolean("shared")); @@ -118,9 +117,6 @@ private RowMapper> instanceRowMapper() { instance.put("isDeleted", rs.getBoolean("is_deleted")); instance.put(LAST_UPDATED_DATE_FIELD, rs.getTimestamp("last_updated_date")); - var jsonContent = jsonConverter.fromJsonToMap(rs.getString("json")); - instance.putAll(jsonContent); - return instance; }; } diff --git a/src/main/java/org/folio/search/service/reindex/jdbc/SubjectRepository.java b/src/main/java/org/folio/search/service/reindex/jdbc/SubjectRepository.java index e199aa9b7..53dddbc39 100644 --- a/src/main/java/org/folio/search/service/reindex/jdbc/SubjectRepository.java +++ b/src/main/java/org/folio/search/service/reindex/jdbc/SubjectRepository.java @@ -186,21 +186,7 @@ protected String getFetchBySql() { @Override protected RowMapper> rowToMapMapper() { - return (rs, rowNum) -> { - Map subject = new HashMap<>(); - subject.put("id", getId(rs)); - subject.put(SUBJECT_VALUE_FIELD, getValue(rs)); - subject.put(AUTHORITY_ID_FIELD, getAuthorityId(rs)); - subject.put("sourceId", getSourceId(rs)); - subject.put("typeId", getTypeId(rs)); - - var maps = jsonConverter.fromJsonToListOfMaps(getInstances(rs)).stream().filter(Objects::nonNull).toList(); - if (!maps.isEmpty()) { - subject.put(SUB_RESOURCE_INSTANCES_FIELD, maps); - } - - return subject; - }; + return (rs, rowNum) -> buildSubjectMap(rs); } @Override @@ -249,23 +235,28 @@ public void saveAll(ChildResourceEntityBatch entityBatch) { protected RowMapper> rowToMapMapper2() { return (rs, rowNum) -> { - Map subject = new HashMap<>(); - subject.put("id", getId(rs)); - subject.put(SUBJECT_VALUE_FIELD, getValue(rs)); - subject.put(AUTHORITY_ID_FIELD, getAuthorityId(rs)); - subject.put("sourceId", getSourceId(rs)); - subject.put("typeId", getTypeId(rs)); + var subject = buildSubjectMap(rs); subject.put(LAST_UPDATED_DATE_FIELD, rs.getTimestamp("last_updated_date")); - - var maps = jsonConverter.fromJsonToListOfMaps(getInstances(rs)).stream().filter(Objects::nonNull).toList(); - if (!maps.isEmpty()) { - subject.put(SUB_RESOURCE_INSTANCES_FIELD, maps); - } - return subject; }; } + private Map buildSubjectMap(ResultSet rs) throws SQLException { + Map subject = new HashMap<>(); + subject.put("id", getId(rs)); + subject.put(SUBJECT_VALUE_FIELD, getValue(rs)); + subject.put(AUTHORITY_ID_FIELD, getAuthorityId(rs)); + subject.put("sourceId", getSourceId(rs)); + subject.put("typeId", getTypeId(rs)); + + var maps = jsonConverter.fromJsonToListOfMaps(getInstances(rs)).stream().filter(Objects::nonNull).toList(); + if (!maps.isEmpty()) { + subject.put(SUB_RESOURCE_INSTANCES_FIELD, maps); + } + + return subject; + } + private String getId(ResultSet rs) throws SQLException { return rs.getString("id"); } diff --git a/src/main/java/org/folio/search/service/reindex/jdbc/UploadRangeRepository.java b/src/main/java/org/folio/search/service/reindex/jdbc/UploadRangeRepository.java index 10e7e46df..294c71f6e 100644 --- a/src/main/java/org/folio/search/service/reindex/jdbc/UploadRangeRepository.java +++ b/src/main/java/org/folio/search/service/reindex/jdbc/UploadRangeRepository.java @@ -13,13 +13,10 @@ import java.sql.Timestamp; import java.time.Instant; -import java.util.LinkedHashSet; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.UUID; import org.folio.search.configuration.properties.ReindexConfigurationProperties; -import org.folio.search.model.index.InstanceSubResource; import org.folio.search.model.reindex.UploadRangeEntity; import org.folio.search.model.types.ReindexEntityType; import org.folio.search.model.types.ReindexRangeStatus; @@ -28,7 +25,6 @@ import org.folio.spring.FolioExecutionContext; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.RowMapper; -import tools.jackson.core.type.TypeReference; public abstract class UploadRangeRepository extends ReindexJdbcRepository { @@ -47,7 +43,6 @@ ON CONFLICT (id) private static final String SELECT_UPLOAD_RANGE_BY_ENTITY_TYPE_SQL = "SELECT * FROM %s WHERE entity_type = ?;"; private static final String DELETE_UPLOAD_RANGE_SQL = "DELETE FROM %s WHERE entity_type = ?;"; - private static final TypeReference> VALUE_TYPE_REF = new TypeReference<>() { }; protected final ReindexConfigurationProperties reindexConfig; @@ -90,14 +85,6 @@ protected String rangeTable() { protected abstract RowMapper> rowToMapMapper(); - protected Set parseInstanceSubResources(String instancesJson) { - try { - return jsonConverter.fromJson(instancesJson, VALUE_TYPE_REF); - } catch (Exception e) { - throw new IllegalArgumentException(e); - } - } - protected List createRanges() { var uploadRangeLevel = reindexConfig.getUploadRangeLevel(); return RangeGenerator.createHexRanges(uploadRangeLevel); diff --git a/src/test/java/org/folio/indexing/IndexingInstanceCallNumberConsortiumIT.java b/src/test/java/org/folio/indexing/IndexingInstanceCallNumberConsortiumIT.java index ff6a41564..487ef50e0 100644 --- a/src/test/java/org/folio/indexing/IndexingInstanceCallNumberConsortiumIT.java +++ b/src/test/java/org/folio/indexing/IndexingInstanceCallNumberConsortiumIT.java @@ -36,6 +36,7 @@ class IndexingInstanceCallNumberConsortiumIT extends BaseIntegrationTest { private static final String INSTANCE_ID = randomId(); + private static final String LOCATION_ID = randomId(); private static final String INSTANCE_TITLE = "title"; private static final String CALL_NUMBER = "test"; @@ -94,6 +95,7 @@ private static void assertInstanceCallNumberTenantId(String expectedTenantId, bo private static void setUpTestData() { var holdings = new Holding().id(randomId()); var item = new Item().id(randomId()).holdingsRecordId(holdings.getId()) + .effectiveLocationId(LOCATION_ID) .effectiveCallNumberComponents(new ItemEffectiveCallNumberComponents().callNumber(CALL_NUMBER)); var instance = new Instance().id(INSTANCE_ID).title(INSTANCE_TITLE).source("FOLIO") .holdings(List.of(holdings)) diff --git a/src/test/java/org/folio/indexing/IndexingInstanceCallNumberIT.java b/src/test/java/org/folio/indexing/IndexingInstanceCallNumberIT.java index d9ec02caf..ba77a0873 100644 --- a/src/test/java/org/folio/indexing/IndexingInstanceCallNumberIT.java +++ b/src/test/java/org/folio/indexing/IndexingInstanceCallNumberIT.java @@ -34,6 +34,7 @@ class IndexingInstanceCallNumberIT extends BaseIntegrationTest { private static final String INSTANCE_ID_1 = randomId(); private static final String INSTANCE_ID_2 = randomId(); + private static final String LOCATION_ID = randomId(); @BeforeAll static void prepare() { @@ -77,7 +78,7 @@ void shouldIndexInstanceCallNumber_createNewDocument_onItemCreate() { // assert that the document contains the expected fields assertCallNumberDocFields(sourceAsMap); - // assert that the document contains the expected instances object with count 2 + // assert that the document contains the expected instances object with count 1 @SuppressWarnings("unchecked") var instances = (List>) sourceAsMap.get("instances"); assertThat(instances) @@ -137,7 +138,8 @@ private void assertCallNumberDocFields(Map sourceAsMap) { } private Item getItem(String itemId) { - return new Item().id(itemId).holdingsRecordId(randomId()).effectiveCallNumberComponents(callNumberComponents()); + return new Item().id(itemId).holdingsRecordId(randomId()).effectiveLocationId(LOCATION_ID) + .effectiveCallNumberComponents(callNumberComponents()); } private ItemEffectiveCallNumberComponents callNumberComponents() { diff --git a/src/test/java/org/folio/search/service/converter/preprocessor/extractor/impl/CallNumberResourceExtractorTest.java b/src/test/java/org/folio/search/service/converter/preprocessor/extractor/impl/CallNumberResourceExtractorTest.java index 3e7bc2634..28e46936f 100644 --- a/src/test/java/org/folio/search/service/converter/preprocessor/extractor/impl/CallNumberResourceExtractorTest.java +++ b/src/test/java/org/folio/search/service/converter/preprocessor/extractor/impl/CallNumberResourceExtractorTest.java @@ -17,17 +17,14 @@ import java.util.function.Supplier; import org.folio.search.domain.dto.TenantConfiguredFeature; import org.folio.search.service.FeatureConfigService; -import org.folio.search.service.consortium.ConsortiumTenantProvider; import org.folio.search.service.reindex.jdbc.CallNumberRepository; -import org.folio.search.utils.JsonConverter; import org.folio.spring.testing.type.UnitTest; import org.folio.support.base.ChildResourceExtractorTestBase; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import tools.jackson.databind.json.JsonMapper; @UnitTest @ExtendWith(MockitoExtension.class) @@ -37,9 +34,8 @@ class CallNumberResourceExtractorTest extends ChildResourceExtractorTestBase { private CallNumberRepository repository; @Mock private FeatureConfigService configService; - @Mock - private ConsortiumTenantProvider tenantProvider; + @InjectMocks private CallNumberResourceExtractor extractor; @Override @@ -47,11 +43,6 @@ protected int getExpectedEntitiesSize() { return 1; } - @BeforeEach - void setUp() { - extractor = new CallNumberResourceExtractor(repository, new JsonConverter(new JsonMapper()), configService); - } - @Test void persistChildren() { when(configService.isEnabled(TenantConfiguredFeature.BROWSE_CALL_NUMBERS)).thenReturn(true); @@ -72,7 +63,10 @@ private static Supplier> callNumberBodySupplier() { SUFFIX_FIELD, "suffix", PREFIX_FIELD, "prefix", TYPE_ID_FIELD, "type-id" - ) + ), + "id", "id", + "instanceId", "instance-id", + "effectiveLocationId", "location-id" )); } diff --git a/src/test/java/org/folio/search/service/reindex/ReindexOrchestrationServiceTest.java b/src/test/java/org/folio/search/service/reindex/ReindexOrchestrationServiceTest.java index cb77c138d..b032f73e5 100644 --- a/src/test/java/org/folio/search/service/reindex/ReindexOrchestrationServiceTest.java +++ b/src/test/java/org/folio/search/service/reindex/ReindexOrchestrationServiceTest.java @@ -10,7 +10,6 @@ import static org.mockito.Mockito.when; import java.util.List; -import java.util.Map; import java.util.UUID; import org.folio.search.domain.dto.FolioIndexOperationResponse; import org.folio.search.domain.dto.ResourceEvent; @@ -59,8 +58,8 @@ void process_shouldProcessSuccessfully() { .status(FolioIndexOperationResponse.StatusEnum.SUCCESS); when(uploadRangeIndexService.fetchRecordRange(event)).thenReturn(List.of(resourceEvent)); - when(documentConverter.convert(List.of(resourceEvent))).thenReturn(Map.of("key", List.of(SearchDocumentBody.of(null, - IndexingDataFormat.JSON, resourceEvent, IndexActionType.INDEX)))); + when(documentConverter.convertForReindex(List.of(resourceEvent))).thenReturn(List.of(SearchDocumentBody.of(null, + IndexingDataFormat.JSON, resourceEvent, IndexActionType.INDEX))); when(elasticRepository.indexResources(any())).thenReturn(folioIndexOperationResponse); // Act @@ -69,7 +68,7 @@ void process_shouldProcessSuccessfully() { // Assert assertTrue(result); verify(uploadRangeIndexService).fetchRecordRange(event); - verify(documentConverter).convert(List.of(resourceEvent)); + verify(documentConverter).convertForReindex(List.of(resourceEvent)); verify(elasticRepository).indexResources(any()); verify(reindexStatusService).addProcessedUploadRanges(event.getEntityType(), 1); } @@ -84,15 +83,15 @@ void process_shouldThrowReindexException_whenElasticSearchReportsError() { .errorMessage("Error occurred during indexing."); when(uploadRangeIndexService.fetchRecordRange(event)).thenReturn(List.of(resourceEvent)); - when(documentConverter.convert(List.of(resourceEvent))).thenReturn(Map.of("key", List.of(SearchDocumentBody.of(null, - IndexingDataFormat.JSON, resourceEvent, IndexActionType.INDEX)))); + when(documentConverter.convertForReindex(List.of(resourceEvent))).thenReturn(List.of(SearchDocumentBody.of(null, + IndexingDataFormat.JSON, resourceEvent, IndexActionType.INDEX))); when(elasticRepository.indexResources(any())).thenReturn(folioIndexOperationResponse); // Act & Assert assertThrows(ReindexException.class, () -> service.process(event)); verify(uploadRangeIndexService).fetchRecordRange(event); - verify(documentConverter).convert(List.of(resourceEvent)); + verify(documentConverter).convertForReindex(List.of(resourceEvent)); verify(elasticRepository).indexResources(any()); verify(reindexStatusService).updateReindexUploadFailed(event.getEntityType()); } @@ -120,13 +119,13 @@ void process_shouldThrowReindexException_whenExceptionOccursDuringDocumentConver var exceptionMessage = "Failed to convert documents"; when(uploadRangeIndexService.fetchRecordRange(event)).thenReturn(List.of(resourceEvent)); - when(documentConverter.convert(List.of(resourceEvent))).thenThrow(new RuntimeException(exceptionMessage)); + when(documentConverter.convertForReindex(List.of(resourceEvent))).thenThrow(new RuntimeException(exceptionMessage)); // Act & Assert assertThrows(ReindexException.class, () -> service.process(event)); verify(uploadRangeIndexService).fetchRecordRange(event); - verify(documentConverter).convert(List.of(resourceEvent)); + verify(documentConverter).convertForReindex(List.of(resourceEvent)); verify(reindexStatusService).updateReindexUploadFailed(event.getEntityType()); } @@ -138,15 +137,15 @@ void process_shouldThrowReindexException_whenExceptionOccursDuringIndexing() { var exceptionMessage = "Failed to index documents in Elasticsearch"; when(uploadRangeIndexService.fetchRecordRange(event)).thenReturn(List.of(resourceEvent)); - when(documentConverter.convert(List.of(resourceEvent))).thenReturn(Map.of("key", List.of(SearchDocumentBody.of(null, - IndexingDataFormat.JSON, resourceEvent, IndexActionType.INDEX)))); + when(documentConverter.convertForReindex(List.of(resourceEvent))).thenReturn(List.of(SearchDocumentBody.of(null, + IndexingDataFormat.JSON, resourceEvent, IndexActionType.INDEX))); when(elasticRepository.indexResources(any())).thenThrow(new RuntimeException(exceptionMessage)); // Act & Assert assertThrows(ReindexException.class, () -> service.process(event)); verify(uploadRangeIndexService).fetchRecordRange(event); - verify(documentConverter).convert(List.of(resourceEvent)); + verify(documentConverter).convertForReindex(List.of(resourceEvent)); verify(elasticRepository).indexResources(any()); verify(reindexStatusService).updateReindexUploadFailed(event.getEntityType()); } diff --git a/src/test/java/org/folio/search/service/reindex/jdbc/CallNumberRepositoryIT.java b/src/test/java/org/folio/search/service/reindex/jdbc/CallNumberRepositoryIT.java index a00d16a71..0c45707c1 100644 --- a/src/test/java/org/folio/search/service/reindex/jdbc/CallNumberRepositoryIT.java +++ b/src/test/java/org/folio/search/service/reindex/jdbc/CallNumberRepositoryIT.java @@ -108,13 +108,11 @@ void saveAll() { .extracting("callNumber", "instances") .contains( tuple("number1", - List.of(mapOf("count", null, "instanceContributors", null, - "instanceId", List.of("9f8febd1-e96c-46c4-a5f4-84a45cc499a2"), "instanceTitle", null, - "locationId", null, "resourceId", null, "shared", false, "tenantId", TENANT_ID, "typeId", null))), + List.of(mapOf("instanceId", List.of("9f8febd1-e96c-46c4-a5f4-84a45cc499a2"), + "locationId", null, "shared", false, "tenantId", TENANT_ID))), tuple("number2", - List.of(mapOf("count", null, "instanceContributors", null, - "instanceId", List.of("9f8febd1-e96c-46c4-a5f4-84a45cc499a2"), "instanceTitle", null, - "locationId", null, "resourceId", null, "shared", false, "tenantId", TENANT_ID, "typeId", null)))); + List.of(mapOf("instanceId", List.of("9f8febd1-e96c-46c4-a5f4-84a45cc499a2"), + "locationId", null, "shared", false, "tenantId", TENANT_ID)))); } @Test From 19bf949ed2595bab875f2cfd89afd867857f377d Mon Sep 17 00:00:00 2001 From: viacheslav_kolesnyk Date: Tue, 17 Mar 2026 15:21:59 +0100 Subject: [PATCH 4/9] Improve test coverage --- .../InstanceChildrenResourceServiceTest.java | 2 + ...ultiTenantSearchDocumentConverterTest.java | 88 +++++++++++++++++++ 2 files changed, 90 insertions(+) diff --git a/src/test/java/org/folio/search/service/InstanceChildrenResourceServiceTest.java b/src/test/java/org/folio/search/service/InstanceChildrenResourceServiceTest.java index 8afedaaa3..3c153cb6f 100644 --- a/src/test/java/org/folio/search/service/InstanceChildrenResourceServiceTest.java +++ b/src/test/java/org/folio/search/service/InstanceChildrenResourceServiceTest.java @@ -5,6 +5,7 @@ import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; import java.util.ArrayList; @@ -128,6 +129,7 @@ void persistChildrenOnReindex(boolean shared) { instanceResourceExtractors.forEach(resourceExtractor -> verify(resourceExtractor).persistChildren(TENANT_ID, shared, expectedEvents)); + verifyNoInteractions(callNumberRepository); } private ResourceEvent getResourceEvent(UUID id1, Map payload) { diff --git a/src/test/java/org/folio/search/service/converter/MultiTenantSearchDocumentConverterTest.java b/src/test/java/org/folio/search/service/converter/MultiTenantSearchDocumentConverterTest.java index 4d1709c44..25d29f9c3 100644 --- a/src/test/java/org/folio/search/service/converter/MultiTenantSearchDocumentConverterTest.java +++ b/src/test/java/org/folio/search/service/converter/MultiTenantSearchDocumentConverterTest.java @@ -154,6 +154,94 @@ void convert_positive_emptyList() { assertThat(actual).isEqualTo(emptyMap()); } + @Test + void convertForReindex_positive_null() { + var actual = multiTenantConverter.convertForReindex(null); + assertThat(actual).isEmpty(); + verifyNoInteractions(executionService, folioExecutionContext); + } + + @Test + void convertForReindex_positive_emptyList() { + var actual = multiTenantConverter.convertForReindex(emptyList()); + assertThat(actual).isEmpty(); + verifyNoInteractions(executionService, folioExecutionContext); + } + + @Test + void convertForReindex_positive_singleEvent() { + var event = resourceEvent(UNKNOWN, mapOf("id", RESOURCE_ID)); + var searchDocument = searchDocument(event, INDEX); + + when(resourceDescriptionService.find(UNKNOWN)).thenReturn(of(resourceDescription(UNKNOWN))); + when(searchDocumentConverter.convert(event)).thenReturn(of(searchDocument)); + + var actual = multiTenantConverter.convertForReindex(List.of(event)); + + assertThat(actual).containsExactly(searchDocument); + verifyNoInteractions(executionService, folioExecutionContext); + } + + @Test + void convertForReindex_positive_multipleEvents() { + var events = List.of( + resourceEvent(null, UNKNOWN, mapOf("id", randomId())).type(ResourceEventType.UPDATE), + resourceEvent(null, UNKNOWN, mapOf("id", randomId())).type(ResourceEventType.DELETE)); + + when(resourceDescriptionService.find(UNKNOWN)).thenReturn(of(resourceDescription(UNKNOWN))); + when(searchDocumentConverter.convert(events.get(0))).thenReturn(of(searchDocument(events.get(0), INDEX))); + when(searchDocumentConverter.convert(events.get(1))).thenReturn(of(searchDocument(events.get(1), DELETE))); + + var actual = multiTenantConverter.convertForReindex(events); + + assertThat(actual).containsExactly( + searchDocument(events.get(0), INDEX), + searchDocument(events.get(1), DELETE)); + verifyNoInteractions(executionService, folioExecutionContext); + } + + @Test + void convertForReindex_positive_eventThatIsNotConverted() { + var event = resourceEvent(UNKNOWN, mapOf("id", RESOURCE_ID)); + + when(resourceDescriptionService.find(UNKNOWN)).thenReturn(of(resourceDescription(UNKNOWN))); + when(searchDocumentConverter.convert(event)).thenReturn(Optional.empty()); + + var actual = multiTenantConverter.convertForReindex(List.of(event)); + + assertThat(actual).isEmpty(); + verifyNoInteractions(executionService, folioExecutionContext); + } + + @Test + void convertForReindex_positive_eventWithoutId() { + var event = resourceEvent(null, UNKNOWN, mapOf("id", RESOURCE_ID)).tenant(TENANT_ID); + + when(resourceDescriptionService.find(UNKNOWN)).thenReturn(of(resourceDescription(UNKNOWN))); + when(searchDocumentConverter.convert(any(ResourceEvent.class))).thenReturn(of(searchDocument(event, INDEX))); + + var actual = multiTenantConverter.convertForReindex(List.of(event)); + + assertThat(actual).hasSize(1); + verifyNoInteractions(executionService, folioExecutionContext); + } + + @Test + void convertForReindex_positive_eventWithCustomPreProcessor() { + var event = resourceEvent(UNKNOWN, mapOf("id", RESOURCE_ID)); + var searchDocument = searchDocument(event, INDEX); + + when(resourceDescriptionService.find(UNKNOWN)).thenReturn(of(resourceDescriptionWithPreProcessor())); + when(eventPreProcessorBeans.get(CUSTOM_PRE_PROCESSOR)).thenReturn(customEventPreProcessor); + when(customEventPreProcessor.preProcess(event)).thenReturn(List.of(event)); + when(searchDocumentConverter.convert(event)).thenReturn(of(searchDocument)); + + var actual = multiTenantConverter.convertForReindex(List.of(event)); + + assertThat(actual).containsExactly(searchDocument); + verifyNoInteractions(executionService, folioExecutionContext); + } + @SneakyThrows private static SearchDocumentBody searchDocument(ResourceEvent event, IndexActionType type) { return SearchDocumentBody.of(type == INDEX ? new BytesArray(SMILE_MAPPER.writeValueAsBytes(event.getNew())) : null, From 452dd72a419735f6e07ea9cfc210ea75b8fc3813 Mon Sep 17 00:00:00 2001 From: viacheslav_kolesnyk Date: Wed, 18 Mar 2026 15:46:53 +0100 Subject: [PATCH 5/9] Use ResultSet.getCharacterStream for "instances" on sub-resources and Reader for objectMapper to avoid String intermediary when reading data from postgres into a list of maps --- .../reindex/jdbc/CallNumberRepository.java | 8 +++++--- .../jdbc/ClassificationRepository.java | 7 ++++--- .../reindex/jdbc/ContributorRepository.java | 7 ++++--- .../reindex/jdbc/SubjectRepository.java | 7 ++++--- .../org/folio/search/utils/JsonConverter.java | 19 ++++++++++++++----- 5 files changed, 31 insertions(+), 17 deletions(-) diff --git a/src/main/java/org/folio/search/service/reindex/jdbc/CallNumberRepository.java b/src/main/java/org/folio/search/service/reindex/jdbc/CallNumberRepository.java index a6bb6d8dc..1f96dccab 100644 --- a/src/main/java/org/folio/search/service/reindex/jdbc/CallNumberRepository.java +++ b/src/main/java/org/folio/search/service/reindex/jdbc/CallNumberRepository.java @@ -14,6 +14,7 @@ import static org.folio.search.utils.SearchUtils.ID_FIELD; import static org.folio.search.utils.SearchUtils.SUB_RESOURCE_INSTANCES_FIELD; +import java.io.Reader; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Timestamp; @@ -274,7 +275,8 @@ private Map getCallNumberMap(ResultSet rs) throws SQLException { callNumberMap.put(CALL_NUMBER_PREFIX_FIELD, getCallNumberPrefix(rs)); callNumberMap.put(CALL_NUMBER_SUFFIX_FIELD, callNumberSuffix); callNumberMap.put(CALL_NUMBER_TYPE_ID_FIELD, getCallNumberTypeId(rs)); - var maps = jsonConverter.fromJsonToListOfMaps(getInstances(rs)).stream().filter(Objects::nonNull).toList(); + var instancesReader = getInstancesReader(rs); + var maps = jsonConverter.fromJsonToListOfMaps(instancesReader).stream().filter(Objects::nonNull).toList(); if (!maps.isEmpty()) { callNumberMap.put(SUB_RESOURCE_INSTANCES_FIELD, maps); } @@ -337,8 +339,8 @@ private String getCallNumberTypeId(ResultSet rs) throws SQLException { return rs.getString("call_number_type_id"); } - private String getInstances(ResultSet rs) throws SQLException { - return rs.getString("instances"); + private Reader getInstancesReader(ResultSet rs) throws SQLException { + return rs.getCharacterStream("instances"); } private String getCallNumber(Map callNumberComponents) { diff --git a/src/main/java/org/folio/search/service/reindex/jdbc/ClassificationRepository.java b/src/main/java/org/folio/search/service/reindex/jdbc/ClassificationRepository.java index 0989b268e..02b742a9e 100644 --- a/src/main/java/org/folio/search/service/reindex/jdbc/ClassificationRepository.java +++ b/src/main/java/org/folio/search/service/reindex/jdbc/ClassificationRepository.java @@ -6,6 +6,7 @@ import static org.folio.search.utils.SearchUtils.CLASSIFICATION_TYPE_FIELD; import static org.folio.search.utils.SearchUtils.SUB_RESOURCE_INSTANCES_FIELD; +import java.io.Reader; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Timestamp; @@ -238,7 +239,7 @@ private Map buildClassificationMap(ResultSet rs) throws SQLExcep classification.put(CLASSIFICATION_NUMBER_ENTITY_FIELD, getNumber(rs)); classification.put("typeId", getTypeId(rs)); - var maps = jsonConverter.fromJsonToListOfMaps(getInstances(rs)).stream().filter(Objects::nonNull).toList(); + var maps = jsonConverter.fromJsonToListOfMaps(getInstancesReader(rs)).stream().filter(Objects::nonNull).toList(); if (!maps.isEmpty()) { classification.put(SUB_RESOURCE_INSTANCES_FIELD, maps); } @@ -258,7 +259,7 @@ private String getNumber(ResultSet rs) throws SQLException { return rs.getString(CLASSIFICATION_NUMBER_ENTITY_FIELD); } - private String getInstances(ResultSet rs) throws SQLException { - return rs.getString(SUB_RESOURCE_INSTANCES_FIELD); + private Reader getInstancesReader(ResultSet rs) throws SQLException { + return rs.getCharacterStream(SUB_RESOURCE_INSTANCES_FIELD); } } diff --git a/src/main/java/org/folio/search/service/reindex/jdbc/ContributorRepository.java b/src/main/java/org/folio/search/service/reindex/jdbc/ContributorRepository.java index 9e3ee79df..6dee871ee 100644 --- a/src/main/java/org/folio/search/service/reindex/jdbc/ContributorRepository.java +++ b/src/main/java/org/folio/search/service/reindex/jdbc/ContributorRepository.java @@ -5,6 +5,7 @@ import static org.folio.search.utils.SearchUtils.CONTRIBUTOR_TYPE_FIELD; import static org.folio.search.utils.SearchUtils.SUB_RESOURCE_INSTANCES_FIELD; +import java.io.Reader; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Timestamp; @@ -245,7 +246,7 @@ private Map buildContributorMap(ResultSet rs) throws SQLExceptio contributor.put("contributorNameTypeId", getNameTypeId(rs)); contributor.put(AUTHORITY_ID_FIELD, getAuthorityId(rs)); - var maps = jsonConverter.fromJsonToListOfMaps(getInstances(rs)).stream().filter(Objects::nonNull).toList(); + var maps = jsonConverter.fromJsonToListOfMaps(getInstancesReader(rs)).stream().filter(Objects::nonNull).toList(); if (!maps.isEmpty()) { contributor.put(SUB_RESOURCE_INSTANCES_FIELD, maps); } @@ -269,7 +270,7 @@ private String getAuthorityId(ResultSet rs) throws SQLException { return rs.getString("authority_id"); } - private String getInstances(ResultSet rs) throws SQLException { - return rs.getString(SUB_RESOURCE_INSTANCES_FIELD); + private Reader getInstancesReader(ResultSet rs) throws SQLException { + return rs.getCharacterStream(SUB_RESOURCE_INSTANCES_FIELD); } } diff --git a/src/main/java/org/folio/search/service/reindex/jdbc/SubjectRepository.java b/src/main/java/org/folio/search/service/reindex/jdbc/SubjectRepository.java index 53dddbc39..598422d18 100644 --- a/src/main/java/org/folio/search/service/reindex/jdbc/SubjectRepository.java +++ b/src/main/java/org/folio/search/service/reindex/jdbc/SubjectRepository.java @@ -7,6 +7,7 @@ import static org.folio.search.utils.SearchUtils.SUBJECT_VALUE_FIELD; import static org.folio.search.utils.SearchUtils.SUB_RESOURCE_INSTANCES_FIELD; +import java.io.Reader; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Timestamp; @@ -249,7 +250,7 @@ private Map buildSubjectMap(ResultSet rs) throws SQLException { subject.put("sourceId", getSourceId(rs)); subject.put("typeId", getTypeId(rs)); - var maps = jsonConverter.fromJsonToListOfMaps(getInstances(rs)).stream().filter(Objects::nonNull).toList(); + var maps = jsonConverter.fromJsonToListOfMaps(getInstancesReader(rs)).stream().filter(Objects::nonNull).toList(); if (!maps.isEmpty()) { subject.put(SUB_RESOURCE_INSTANCES_FIELD, maps); } @@ -277,7 +278,7 @@ private String getTypeId(ResultSet rs) throws SQLException { return rs.getString("type_id"); } - private String getInstances(ResultSet rs) throws SQLException { - return rs.getString(SUB_RESOURCE_INSTANCES_FIELD); + private Reader getInstancesReader(ResultSet rs) throws SQLException { + return rs.getCharacterStream(SUB_RESOURCE_INSTANCES_FIELD); } } diff --git a/src/main/java/org/folio/search/utils/JsonConverter.java b/src/main/java/org/folio/search/utils/JsonConverter.java index 0ba324bd1..8608ce8bc 100644 --- a/src/main/java/org/folio/search/utils/JsonConverter.java +++ b/src/main/java/org/folio/search/utils/JsonConverter.java @@ -1,6 +1,7 @@ package org.folio.search.utils; import java.io.InputStream; +import java.io.Reader; import java.util.List; import java.util.Map; import lombok.RequiredArgsConstructor; @@ -80,13 +81,21 @@ public Map fromJsonToMap(String value) { } /** - * Converts {@link String} value to the {@link Map} . + * Converts {@link Reader} value to the list of maps using streaming deserialization. + * This avoids materializing the full JSON string in memory, which is important for large payloads. * - * @param value object value to convert - * @return converted value + * @param reader reader providing JSON content + * @return converted list of maps */ - public List> fromJsonToListOfMaps(String value) { - return fromJson(value, LIST_OF_MAP_TYPE_REF); + public List> fromJsonToListOfMaps(Reader reader) { + if (reader == null) { + return List.of(); + } + try { + return objectMapper.readValue(reader, LIST_OF_MAP_TYPE_REF); + } catch (JacksonException e) { + throw new SerializationException("Failed to deserialize value from reader", e); + } } /** From fcba590194e3c8c2d85e7da79697464a1c10fd19 Mon Sep 17 00:00:00 2001 From: viacheslav_kolesnyk Date: Wed, 18 Mar 2026 16:18:46 +0100 Subject: [PATCH 6/9] Prevent stale lock release on reindex failure --- .../service/reindex/ReindexStatusService.java | 13 ++++-- .../ScheduledInstanceSubResourcesService.java | 11 ++--- .../reindex/ReindexStatusServiceTest.java | 40 ++++++++++++++++--- ...eduledInstanceSubResourcesServiceTest.java | 12 +++--- 4 files changed, 55 insertions(+), 21 deletions(-) diff --git a/src/main/java/org/folio/search/service/reindex/ReindexStatusService.java b/src/main/java/org/folio/search/service/reindex/ReindexStatusService.java index db5dc40f3..f85bd5e97 100644 --- a/src/main/java/org/folio/search/service/reindex/ReindexStatusService.java +++ b/src/main/java/org/folio/search/service/reindex/ReindexStatusService.java @@ -109,14 +109,19 @@ public boolean isMergeCompleted() { } /** - * Checks if any reindex operation is currently in progress (merge or upload). + * Checks if a reindex operation for a specific entity type is currently in progress or has failed. * - * @return true if any entity type has a status of MERGE_IN_PROGRESS or UPLOAD_IN_PROGRESS + * @param entityType the entity type to check + * @return true if the given entity type has a status of MERGE_IN_PROGRESS, UPLOAD_IN_PROGRESS, + * MERGE_FAILED, or UPLOAD_FAILED */ - public boolean isReindexInProgress() { + public boolean isReindexInProgressOrFailed(ReindexEntityType entityType) { return statusRepository.getReindexStatuses().stream() + .filter(status -> status.getEntityType() == entityType) .anyMatch(status -> status.getStatus() == ReindexStatus.MERGE_IN_PROGRESS - || status.getStatus() == ReindexStatus.UPLOAD_IN_PROGRESS); + || status.getStatus() == ReindexStatus.UPLOAD_IN_PROGRESS + || status.getStatus() == ReindexStatus.MERGE_FAILED + || status.getStatus() == ReindexStatus.UPLOAD_FAILED); } private List constructNewStatusRecords(List entityTypes, diff --git a/src/main/java/org/folio/search/service/scheduled/ScheduledInstanceSubResourcesService.java b/src/main/java/org/folio/search/service/scheduled/ScheduledInstanceSubResourcesService.java index f2f1b4e98..85bac03bc 100644 --- a/src/main/java/org/folio/search/service/scheduled/ScheduledInstanceSubResourcesService.java +++ b/src/main/java/org/folio/search/service/scheduled/ScheduledInstanceSubResourcesService.java @@ -119,8 +119,9 @@ private void processEntityTypeWithLock(ReindexEntityType entityType, String tena } private void handleLockAcquisitionFailure(ReindexEntityType entityType, String tenant) { - if (isReindexInProgress()) { - log.info("persistChildren::Skipping stale lock check for entity type {} in tenant {} - reindex is in progress", + if (isReindexInProgressOrFailed(entityType)) { + log.info( + "persistChildren::Skipping stale lock check for entity type {} in tenant {} - reindex is in progress or failed", entityType, tenant); return; } @@ -132,11 +133,11 @@ private void handleLockAcquisitionFailure(ReindexEntityType entityType, String t } } - private boolean isReindexInProgress() { + private boolean isReindexInProgressOrFailed(ReindexEntityType entityType) { try { - return reindexStatusService.isReindexInProgress(); + return reindexStatusService.isReindexInProgressOrFailed(entityType); } catch (Exception e) { - log.warn("persistChildren::Failed to check reindex status, assuming no reindex in progress", e); + log.warn("persistChildren::Failed to check reindex status, assuming no reindex in progress or failed", e); return false; } } diff --git a/src/test/java/org/folio/search/service/reindex/ReindexStatusServiceTest.java b/src/test/java/org/folio/search/service/reindex/ReindexStatusServiceTest.java index 32655e00b..ef6119bcb 100644 --- a/src/test/java/org/folio/search/service/reindex/ReindexStatusServiceTest.java +++ b/src/test/java/org/folio/search/service/reindex/ReindexStatusServiceTest.java @@ -181,42 +181,70 @@ void updateReindexMergeInProgress() { } @Test - void isReindexInProgress_trueWhenMerge() { + void isReindexInProgressOrFailed_trueWhenMergeInProgress() { // given when(statusRepository.getReindexStatuses()).thenReturn(List.of( new ReindexStatusEntity(ReindexEntityType.INSTANCE, ReindexStatus.MERGE_IN_PROGRESS), new ReindexStatusEntity(ReindexEntityType.HOLDINGS, ReindexStatus.MERGE_COMPLETED))); // act - var actual = service.isReindexInProgress(); + var actual = service.isReindexInProgressOrFailed(ReindexEntityType.INSTANCE); // assert assertThat(actual).isTrue(); } @Test - void isReindexInProgress_trueWhenUpload() { + void isReindexInProgressOrFailed_trueWhenUploadInProgress() { // given when(statusRepository.getReindexStatuses()).thenReturn(List.of( new ReindexStatusEntity(ReindexEntityType.INSTANCE, ReindexStatus.UPLOAD_IN_PROGRESS), new ReindexStatusEntity(ReindexEntityType.HOLDINGS, ReindexStatus.MERGE_COMPLETED))); // act - var actual = service.isReindexInProgress(); + var actual = service.isReindexInProgressOrFailed(ReindexEntityType.INSTANCE); // assert assertThat(actual).isTrue(); } @Test - void isReindexInProgress_false() { + void isReindexInProgressOrFailed_trueWhenMergeFailed() { + // given + when(statusRepository.getReindexStatuses()).thenReturn(List.of( + new ReindexStatusEntity(ReindexEntityType.INSTANCE, ReindexStatus.MERGE_FAILED), + new ReindexStatusEntity(ReindexEntityType.HOLDINGS, ReindexStatus.MERGE_COMPLETED))); + + // act + var actual = service.isReindexInProgressOrFailed(ReindexEntityType.INSTANCE); + + // assert + assertThat(actual).isTrue(); + } + + @Test + void isReindexInProgressOrFailed_trueWhenUploadFailed() { + // given + when(statusRepository.getReindexStatuses()).thenReturn(List.of( + new ReindexStatusEntity(ReindexEntityType.INSTANCE, ReindexStatus.UPLOAD_FAILED), + new ReindexStatusEntity(ReindexEntityType.HOLDINGS, ReindexStatus.MERGE_COMPLETED))); + + // act + var actual = service.isReindexInProgressOrFailed(ReindexEntityType.INSTANCE); + + // assert + assertThat(actual).isTrue(); + } + + @Test + void isReindexInProgressOrFailed_falseWhenCompleted() { // given when(statusRepository.getReindexStatuses()).thenReturn(List.of( new ReindexStatusEntity(ReindexEntityType.INSTANCE, ReindexStatus.UPLOAD_COMPLETED), new ReindexStatusEntity(ReindexEntityType.HOLDINGS, ReindexStatus.MERGE_COMPLETED))); // act - var actual = service.isReindexInProgress(); + var actual = service.isReindexInProgressOrFailed(ReindexEntityType.INSTANCE); // assert assertThat(actual).isFalse(); diff --git a/src/test/java/org/folio/search/service/scheduled/ScheduledInstanceSubResourcesServiceTest.java b/src/test/java/org/folio/search/service/scheduled/ScheduledInstanceSubResourcesServiceTest.java index a36c90fcb..ae492df97 100644 --- a/src/test/java/org/folio/search/service/scheduled/ScheduledInstanceSubResourcesServiceTest.java +++ b/src/test/java/org/folio/search/service/scheduled/ScheduledInstanceSubResourcesServiceTest.java @@ -182,7 +182,7 @@ void persistChildren_ShouldReleaseStaleLockWhenLockAcquisitionFails() { .when(executionService).executeSystemUserScoped(anyString(), any()); when(tenantRepository.fetchDataTenantIds()).thenReturn(List.of(TENANT_ID)); when(subResourcesLockRepository.lockSubResource(any(), eq(TENANT_ID))).thenReturn(Optional.empty()); - when(reindexStatusService.isReindexInProgress()).thenReturn(false); + when(reindexStatusService.isReindexInProgressOrFailed(any())).thenReturn(false); when(subResourcesLockRepository.checkAndReleaseStaleLock(any(), eq(TENANT_ID), anyLong())).thenReturn(true); // Act @@ -190,7 +190,7 @@ void persistChildren_ShouldReleaseStaleLockWhenLockAcquisitionFails() { // Assert verify(subResourcesLockRepository, times(3)).lockSubResource(any(), eq(TENANT_ID)); - verify(reindexStatusService, times(3)).isReindexInProgress(); + verify(reindexStatusService, times(3)).isReindexInProgressOrFailed(any()); verify(subResourcesLockRepository, times(3)).checkAndReleaseStaleLock(any(), eq(TENANT_ID), anyLong()); verify(subResourcesLockRepository, never()).unlockSubResource(any(), any(), any()); verify(subjectRepository, never()).fetchByTimestamp(anyString(), any(), anyInt()); @@ -204,7 +204,7 @@ void persistChildren_ShouldNotProcessWhenLockFailsAndNoStaleLock() { .when(executionService).executeSystemUserScoped(anyString(), any()); when(tenantRepository.fetchDataTenantIds()).thenReturn(List.of(TENANT_ID)); when(subResourcesLockRepository.lockSubResource(any(), eq(TENANT_ID))).thenReturn(Optional.empty()); - when(reindexStatusService.isReindexInProgress()).thenReturn(false); + when(reindexStatusService.isReindexInProgressOrFailed(any())).thenReturn(false); when(subResourcesLockRepository.checkAndReleaseStaleLock(any(), eq(TENANT_ID), anyLong())).thenReturn(false); // Act @@ -212,7 +212,7 @@ void persistChildren_ShouldNotProcessWhenLockFailsAndNoStaleLock() { // Assert verify(subResourcesLockRepository, times(3)).lockSubResource(any(), eq(TENANT_ID)); - verify(reindexStatusService, times(3)).isReindexInProgress(); + verify(reindexStatusService, times(3)).isReindexInProgressOrFailed(any()); verify(subResourcesLockRepository, times(3)).checkAndReleaseStaleLock(any(), eq(TENANT_ID), anyLong()); verify(subResourcesLockRepository, never()).unlockSubResource(any(), any(), any()); verify(subjectRepository, never()).fetchByTimestamp(anyString(), any(), anyInt()); @@ -226,14 +226,14 @@ void persistChildren_ShouldSkipStaleLockCheckWhenReindexInProgress() { .when(executionService).executeSystemUserScoped(anyString(), any()); when(tenantRepository.fetchDataTenantIds()).thenReturn(List.of(TENANT_ID)); when(subResourcesLockRepository.lockSubResource(any(), eq(TENANT_ID))).thenReturn(Optional.empty()); - when(reindexStatusService.isReindexInProgress()).thenReturn(true); + when(reindexStatusService.isReindexInProgressOrFailed(any())).thenReturn(true); // Act service.persistChildren(); // Assert verify(subResourcesLockRepository, times(3)).lockSubResource(any(), eq(TENANT_ID)); - verify(reindexStatusService, times(3)).isReindexInProgress(); + verify(reindexStatusService, times(3)).isReindexInProgressOrFailed(any()); verify(subResourcesLockRepository, never()).checkAndReleaseStaleLock(any(), any(), anyLong()); verify(subResourcesLockRepository, never()).unlockSubResource(any(), any(), any()); verify(subjectRepository, never()).fetchByTimestamp(anyString(), any(), anyInt()); From f08a20b58e96e4c4d428321471b0c998bd4c10a8 Mon Sep 17 00:00:00 2001 From: viacheslav_kolesnyk Date: Wed, 18 Mar 2026 17:31:07 +0100 Subject: [PATCH 7/9] Add log to display bulk indexing request size --- .../folio/search/repository/AbstractResourceRepository.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/org/folio/search/repository/AbstractResourceRepository.java b/src/main/java/org/folio/search/repository/AbstractResourceRepository.java index 9d1d5ee00..ee00f6397 100644 --- a/src/main/java/org/folio/search/repository/AbstractResourceRepository.java +++ b/src/main/java/org/folio/search/repository/AbstractResourceRepository.java @@ -71,6 +71,9 @@ public void setElasticsearchClient(RestHighLevelClient elasticsearchClient) { protected BulkResponse executeBulkRequest(BulkRequest bulkRequest) { var indicesString = bulkRequest.requests().stream().map(DocWriteRequest::index).collect(joining(",")); + //todo: remove + log.info("executeBulkRequest:: Sending bulk request [actions: {}, estimatedSize: {} bytes]", + bulkRequest.numberOfActions(), bulkRequest.estimatedSizeInBytes()); return performExceptionalOperation(() -> elasticsearchClient.bulk(bulkRequest, DEFAULT), indicesString, "bulkApi"); } From 8dec07405206ca639131f87f0d8c5e2c5067de54 Mon Sep 17 00:00:00 2001 From: viacheslav_kolesnyk Date: Thu, 19 Mar 2026 12:37:55 +0100 Subject: [PATCH 8/9] Remove debug log --- .../folio/search/repository/AbstractResourceRepository.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/main/java/org/folio/search/repository/AbstractResourceRepository.java b/src/main/java/org/folio/search/repository/AbstractResourceRepository.java index ee00f6397..53455bbd3 100644 --- a/src/main/java/org/folio/search/repository/AbstractResourceRepository.java +++ b/src/main/java/org/folio/search/repository/AbstractResourceRepository.java @@ -10,7 +10,6 @@ import static org.opensearch.index.query.QueryBuilders.termQuery; import java.util.List; -import lombok.extern.log4j.Log4j2; import org.apache.commons.collections4.CollectionUtils; import org.folio.search.domain.dto.FolioIndexOperationResponse; import org.folio.search.model.index.SearchDocumentBody; @@ -26,7 +25,6 @@ import org.opensearch.index.reindex.DeleteByQueryRequest; import org.springframework.beans.factory.annotation.Autowired; -@Log4j2 public abstract class AbstractResourceRepository implements ResourceRepository { protected RestHighLevelClient elasticsearchClient; @@ -71,9 +69,6 @@ public void setElasticsearchClient(RestHighLevelClient elasticsearchClient) { protected BulkResponse executeBulkRequest(BulkRequest bulkRequest) { var indicesString = bulkRequest.requests().stream().map(DocWriteRequest::index).collect(joining(",")); - //todo: remove - log.info("executeBulkRequest:: Sending bulk request [actions: {}, estimatedSize: {} bytes]", - bulkRequest.numberOfActions(), bulkRequest.estimatedSizeInBytes()); return performExceptionalOperation(() -> elasticsearchClient.bulk(bulkRequest, DEFAULT), indicesString, "bulkApi"); } From 1d6c24fbe5bc61aaf6eaa190d18f313fa7e291e9 Mon Sep 17 00:00:00 2001 From: viacheslav_kolesnyk Date: Thu, 19 Mar 2026 14:59:33 +0100 Subject: [PATCH 9/9] Remove entity specific reindexing check on scheduled job, check if any reindex is running for tenant --- .../search/service/reindex/ReindexStatusService.java | 8 +++----- .../ScheduledInstanceSubResourcesService.java | 6 +++--- .../service/reindex/ReindexStatusServiceTest.java | 10 +++++----- .../ScheduledInstanceSubResourcesServiceTest.java | 12 ++++++------ 4 files changed, 17 insertions(+), 19 deletions(-) diff --git a/src/main/java/org/folio/search/service/reindex/ReindexStatusService.java b/src/main/java/org/folio/search/service/reindex/ReindexStatusService.java index f85bd5e97..b5bb76b73 100644 --- a/src/main/java/org/folio/search/service/reindex/ReindexStatusService.java +++ b/src/main/java/org/folio/search/service/reindex/ReindexStatusService.java @@ -109,15 +109,13 @@ public boolean isMergeCompleted() { } /** - * Checks if a reindex operation for a specific entity type is currently in progress or has failed. + * Checks if any reindex operation is currently in progress or has failed. * - * @param entityType the entity type to check - * @return true if the given entity type has a status of MERGE_IN_PROGRESS, UPLOAD_IN_PROGRESS, + * @return true if any entity type has a status of MERGE_IN_PROGRESS, UPLOAD_IN_PROGRESS, * MERGE_FAILED, or UPLOAD_FAILED */ - public boolean isReindexInProgressOrFailed(ReindexEntityType entityType) { + public boolean isReindexInProgressOrFailed() { return statusRepository.getReindexStatuses().stream() - .filter(status -> status.getEntityType() == entityType) .anyMatch(status -> status.getStatus() == ReindexStatus.MERGE_IN_PROGRESS || status.getStatus() == ReindexStatus.UPLOAD_IN_PROGRESS || status.getStatus() == ReindexStatus.MERGE_FAILED diff --git a/src/main/java/org/folio/search/service/scheduled/ScheduledInstanceSubResourcesService.java b/src/main/java/org/folio/search/service/scheduled/ScheduledInstanceSubResourcesService.java index 85bac03bc..4504de908 100644 --- a/src/main/java/org/folio/search/service/scheduled/ScheduledInstanceSubResourcesService.java +++ b/src/main/java/org/folio/search/service/scheduled/ScheduledInstanceSubResourcesService.java @@ -119,7 +119,7 @@ private void processEntityTypeWithLock(ReindexEntityType entityType, String tena } private void handleLockAcquisitionFailure(ReindexEntityType entityType, String tenant) { - if (isReindexInProgressOrFailed(entityType)) { + if (isReindexInProgressOrFailed()) { log.info( "persistChildren::Skipping stale lock check for entity type {} in tenant {} - reindex is in progress or failed", entityType, tenant); @@ -133,9 +133,9 @@ private void handleLockAcquisitionFailure(ReindexEntityType entityType, String t } } - private boolean isReindexInProgressOrFailed(ReindexEntityType entityType) { + private boolean isReindexInProgressOrFailed() { try { - return reindexStatusService.isReindexInProgressOrFailed(entityType); + return reindexStatusService.isReindexInProgressOrFailed(); } catch (Exception e) { log.warn("persistChildren::Failed to check reindex status, assuming no reindex in progress or failed", e); return false; diff --git a/src/test/java/org/folio/search/service/reindex/ReindexStatusServiceTest.java b/src/test/java/org/folio/search/service/reindex/ReindexStatusServiceTest.java index ef6119bcb..760c0e00e 100644 --- a/src/test/java/org/folio/search/service/reindex/ReindexStatusServiceTest.java +++ b/src/test/java/org/folio/search/service/reindex/ReindexStatusServiceTest.java @@ -188,7 +188,7 @@ void isReindexInProgressOrFailed_trueWhenMergeInProgress() { new ReindexStatusEntity(ReindexEntityType.HOLDINGS, ReindexStatus.MERGE_COMPLETED))); // act - var actual = service.isReindexInProgressOrFailed(ReindexEntityType.INSTANCE); + var actual = service.isReindexInProgressOrFailed(); // assert assertThat(actual).isTrue(); @@ -202,7 +202,7 @@ void isReindexInProgressOrFailed_trueWhenUploadInProgress() { new ReindexStatusEntity(ReindexEntityType.HOLDINGS, ReindexStatus.MERGE_COMPLETED))); // act - var actual = service.isReindexInProgressOrFailed(ReindexEntityType.INSTANCE); + var actual = service.isReindexInProgressOrFailed(); // assert assertThat(actual).isTrue(); @@ -216,7 +216,7 @@ void isReindexInProgressOrFailed_trueWhenMergeFailed() { new ReindexStatusEntity(ReindexEntityType.HOLDINGS, ReindexStatus.MERGE_COMPLETED))); // act - var actual = service.isReindexInProgressOrFailed(ReindexEntityType.INSTANCE); + var actual = service.isReindexInProgressOrFailed(); // assert assertThat(actual).isTrue(); @@ -230,7 +230,7 @@ void isReindexInProgressOrFailed_trueWhenUploadFailed() { new ReindexStatusEntity(ReindexEntityType.HOLDINGS, ReindexStatus.MERGE_COMPLETED))); // act - var actual = service.isReindexInProgressOrFailed(ReindexEntityType.INSTANCE); + var actual = service.isReindexInProgressOrFailed(); // assert assertThat(actual).isTrue(); @@ -244,7 +244,7 @@ void isReindexInProgressOrFailed_falseWhenCompleted() { new ReindexStatusEntity(ReindexEntityType.HOLDINGS, ReindexStatus.MERGE_COMPLETED))); // act - var actual = service.isReindexInProgressOrFailed(ReindexEntityType.INSTANCE); + var actual = service.isReindexInProgressOrFailed(); // assert assertThat(actual).isFalse(); diff --git a/src/test/java/org/folio/search/service/scheduled/ScheduledInstanceSubResourcesServiceTest.java b/src/test/java/org/folio/search/service/scheduled/ScheduledInstanceSubResourcesServiceTest.java index ae492df97..ce3f21485 100644 --- a/src/test/java/org/folio/search/service/scheduled/ScheduledInstanceSubResourcesServiceTest.java +++ b/src/test/java/org/folio/search/service/scheduled/ScheduledInstanceSubResourcesServiceTest.java @@ -182,7 +182,7 @@ void persistChildren_ShouldReleaseStaleLockWhenLockAcquisitionFails() { .when(executionService).executeSystemUserScoped(anyString(), any()); when(tenantRepository.fetchDataTenantIds()).thenReturn(List.of(TENANT_ID)); when(subResourcesLockRepository.lockSubResource(any(), eq(TENANT_ID))).thenReturn(Optional.empty()); - when(reindexStatusService.isReindexInProgressOrFailed(any())).thenReturn(false); + when(reindexStatusService.isReindexInProgressOrFailed()).thenReturn(false); when(subResourcesLockRepository.checkAndReleaseStaleLock(any(), eq(TENANT_ID), anyLong())).thenReturn(true); // Act @@ -190,7 +190,7 @@ void persistChildren_ShouldReleaseStaleLockWhenLockAcquisitionFails() { // Assert verify(subResourcesLockRepository, times(3)).lockSubResource(any(), eq(TENANT_ID)); - verify(reindexStatusService, times(3)).isReindexInProgressOrFailed(any()); + verify(reindexStatusService, times(3)).isReindexInProgressOrFailed(); verify(subResourcesLockRepository, times(3)).checkAndReleaseStaleLock(any(), eq(TENANT_ID), anyLong()); verify(subResourcesLockRepository, never()).unlockSubResource(any(), any(), any()); verify(subjectRepository, never()).fetchByTimestamp(anyString(), any(), anyInt()); @@ -204,7 +204,7 @@ void persistChildren_ShouldNotProcessWhenLockFailsAndNoStaleLock() { .when(executionService).executeSystemUserScoped(anyString(), any()); when(tenantRepository.fetchDataTenantIds()).thenReturn(List.of(TENANT_ID)); when(subResourcesLockRepository.lockSubResource(any(), eq(TENANT_ID))).thenReturn(Optional.empty()); - when(reindexStatusService.isReindexInProgressOrFailed(any())).thenReturn(false); + when(reindexStatusService.isReindexInProgressOrFailed()).thenReturn(false); when(subResourcesLockRepository.checkAndReleaseStaleLock(any(), eq(TENANT_ID), anyLong())).thenReturn(false); // Act @@ -212,7 +212,7 @@ void persistChildren_ShouldNotProcessWhenLockFailsAndNoStaleLock() { // Assert verify(subResourcesLockRepository, times(3)).lockSubResource(any(), eq(TENANT_ID)); - verify(reindexStatusService, times(3)).isReindexInProgressOrFailed(any()); + verify(reindexStatusService, times(3)).isReindexInProgressOrFailed(); verify(subResourcesLockRepository, times(3)).checkAndReleaseStaleLock(any(), eq(TENANT_ID), anyLong()); verify(subResourcesLockRepository, never()).unlockSubResource(any(), any(), any()); verify(subjectRepository, never()).fetchByTimestamp(anyString(), any(), anyInt()); @@ -226,14 +226,14 @@ void persistChildren_ShouldSkipStaleLockCheckWhenReindexInProgress() { .when(executionService).executeSystemUserScoped(anyString(), any()); when(tenantRepository.fetchDataTenantIds()).thenReturn(List.of(TENANT_ID)); when(subResourcesLockRepository.lockSubResource(any(), eq(TENANT_ID))).thenReturn(Optional.empty()); - when(reindexStatusService.isReindexInProgressOrFailed(any())).thenReturn(true); + when(reindexStatusService.isReindexInProgressOrFailed()).thenReturn(true); // Act service.persistChildren(); // Assert verify(subResourcesLockRepository, times(3)).lockSubResource(any(), eq(TENANT_ID)); - verify(reindexStatusService, times(3)).isReindexInProgressOrFailed(any()); + verify(reindexStatusService, times(3)).isReindexInProgressOrFailed(); verify(subResourcesLockRepository, never()).checkAndReleaseStaleLock(any(), any(), anyLong()); verify(subResourcesLockRepository, never()).unlockSubResource(any(), any(), any()); verify(subjectRepository, never()).fetchByTimestamp(anyString(), any(), anyInt());