Skip to content

Commit 822c19e

Browse files
author
Jorgen-5
committed
Add indexing service
1 parent 83bc484 commit 822c19e

16 files changed

+269
-243
lines changed

‎klass-api/src/main/java/no/ssb/klass/api/config/MockSearchConfig.java‎

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,7 @@
11
package no.ssb.klass.api.config;
22

33
import no.ssb.klass.api.services.SearchService;
4-
import no.ssb.klass.api.services.search.OpenSearchResult;
5-
import no.ssb.klass.core.model.ClassificationSeries;
4+
import no.ssb.klass.api.services.OpenSearchResult;
65
import org.springframework.context.annotation.Bean;
76
import org.springframework.context.annotation.Configuration;
87
import org.springframework.context.annotation.Profile;
@@ -22,12 +21,6 @@ public Page<OpenSearchResult> publicSearch(String query, Pageable pageable, Stri
2221
boolean includeCodeLists) {
2322
return Page.empty();
2423
}
25-
26-
@Override
27-
public void indexAsync(Long classificationSeriesId) {}
28-
29-
@Override
30-
public void indexSync(ClassificationSeries classificationSeries) {}
3124
};
3225
}
3326
}

‎klass-api/src/main/java/no/ssb/klass/api/controllers/ClassificationController.java‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
import no.ssb.klass.api.dto.*;
66
import no.ssb.klass.api.dto.hal.*;
77
import no.ssb.klass.api.services.SearchService;
8-
import no.ssb.klass.api.services.search.OpenSearchResult;
8+
import no.ssb.klass.api.services.OpenSearchResult;
99
import no.ssb.klass.api.util.RestConstants;
1010
import no.ssb.klass.core.exception.KlassEmailException;
1111
import no.ssb.klass.core.model.*;
@@ -19,15 +19,12 @@
1919
import no.ssb.klass.core.util.ClientException;
2020
import no.ssb.klass.core.util.DateRange;
2121
import no.ssb.klass.core.util.KlassResourceNotFoundException;
22-
import org.opensearch.data.client.orhlc.NativeSearchQueryBuilder;
23-
import org.opensearch.index.query.QueryBuilders;
2422
import org.slf4j.Logger;
2523
import org.slf4j.LoggerFactory;
2624
import org.springframework.beans.factory.annotation.Autowired;
2725
import org.springframework.beans.factory.annotation.Value;
2826
import org.springframework.data.domain.Page;
2927
import org.springframework.data.domain.Pageable;
30-
import org.springframework.data.elasticsearch.core.query.Query;
3128
import org.springframework.data.web.PagedResourcesAssembler;
3229
import org.springframework.format.annotation.DateTimeFormat;
3330
import org.springframework.format.annotation.DateTimeFormat.ISO;

‎klass-api/src/main/java/no/ssb/klass/api/dto/hal/SearchResultResource.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
import java.util.StringJoiner;
88

99
import com.fasterxml.jackson.annotation.JsonPropertyOrder;
10-
import no.ssb.klass.api.services.search.OpenSearchResult;
10+
import no.ssb.klass.api.services.OpenSearchResult;
1111
import org.springframework.hateoas.Link;
1212
import org.springframework.hateoas.server.core.Relation;
1313

‎klass-api/src/main/java/no/ssb/klass/api/services/IndexService.java‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
package no.ssb.klass.api.services;
2+
3+
import no.ssb.klass.core.model.ClassificationSeries;
4+
import org.springframework.scheduling.annotation.Async;
5+
import org.springframework.transaction.annotation.Transactional;
6+
7+
public interface IndexService {
8+
9+
/**
10+
* Indexes a classification and makes it searchable.
11+
*
12+
* <p>
13+
* Note:
14+
* <ul>
15+
* <li>If classification is copyrighted the classification is not made searchable</li>
16+
* <li>Classification is indexed in each language</li>
17+
* </ul>
18+
*
19+
* <p>
20+
* Implementation note: Indexing is done asynchronously in order to be more responsive for front end application.
21+
*
22+
* @param classificationSeriesId
23+
*/
24+
@Async
25+
@Transactional(readOnly = true)
26+
void indexAsync(Long classificationSeriesId);
27+
28+
29+
/**
30+
* Same as indexAsync, but performs indexing within same thread. This means user must wait while indexing, so in
31+
* normal cases prefer indexAsync.
32+
* <p>
33+
* Mostly to be used by unit tests
34+
*
35+
* @param classificationSeries
36+
*/
37+
@Transactional(readOnly = true)
38+
void indexSync(ClassificationSeries classificationSeries);
39+
}

‎klass-api/src/main/java/no/ssb/klass/api/services/IndexServiceImpl.java‎

Lines changed: 172 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,172 @@
1+
package no.ssb.klass.api.services;
2+
3+
import static com.google.common.base.Preconditions.checkNotNull;
4+
5+
import java.util.*;
6+
7+
import no.ssb.klass.core.model.*;
8+
import no.ssb.klass.core.repository.ClassificationSeriesRepository;
9+
import no.ssb.klass.core.util.TimeUtil;
10+
import org.opensearch.data.client.orhlc.OpenSearchRestTemplate;
11+
import org.slf4j.Logger;
12+
import org.slf4j.LoggerFactory;
13+
import org.springframework.beans.factory.annotation.Autowired;
14+
import org.springframework.beans.factory.annotation.Qualifier;
15+
import org.springframework.beans.factory.annotation.Value;
16+
import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates;
17+
import org.springframework.data.elasticsearch.core.query.IndexQuery;
18+
import org.springframework.data.elasticsearch.core.query.IndexQueryBuilder;
19+
import org.springframework.scheduling.annotation.Async;
20+
import org.springframework.stereotype.Service;
21+
import org.springframework.transaction.annotation.Transactional;
22+
23+
@Service
24+
public class IndexServiceImpl implements IndexService {
25+
26+
private static final Logger log = LoggerFactory.getLogger(IndexServiceImpl.class);
27+
28+
@Value("${klass.env.search.elasticsearch.index:klass}")
29+
protected String elasticsearchIndex;
30+
31+
private final ClassificationSeriesRepository classificationRepository;
32+
private final OpenSearchRestTemplate elasticsearchOperations;
33+
34+
@Autowired
35+
public IndexServiceImpl(ClassificationSeriesRepository classificationRepository,
36+
@Qualifier("opensearchRestTemplate") OpenSearchRestTemplate elasticsearchOperations) {
37+
this.classificationRepository = classificationRepository;
38+
this.elasticsearchOperations = elasticsearchOperations;
39+
}
40+
41+
private IndexCoordinates getIndexCoordinates() {
42+
return IndexCoordinates.of(elasticsearchIndex);
43+
}
44+
45+
@Override
46+
@Async
47+
@Transactional(readOnly = true)
48+
public void indexAsync(Long classificationSeriesId) {
49+
checkNotNull(classificationSeriesId);
50+
try {
51+
ClassificationSeries classification = classificationRepository.getOne(classificationSeriesId);
52+
indexSync(classification);
53+
} catch (Exception e) {
54+
log.warn("Failed to index classification {}: {}", classificationSeriesId, e.getMessage());
55+
}
56+
}
57+
58+
@Override
59+
@Transactional(readOnly = true)
60+
public void indexSync(ClassificationSeries classification) {
61+
Date start = TimeUtil.now();
62+
for (Language language : Language.values()) {
63+
if (!classification.getName(language).isEmpty()) {
64+
Map<String, Object> doc = new HashMap<>();
65+
doc.put("itemid", classification.getId());
66+
doc.put("uuid", language.getLanguageCode() + "_" + classification.getUuid());
67+
doc.put("language", language.getLanguageCode());
68+
doc.put("type", classification.getClassificationType().getDisplayName(Language.EN));
69+
doc.put("copyrighted", classification.isCopyrighted());
70+
doc.put("published", classification.isPublished(language));
71+
doc.put("title", classification.getName(language));
72+
doc.put("description", classification.getDescription(language));
73+
doc.put("family", classification.getClassificationFamily().getName(language));
74+
doc.put("section", classification.getContactPerson().getSection());
75+
76+
List<String> codes = classification.getClassificationVersions().stream()
77+
.flatMap(version -> version.getAllClassificationItems().stream())
78+
.map(item -> formatClassificationItem(language, item))
79+
.toList();
80+
doc.put("codes", codes);
81+
82+
for (ClassificationVersion version : classification.getClassificationVersions()) {
83+
recursiveIndex(version, language);
84+
}
85+
86+
updateElasticsearch(classification, doc);
87+
}
88+
}
89+
elasticsearchOperations.indexOps(getIndexCoordinates()).refresh();
90+
log.info("Indexing: {} took (ms): {}", classification.getNameInPrimaryLanguage(),
91+
TimeUtil.millisecondsSince(start));
92+
}
93+
94+
private void recursiveIndex(ClassificationVersion version, Language language) {
95+
Map<String, Object> doc = new HashMap<>();
96+
doc.put("itemid", version.getId());
97+
doc.put("uuid", language.getLanguageCode() + "_" + version.getUuid());
98+
doc.put("classificationId", version.getClassification().getId());
99+
doc.put("language", language.getLanguageCode());
100+
doc.put("type", "Version");
101+
doc.put("title", version.getName(language));
102+
doc.put("legalBase", version.getLegalBase(language));
103+
doc.put("publications", version.getPublications(language));
104+
doc.put("derivedFrom", version.getDerivedFrom(language));
105+
doc.put("description", version.getIntroduction(language));
106+
doc.put("section", version.getContactPerson().getSection());
107+
doc.put("copyrighted", version.getOwnerClassification().isCopyrighted());
108+
doc.put("published", version.isPublished(language));
109+
110+
List<String> codes = version.getAllClassificationItems().stream()
111+
.map(item -> formatClassificationItem(language, item))
112+
.toList();
113+
doc.put("codes", codes);
114+
115+
indexVariants(version.getClassificationVariants(), language);
116+
indexCorrespondenceTables(version.getCorrespondenceTables(), language);
117+
118+
updateElasticsearch(version, doc);
119+
}
120+
121+
private void indexCorrespondenceTables(List<CorrespondenceTable> correspondenceTables, Language language) {
122+
for (CorrespondenceTable correspondenceTable : correspondenceTables) {
123+
Map<String, Object> doc = new HashMap<>();
124+
doc.put("itemid", correspondenceTable.getId());
125+
doc.put("uuid", language.getLanguageCode() + "_" + correspondenceTable.getUuid());
126+
doc.put("language", language.getLanguageCode());
127+
doc.put("type", "Correspondencetable");
128+
doc.put("title", correspondenceTable.getName(language));
129+
doc.put("description", correspondenceTable.getDescription(language));
130+
updateElasticsearch(correspondenceTable, doc);
131+
}
132+
}
133+
134+
private void indexVariants(List<ClassificationVariant> variants, Language language) {
135+
for (ClassificationVariant variant : variants) {
136+
Map<String, Object> doc = new HashMap<>();
137+
doc.put("itemid", variant.getId());
138+
doc.put("uuid", language.getLanguageCode() + "_" + variant.getUuid());
139+
doc.put("language", language.getLanguageCode());
140+
doc.put("type", "Variant");
141+
doc.put("title", variant.getFullName(language));
142+
doc.put("copyrighted", variant.getOwnerClassification().isCopyrighted());
143+
doc.put("published", variant.isPublished(language));
144+
doc.put("description", variant.getIntroduction(language));
145+
doc.put("section", variant.getContactPerson().getSection());
146+
147+
List<String> codes = variant.getAllClassificationItems().stream()
148+
.map(item -> formatClassificationItem(language, item))
149+
.toList();
150+
doc.put("codes", codes);
151+
152+
updateElasticsearch(variant, doc);
153+
}
154+
}
155+
156+
private void updateElasticsearch(SoftDeletable entity, Map<String, Object> doc) {
157+
String uuid = (String) doc.get("uuid");
158+
if (!entity.isDeleted()) {
159+
IndexQuery indexQuery = new IndexQueryBuilder()
160+
.withId(uuid)
161+
.withObject(doc)
162+
.build();
163+
elasticsearchOperations.index(indexQuery, getIndexCoordinates());
164+
} else {
165+
elasticsearchOperations.delete(uuid, getIndexCoordinates());
166+
}
167+
}
168+
169+
private String formatClassificationItem(Language language, ClassificationItem item) {
170+
return item.getCode() + " - " + item.getOfficialName(language);
171+
}
172+
}

klass-api/src/main/java/no/ssb/klass/api/services/search/OpenSearchResult.java renamed to klass-api/src/main/java/no/ssb/klass/api/services/OpenSearchResult.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package no.ssb.klass.api.services.search;
1+
package no.ssb.klass.api.services;
22

33
import com.fasterxml.jackson.annotation.JsonProperty;
44
import org.springframework.data.annotation.Id;

klass-api/src/main/java/no/ssb/klass/api/services/search/PublicSearchQuery.java renamed to klass-api/src/main/java/no/ssb/klass/api/services/PublicSearchQuery.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package no.ssb.klass.api.services.search;
1+
package no.ssb.klass.api.services;
22

33
import no.ssb.klass.core.model.ClassificationType;
44
import no.ssb.klass.core.model.Language;

klass-api/src/main/java/no/ssb/klass/api/services/search/SearchIndexPopulator.java renamed to klass-api/src/main/java/no/ssb/klass/api/services/SearchIndexPopulator.java

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
1-
package no.ssb.klass.api.services.search;
1+
package no.ssb.klass.api.services;
22

3-
import no.ssb.klass.api.services.SearchService;
43
import no.ssb.klass.core.config.ConfigurationProfiles;
54
import no.ssb.klass.core.repository.ClassificationSeriesRepository;
65
import org.slf4j.Logger;
@@ -26,14 +25,14 @@ public class SearchIndexPopulator implements CommandLineRunner {
2625
@Autowired
2726
private ClassificationSeriesRepository classificationSeriesRepository;
2827
@Autowired
29-
private SearchService searchService;
28+
private IndexService indexService;
3029

3130
@Override
3231
public void run(String... args) {
3332
CompletableFuture.runAsync(() -> {
3433
List<Long> ids = classificationSeriesRepository.findAllClassificationIds();
3534
log.info("Starting to index {} classifications", ids.size());
36-
ids.forEach(searchService::indexAsync);
35+
ids.forEach(indexService::indexAsync);
3736
log.info("Finished indexing for {} classifications", ids.size());
3837
});
3938
}

‎klass-api/src/main/java/no/ssb/klass/api/services/SearchService.java‎

Lines changed: 0 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,5 @@
11
package no.ssb.klass.api.services;
22

3-
import no.ssb.klass.api.services.search.OpenSearchResult;
4-
import no.ssb.klass.core.model.ClassificationSeries;
53
import org.springframework.data.domain.Pageable;
64

75

@@ -10,31 +8,4 @@ public interface SearchService {
108
org.springframework.data.domain.Page<OpenSearchResult> publicSearch(String query, Pageable pageable, String filterOnSection,
119
boolean includeCodeLists);
1210

13-
/**
14-
* Indexes a classification and makes it searchable.
15-
*
16-
* <p>
17-
* Note:
18-
* <ul>
19-
* <li>If classification is copyrighted the classification is not made searchable</li>
20-
* <li>Classification is indexed in each language</li>
21-
* </ul>
22-
*
23-
* <p>
24-
* Implementation note: Indexing is done asynchronously in order to be more responsive for front end application.
25-
*
26-
* @param classificationSeriesId
27-
*/
28-
void indexAsync(Long classificationSeriesId);
29-
30-
31-
/**
32-
* Same as indexAsync, but performs indexing within same thread. This means user must wait while indexing, so in
33-
* normal cases prefer indexAsync.
34-
* <p>
35-
* Mostly to be used by unit tests
36-
*
37-
* @param classificationSeries
38-
*/
39-
void indexSync(ClassificationSeries classificationSeries);
4011
}

0 commit comments

Comments
 (0)