diff --git a/mycore-classeditor/src/main/java/org/mycore/frontend/classeditor/resources/MCRClassificationEditorResource.java b/mycore-classeditor/src/main/java/org/mycore/frontend/classeditor/resources/MCRClassificationEditorResource.java index 7b80dd76f0..49541bb690 100644 --- a/mycore-classeditor/src/main/java/org/mycore/frontend/classeditor/resources/MCRClassificationEditorResource.java +++ b/mycore-classeditor/src/main/java/org/mycore/frontend/classeditor/resources/MCRClassificationEditorResource.java @@ -58,7 +58,7 @@ import org.mycore.frontend.classeditor.json.MCRJSONCategoryHelper; import org.mycore.frontend.classeditor.wrapper.MCRCategoryListWrapper; import org.mycore.frontend.jersey.filter.access.MCRRestrictedAccess; -import org.mycore.solr.MCRSolrCoreManager; +import org.mycore.solr.MCRSolrIndexManager; import org.mycore.solr.MCRSolrUtils; import org.mycore.solr.auth.MCRSolrAuthenticationLevel; import org.mycore.solr.auth.MCRSolrAuthenticationManager; @@ -263,7 +263,7 @@ public Response importClassification(@FormDataParam("classificationFile") InputS @Path("filter/{text}") @Produces(MediaType.APPLICATION_JSON) public Response filter(@PathParam("text") String text) { - SolrClient solrClient = MCRSolrClassificationUtil.getCore().getClient(); + SolrClient solrClient = MCRSolrClassificationUtil.getIndexList().getFirst().getClient(); ModifiableSolrParams p = new ModifiableSolrParams(); p.set("q", "allMeta:" + "*" + MCRSolrUtils.escapeSearchValue(text) + "*"); p.set("fl", "id,ancestors"); @@ -289,7 +289,7 @@ public Response filter(@PathParam("text") String text) { public Response retrieveLinkedObjects(@PathParam("id") String id, @QueryParam("start") Integer start, @QueryParam("rows") Integer rows) throws SolrServerException, IOException { // do solr query - SolrClient solrClient = MCRSolrCoreManager.getMainSolrClient(); + SolrClient solrClient = MCRSolrIndexManager.obtainInstance().requireMainIndex().getClient(); ModifiableSolrParams params = new ModifiableSolrParams(); params.set("start", start != null ? start : 0); params.set("rows", rows != null ? rows : 50); diff --git a/mycore-impex/src/main/java/org/mycore/impex/MCRTransferPackageCommands.java b/mycore-impex/src/main/java/org/mycore/impex/MCRTransferPackageCommands.java index 8b63a9115e..e00a1d2dc4 100644 --- a/mycore-impex/src/main/java/org/mycore/impex/MCRTransferPackageCommands.java +++ b/mycore-impex/src/main/java/org/mycore/impex/MCRTransferPackageCommands.java @@ -32,6 +32,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.solr.client.solrj.SolrClient; import org.mycore.access.MCRAccessException; import org.mycore.common.MCRUtils; import org.mycore.datamodel.classifications2.utils.MCRClassificationUtils; @@ -42,8 +43,8 @@ import org.mycore.frontend.cli.annotation.MCRCommand; import org.mycore.frontend.cli.annotation.MCRCommandGroup; import org.mycore.services.packaging.MCRPackerManager; -import org.mycore.solr.MCRSolrCoreManager; -import org.mycore.solr.MCRSolrCoreType; +import org.mycore.solr.MCRIndexType; +import org.mycore.solr.MCRSolrIndexManager; import org.mycore.solr.index.MCRSolrIndexer; import org.mycore.solr.search.MCRSolrSearchUtils; @@ -56,7 +57,8 @@ public class MCRTransferPackageCommands { @MCRCommand(help = "Creates multiple transfer packages which matches the solr query in {0}.", syntax = "create transfer package for objects matching {0}") public static void create(String query) throws MCRAccessException { - List ids = MCRSolrSearchUtils.listIDs(MCRSolrCoreManager.getMainSolrClient(), query); + SolrClient solrClient = MCRSolrIndexManager.obtainInstance().requireMainIndex().getClient(); + List ids = MCRSolrSearchUtils.listIDs(solrClient, query); for (String objectId : ids) { Map parameters = new HashMap<>(); parameters.put("packer", "TransferPackage"); @@ -190,7 +192,8 @@ public static void cleanUp(String targetDirectoryPath) throws Exception { markManager.remove(MCRObjectID.getInstance(id)); } // index all objects - MCRSolrIndexer.rebuildMetadataIndex(mcrObjects, MCRSolrCoreManager.getCoresForType(MCRSolrCoreType.MAIN)); + MCRSolrIndexer.rebuildMetadataIndex(mcrObjects, MCRSolrIndexManager.obtainInstance() + .getIndexWithType(MCRIndexType.MAIN)); // deleting expanded directory LOGGER.info("Deleting expanded tar in {}...", targetDirectoryPath); diff --git a/mycore-indexing/src/main/java/org/mycore/frontend/indexbrowser/MCRGoogleSitemapCommon.java b/mycore-indexing/src/main/java/org/mycore/frontend/indexbrowser/MCRGoogleSitemapCommon.java index 63ea4c3003..9393e215c2 100644 --- a/mycore-indexing/src/main/java/org/mycore/frontend/indexbrowser/MCRGoogleSitemapCommon.java +++ b/mycore-indexing/src/main/java/org/mycore/frontend/indexbrowser/MCRGoogleSitemapCommon.java @@ -37,6 +37,8 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; + +import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.request.QueryRequest; import org.apache.solr.client.solrj.request.SolrQuery; @@ -47,7 +49,7 @@ import org.mycore.common.config.MCRConfiguration2; import org.mycore.datamodel.common.MCRObjectIDDate; import org.mycore.datamodel.ifs2.MCRObjectIDDateImpl; -import org.mycore.solr.MCRSolrCoreManager; +import org.mycore.solr.MCRSolrIndexManager; import org.mycore.solr.auth.MCRSolrAuthenticationLevel; import org.mycore.solr.auth.MCRSolrAuthenticationManager; @@ -182,7 +184,8 @@ public int checkSitemapFile() throws IOException { QueryRequest queryRequest = new QueryRequest(query); SOLR_AUTHENTICATION_MANAGER.applyAuthentication(queryRequest, MCRSolrAuthenticationLevel.SEARCH); - response = queryRequest.process(MCRSolrCoreManager.getMainSolrClient()); + SolrClient client = MCRSolrIndexManager.obtainInstance().requireMainIndex().getClient(); + response = queryRequest.process(client); objidlist = response.getResults().stream().map((document) -> { String id = (String) document.getFieldValue("id"); Date modified = (Date) document.getFieldValue("modified"); diff --git a/mycore-mets/src/main/java/org/mycore/mets/resource/MCRAltoHighlightResource.java b/mycore-mets/src/main/java/org/mycore/mets/resource/MCRAltoHighlightResource.java index 042ccab145..e061e53a4a 100644 --- a/mycore-mets/src/main/java/org/mycore/mets/resource/MCRAltoHighlightResource.java +++ b/mycore-mets/src/main/java/org/mycore/mets/resource/MCRAltoHighlightResource.java @@ -28,7 +28,7 @@ import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.common.params.ModifiableSolrParams; import org.mycore.frontend.jersey.MCRJerseyUtil; -import org.mycore.solr.MCRSolrCoreManager; +import org.mycore.solr.MCRSolrIndexManager; import org.mycore.solr.auth.MCRSolrAuthenticationLevel; import org.mycore.solr.auth.MCRSolrAuthenticationManager; @@ -118,7 +118,8 @@ public Response query(@PathParam("derivateId") String derivateId, @QueryParam("q try { QueryRequest queryRequest = new QueryRequest(p); SOLR_AUTHENTICATION_MANAGER.applyAuthentication(queryRequest, MCRSolrAuthenticationLevel.SEARCH); - QueryResponse solrResponse = queryRequest.process(MCRSolrCoreManager.getMainSolrClient()); + QueryResponse solrResponse = + queryRequest.process(MCRSolrIndexManager.obtainInstance().requireMainIndex().getClient()); JsonArray response = buildQueryResponse(solrResponse.getHighlighting()); return Response.ok().entity(new Gson().toJson(response)).build(); } catch (Exception exc) { diff --git a/mycore-mods/src/main/java/org/mycore/mods/MCRMODSEmbargoReleaseCronjob.java b/mycore-mods/src/main/java/org/mycore/mods/MCRMODSEmbargoReleaseCronjob.java index 9ae4396eb1..2e02eaf71f 100644 --- a/mycore-mods/src/main/java/org/mycore/mods/MCRMODSEmbargoReleaseCronjob.java +++ b/mycore-mods/src/main/java/org/mycore/mods/MCRMODSEmbargoReleaseCronjob.java @@ -30,12 +30,11 @@ import org.apache.solr.common.params.ModifiableSolrParams; import org.jdom2.Element; import org.mycore.common.MCRSystemUserInformation; -import org.mycore.common.config.MCRConfiguration2; import org.mycore.datamodel.metadata.MCRMetadataManager; import org.mycore.datamodel.metadata.MCRObject; import org.mycore.datamodel.metadata.MCRObjectID; import org.mycore.mcr.cronjob.MCRCronjob; -import org.mycore.solr.MCRSolrCoreManager; +import org.mycore.solr.MCRSolrIndexManager; import org.mycore.solr.auth.MCRSolrAuthenticationLevel; import org.mycore.solr.auth.MCRSolrAuthenticationManager; import org.mycore.util.concurrent.MCRFixedUserFailableRunnable; @@ -58,7 +57,7 @@ public String getDescription() { @Override public void runJob() { - if (MCRConfiguration2.getString("MCR.Solr.ServerURL").isEmpty()) { + if(MCRSolrIndexManager.obtainInstance().getMainIndex().isPresent()) { return; } @@ -81,7 +80,8 @@ public void runJob() { QueryRequest queryRequest = new QueryRequest(params); SOLR_AUTHENTICATION_MANAGER.applyAuthentication(queryRequest, MCRSolrAuthenticationLevel.SEARCH); - QueryResponse solrResponse = queryRequest.process(MCRSolrCoreManager.getMainSolrClient()); + QueryResponse solrResponse = + queryRequest.process(MCRSolrIndexManager.obtainInstance().requireMainIndex().getClient()); solrResponse .getResults() .stream() diff --git a/mycore-mods/src/main/java/org/mycore/mods/rss/MCRRSSFeedImporter.java b/mycore-mods/src/main/java/org/mycore/mods/rss/MCRRSSFeedImporter.java index 0f0a6a726a..8e76e5cffe 100644 --- a/mycore-mods/src/main/java/org/mycore/mods/rss/MCRRSSFeedImporter.java +++ b/mycore-mods/src/main/java/org/mycore/mods/rss/MCRRSSFeedImporter.java @@ -55,7 +55,7 @@ import org.mycore.datamodel.metadata.MCRObject; import org.mycore.datamodel.metadata.MCRObjectID; import org.mycore.mods.MCRMODSWrapper; -import org.mycore.solr.MCRSolrCoreManager; +import org.mycore.solr.MCRSolrIndexManager; import org.mycore.solr.MCRSolrUtils; import org.mycore.solr.auth.MCRSolrAuthenticationLevel; import org.mycore.solr.auth.MCRSolrAuthenticationManager; @@ -220,7 +220,7 @@ private String getPublicationID(SyndEntry entry) { } private boolean isAlreadyStored(String publicationID) { - SolrClient solrClient = MCRSolrCoreManager.getMainSolrClient(); + SolrClient solrClient = MCRSolrIndexManager.obtainInstance().requireMainIndex().getClient(); SolrQuery query = new SolrQuery(); query.setQuery(field2queryID + ":" + MCRSolrUtils.escapeSearchValue(publicationID)); query.setRows(0); diff --git a/mycore-oai/src/main/java/org/mycore/oai/MCROAISolrSearcher.java b/mycore-oai/src/main/java/org/mycore/oai/MCROAISolrSearcher.java index b51856e0a1..f3f05b7455 100644 --- a/mycore-oai/src/main/java/org/mycore/oai/MCROAISolrSearcher.java +++ b/mycore-oai/src/main/java/org/mycore/oai/MCROAISolrSearcher.java @@ -50,7 +50,7 @@ import org.mycore.oai.set.MCROAISetResolver; import org.mycore.oai.set.MCROAISolrSetHandler; import org.mycore.oai.set.MCRSet; -import org.mycore.solr.MCRSolrCoreManager; +import org.mycore.solr.MCRSolrIndexManager; import org.mycore.solr.MCRSolrUtils; import org.mycore.solr.auth.MCRSolrAuthenticationLevel; import org.mycore.solr.auth.MCRSolrAuthenticationManager; @@ -87,7 +87,7 @@ public Optional
getHeader(String mcrId) { query.set(CommonParams.Q, "id:" + MCRSolrUtils.escapeSearchValue(mcrId)); query.setRows(1); // do the query - SolrClient solrClient = MCRSolrCoreManager.getMainSolrClient(); + SolrClient solrClient = MCRSolrIndexManager.obtainInstance().requireMainIndex().getClient(); try { QueryRequest queryRequest = new QueryRequest(query); queryRequest.setPath(getRequestHandlerPath()); @@ -159,7 +159,7 @@ protected MCROAISolrResult solrQuery(Optional cursor) throws SolrServerE query.set(CommonParams.SORT, "id asc"); // do the query - SolrClient solrClient = MCRSolrCoreManager.getMainSolrClient(); + SolrClient solrClient = MCRSolrIndexManager.obtainInstance().requireMainIndex().getClient(); QueryRequest queryRequest = new QueryRequest(query); queryRequest.setPath(getRequestHandlerPath()); MCRSolrAuthenticationManager.obtainInstance().applyAuthentication(queryRequest, @@ -270,7 +270,7 @@ public Optional getEarliestTimestamp() { params.add(CommonParams.FQ, fieldName + ":[* TO *]"); params.add(CommonParams.FL, fieldName); params.add(CommonParams.ROWS, "1"); - SolrClient solrClient = MCRSolrCoreManager.getMainSolrClient(); + SolrClient solrClient = MCRSolrIndexManager.obtainInstance().requireMainIndex().getClient(); try { QueryRequest queryRequest = new QueryRequest(params); MCRSolrAuthenticationManager.obtainInstance().applyAuthentication(queryRequest, diff --git a/mycore-oai/src/main/java/org/mycore/oai/set/MCROAIClassificationToSetHandler.java b/mycore-oai/src/main/java/org/mycore/oai/set/MCROAIClassificationToSetHandler.java index a2b60edd13..0a9c705777 100644 --- a/mycore-oai/src/main/java/org/mycore/oai/set/MCROAIClassificationToSetHandler.java +++ b/mycore-oai/src/main/java/org/mycore/oai/set/MCROAIClassificationToSetHandler.java @@ -33,7 +33,7 @@ import org.mycore.common.config.MCRConfiguration2; import org.mycore.oai.MCROAIUtils; import org.mycore.oai.classmapping.MCRClassificationAndSetMapper; -import org.mycore.solr.MCRSolrCoreManager; +import org.mycore.solr.MCRSolrIndexManager; import org.mycore.solr.MCRSolrUtils; import org.mycore.solr.auth.MCRSolrAuthenticationLevel; import org.mycore.solr.auth.MCRSolrAuthenticationManager; @@ -70,7 +70,7 @@ public boolean filter(MCRSet set) { if (!filterEmptySets()) { return false; } - SolrClient solrClient = MCRSolrCoreManager.getMainSolrClient(); + SolrClient solrClient = MCRSolrIndexManager.obtainInstance().requireMainIndex().getClient(); ModifiableSolrParams p = new ModifiableSolrParams(); String value = set.getSpec(); p.set(CommonParams.Q, MCROAIUtils.getDefaultSetQuery(value, getConfigPrefix())); diff --git a/mycore-oai/src/main/java/org/mycore/oai/set/MCROAIQuerySetResolver.java b/mycore-oai/src/main/java/org/mycore/oai/set/MCROAIQuerySetResolver.java index 2f73136cb4..f31f64b4df 100644 --- a/mycore-oai/src/main/java/org/mycore/oai/set/MCROAIQuerySetResolver.java +++ b/mycore-oai/src/main/java/org/mycore/oai/set/MCROAIQuerySetResolver.java @@ -34,7 +34,7 @@ import org.mycore.common.MCRException; import org.mycore.common.config.MCRConfiguration2; import org.mycore.oai.pmh.Set; -import org.mycore.solr.MCRSolrCoreManager; +import org.mycore.solr.MCRSolrIndexManager; import org.mycore.solr.MCRSolrUtils; import org.mycore.solr.auth.MCRSolrAuthenticationLevel; import org.mycore.solr.auth.MCRSolrAuthenticationManager; @@ -63,7 +63,7 @@ public void init(String configPrefix, String setId, Map setMap, idsInSet = Collections.emptySet(); return; } - SolrClient solrClient = MCRSolrCoreManager.getMainSolrClient(); + SolrClient solrClient = MCRSolrIndexManager.obtainInstance().requireMainIndex().getClient(); QueryResponse response; try { QueryRequest queryRequest = new QueryRequest(getQuery()); diff --git a/mycore-restapi/src/main/java/org/mycore/restapi/v1/MCRRestAPIClassifications.java b/mycore-restapi/src/main/java/org/mycore/restapi/v1/MCRRestAPIClassifications.java index 16c09706d6..3aac9066b8 100644 --- a/mycore-restapi/src/main/java/org/mycore/restapi/v1/MCRRestAPIClassifications.java +++ b/mycore-restapi/src/main/java/org/mycore/restapi/v1/MCRRestAPIClassifications.java @@ -53,7 +53,7 @@ import org.mycore.frontend.jersey.MCRCacheControl; import org.mycore.restapi.v1.errors.MCRRestAPIError; import org.mycore.restapi.v1.errors.MCRRestAPIException; -import org.mycore.solr.MCRSolrCoreManager; +import org.mycore.solr.MCRSolrIndexManager; import org.mycore.solr.auth.MCRSolrAuthenticationLevel; import org.mycore.solr.auth.MCRSolrAuthenticationManager; @@ -513,7 +513,7 @@ private String writeJSON(Element eRoot, String lang, String style) throws IOExce } private void filterNonEmpty(String classId, Element e) { - SolrClient solrClient = MCRSolrCoreManager.getMainSolrClient(); + SolrClient solrClient = MCRSolrIndexManager.obtainInstance().requireMainIndex().getClient(); Element[] categories = e.getChildren(ELEMENT_CATEGORY).toArray(Element[]::new); for (Element cat : categories) { SolrQuery solrQuery = new SolrQuery(); diff --git a/mycore-restapi/src/main/java/org/mycore/restapi/v1/MCRRestAPISearch.java b/mycore-restapi/src/main/java/org/mycore/restapi/v1/MCRRestAPISearch.java index ace4cb5a5d..94f6d94a2e 100644 --- a/mycore-restapi/src/main/java/org/mycore/restapi/v1/MCRRestAPISearch.java +++ b/mycore-restapi/src/main/java/org/mycore/restapi/v1/MCRRestAPISearch.java @@ -25,21 +25,21 @@ import static org.mycore.frontend.jersey.MCRJerseyUtil.TEXT_XML_UTF_8; import java.io.IOException; -import java.net.URI; -import java.net.URLEncoder; -import java.net.http.HttpClient; -import java.net.http.HttpRequest; -import java.net.http.HttpResponse; -import java.nio.charset.StandardCharsets; +import java.io.InputStream; import java.util.List; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.mycore.services.http.MCRHttpUtils; -import org.mycore.solr.MCRSolrCoreManager; -import org.mycore.solr.MCRSolrUtils; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.request.QueryRequest; +import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.mycore.solr.MCRSolrIndex; +import org.mycore.solr.MCRSolrIndexManager; import org.mycore.solr.auth.MCRSolrAuthenticationLevel; import org.mycore.solr.auth.MCRSolrAuthenticationManager; +import org.mycore.solr.search.MCRSolrSearchUtils; import jakarta.ws.rs.DefaultValue; import jakarta.ws.rs.GET; @@ -83,68 +83,64 @@ public class MCRRestAPISearch { @SuppressWarnings("PMD.ExcessiveParameterList") @GET @Produces({ TEXT_XML_UTF_8, APPLICATION_JSON_UTF_8, TEXT_PLAIN_ISO_8859_1, TEXT_PLAIN_UTF_8 }) - public Response search(@QueryParam("q") String query, - @QueryParam("sort") String sort, @QueryParam("wt") @DefaultValue("xml") String wt, - @QueryParam("start") String start, @QueryParam("rows") String rows, - @QueryParam("fq") List fq, @QueryParam("fl") List fl, + public Response search(@QueryParam(CommonParams.Q) String query, + @QueryParam(CommonParams.SORT) String sort, @QueryParam(CommonParams.WT) @DefaultValue("xml") String wt, + @QueryParam(CommonParams.START) String start, @QueryParam(CommonParams.ROWS) String rows, + @QueryParam(CommonParams.FQ) List fq, @QueryParam(CommonParams.FL) List fl, @QueryParam("facet") String facet, @QueryParam("facet.sort") String facetSort, @QueryParam("facet.limit") String facetLimit, @QueryParam("facet.field") List facetFields, @QueryParam("facet.mincount") String facetMinCount, @QueryParam("json.wrf") String jsonWrf) { - StringBuilder url = new StringBuilder(MCRSolrCoreManager.getMainSolrCore().getV1CoreURL()); - url.append("/select?"); + ModifiableSolrParams params = new ModifiableSolrParams(); + // Append query parameters using helper methods - appendQueryParam(url, "q", query); - appendQueryParam(url, "sort", sort); - appendQueryParam(url, "wt", wt); - appendQueryParam(url, "start", start); - appendQueryParam(url, "rows", rows); - - appendListQueryParam(url, "fq", fq); - appendListQueryParam(url, "fl", fl); - - appendQueryParam(url, "facet", facet); - appendQueryParam(url, "facet.sort", facetSort); - appendQueryParam(url, "facet.limit", facetLimit); - appendQueryParam(url, "facet.mincount", facetMinCount); - appendListQueryParam(url, "facet.field", facetFields); - appendQueryParam(url, "json.wrf", jsonWrf); - - return executeSolrQuery(url.toString(), wt); + appendQueryParam(params, CommonParams.Q, query); + appendQueryParam(params, CommonParams.SORT, sort); + appendQueryParam(params, CommonParams.WT, wt); + appendQueryParam(params, CommonParams.START, start); + appendQueryParam(params, CommonParams.ROWS, rows); + + appendListQueryParam(params, CommonParams.FQ, fq); + appendListQueryParam(params, CommonParams.FL, fl); + + appendQueryParam(params, "facet", facet); + appendQueryParam(params, "facet.sort", facetSort); + appendQueryParam(params, "facet.limit", facetLimit); + appendQueryParam(params, "facet.mincount", facetMinCount); + appendListQueryParam(params, "facet.field", facetFields); + appendQueryParam(params, "json.wrf", jsonWrf); + + return executeSolrQuery(params, wt); } // Helper method to append single query parameters - private void appendQueryParam(StringBuilder url, String param, String value) { + private void appendQueryParam(ModifiableSolrParams params, String param, String value) { if (value != null) { - url.append('&').append(param).append('=') - .append(URLEncoder.encode(value, StandardCharsets.UTF_8)); + params.set(param, value); } } // Helper method to append list-based query parameters - private void appendListQueryParam(StringBuilder url, String param, List values) { - if (values != null) { - for (String value : values) { - appendQueryParam(url, param, value); - } + private void appendListQueryParam(ModifiableSolrParams params, String param, List values) { + if (values != null && !values.isEmpty()) { + params.add(param, values.toArray(new String[0])); } } // Method to execute the Solr query and handle response - private Response executeSolrQuery(String url, String wt) { - HttpRequest.Builder reqBuilder = MCRSolrUtils.getRequestBuilder().uri(URI.create(url)); - MCRSolrAuthenticationManager.obtainInstance().applyAuthentication(reqBuilder, + private Response executeSolrQuery(ModifiableSolrParams params, String wt) { + MCRSolrIndex solrIndex = MCRSolrIndexManager.obtainInstance().requireMainIndex(); + SolrClient client = solrIndex.getClient(); + QueryRequest queryRequest = new QueryRequest(params); + MCRSolrAuthenticationManager.obtainInstance().applyAuthentication(queryRequest, MCRSolrAuthenticationLevel.SEARCH); - HttpRequest request = reqBuilder.build(); - try (HttpClient client = MCRHttpUtils.getHttpClient()) { - HttpResponse resp = - client.send(request, HttpResponse.BodyHandlers.ofString(StandardCharsets.UTF_8)); - return Response.ok(resp.body()) + try (InputStream is = MCRSolrSearchUtils.streamRequest(client, queryRequest, wt)) { + return Response.ok(is.readAllBytes()) .type(getContentType(wt)) .build(); - } catch (InterruptedException | IOException e) { + } catch (IOException | SolrServerException e) { LOGGER.error("Error while executing Solr query", e); return Response.status(Response.Status.BAD_REQUEST).build(); } diff --git a/mycore-solr/pom.xml b/mycore-solr/pom.xml index cf6acdfbcf..0f729a923b 100644 --- a/mycore-solr/pom.xml +++ b/mycore-solr/pom.xml @@ -112,6 +112,11 @@ hibernate-core provided + + org.apache.solr + solr-solrj-zookeeper + runtime + org.junit.jupiter junit-jupiter-api diff --git a/mycore-solr/src/main/java/org/mycore/solr/MCRAbstractHttpBasedIndexConfigAdapter.java b/mycore-solr/src/main/java/org/mycore/solr/MCRAbstractHttpBasedIndexConfigAdapter.java new file mode 100644 index 0000000000..b0dabf1ec9 --- /dev/null +++ b/mycore-solr/src/main/java/org/mycore/solr/MCRAbstractHttpBasedIndexConfigAdapter.java @@ -0,0 +1,188 @@ +/* + * This file is part of *** M y C o R e *** + * See https://www.mycore.de/ for details. + * + * MyCoRe is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * MyCoRe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with MyCoRe. If not, see . + */ + +package org.mycore.solr; + +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; + +import org.apache.solr.client.solrj.impl.HttpJdkSolrClient; +import org.apache.solr.client.solrj.impl.HttpSolrClientBuilderBase; +import org.apache.solr.client.solrj.jetty.HttpJettySolrClient; +import org.mycore.common.config.MCRConfiguration2; +import org.mycore.common.config.annotation.MCRProperty; + +public abstract class MCRAbstractHttpBasedIndexConfigAdapter implements Supplier { + + private String coreTypes; + + private String idleTimeout; + + private String idleTimeoutUnit; + + private String connectionTimeout; + + private String connectionTimeoutUnit; + + private String requestTimeout; + + private String requestTimeoutUnit; + + private String useHttp11; + + private boolean useJettyHttpClient; + + public String getCoreTypes() { + return coreTypes; + } + + @MCRProperty(name = "CoreTypes") + public void setCoreTypes(String coreTypes) { + this.coreTypes = coreTypes; + } + + public String getIdleTimeout() { + return idleTimeout; + } + + @MCRProperty(name = "IdleTimeout", defaultName = MCRSolrDefaultPropertyConstants.CLIENT_IDLE_TIMEOUT, + required = false) + public void setIdleTimeout(String idleTimeout) { + this.idleTimeout = idleTimeout; + } + + public String getIdleTimeoutUnit() { + return idleTimeoutUnit; + } + + @MCRProperty(name = "IdleTimeout.Unit", defaultName = MCRSolrDefaultPropertyConstants.CLIENT_IDLE_TIMEOUT_UNIT, + required = false) + public void setIdleTimeoutUnit(String idleTimeoutUnit) { + this.idleTimeoutUnit = idleTimeoutUnit; + } + + public String getConnectionTimeout() { + return connectionTimeout; + } + + @MCRProperty(name = "ConnectionTimeout", defaultName = MCRSolrDefaultPropertyConstants.CLIENT_CONNECTION_TIMEOUT, + required = false) + public void setConnectionTimeout(String connectionTimeout) { + this.connectionTimeout = connectionTimeout; + } + + public String getConnectionTimeoutUnit() { + return connectionTimeoutUnit; + } + + @MCRProperty(name = "ConnectionTimeout.Unit", + defaultName = MCRSolrDefaultPropertyConstants.CLIENT_CONNECTION_TIMEOUT_UNIT, + required = false) + public void setConnectionTimeoutUnit(String connectionTimeoutUnit) { + this.connectionTimeoutUnit = connectionTimeoutUnit; + } + + public String getRequestTimeout() { + return requestTimeout; + } + + @MCRProperty(name = "RequestTimeout", defaultName = MCRSolrDefaultPropertyConstants.CLIENT_REQUEST_TIMEOUT, + required = false) + public void setRequestTimeout(String requestTimeout) { + this.requestTimeout = requestTimeout; + } + + public String getRequestTimeoutUnit() { + return requestTimeoutUnit; + } + + @MCRProperty(name = "RequestTimeout.Unit", + defaultName = MCRSolrDefaultPropertyConstants.CLIENT_REQUEST_TIMEOUT_UNIT, + required = false) + public void setRequestTimeoutUnit(String requestTimeoutUnit) { + this.requestTimeoutUnit = requestTimeoutUnit; + } + + public String getUseHttp11() { + return useHttp11; + } + + public void setUseHttp11(String useHttp11) { + this.useHttp11 = useHttp11; + } + + @MCRProperty(name = "UseJettyHttpClient", defaultName = MCRSolrDefaultPropertyConstants.USE_JETTY_HTTP_CLIENT) + public void setUseJettyHttpClient(String useJettyHttpClient) { + this.useJettyHttpClient = Boolean.parseBoolean(useJettyHttpClient); + } + + public boolean useJettyHttpClient() { + return useJettyHttpClient; + } + + protected HttpSolrClientBuilderBase getBuilder() { + return useJettyHttpClient() ? new HttpJettySolrClient.Builder() + : new HttpJdkSolrClient.Builder(); + } + + protected void applySettings(HttpSolrClientBuilderBase builder) { + + String idleTimeout = getIdleTimeout(); + String idleTimeoutUnit = getIdleTimeoutUnit(); + if (idleTimeout != null && idleTimeoutUnit != null) { + builder.withIdleTimeout(Long.parseLong(idleTimeout), parseTimeUnit(idleTimeoutUnit)); + } + + String connectionTimeout = getConnectionTimeout(); + String connectionTimeoutUnit = getConnectionTimeoutUnit(); + if (connectionTimeout != null && connectionTimeoutUnit != null) { + builder.withConnectionTimeout(Long.parseLong(connectionTimeout), + parseTimeUnit(connectionTimeoutUnit)); + } + + String requestTimeout = getRequestTimeout(); + String requestTimeoutUnit = getRequestTimeoutUnit(); + if (requestTimeout != null && requestTimeoutUnit != null) { + builder.withRequestTimeout(Long.parseLong(requestTimeout), parseTimeUnit(requestTimeoutUnit)); + + } + + String useHttp11 = getUseHttp11(); + if (useHttp11 != null) { + builder.useHttp1_1(Boolean.parseBoolean(useHttp11)); + } + + } + + protected Set buildCoreTypes() { + return MCRConfiguration2.splitValue(coreTypes) + .map(String::trim) + .map(MCRIndexType::new) + .collect(java.util.stream.Collectors.toSet()); + } + + private TimeUnit parseTimeUnit(String timeUnit) { + try { + return TimeUnit.valueOf(timeUnit); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Invalid time unit: " + timeUnit, e); + } + } + +} diff --git a/mycore-solr/src/main/java/org/mycore/solr/MCRConfigurableIndexManager.java b/mycore-solr/src/main/java/org/mycore/solr/MCRConfigurableIndexManager.java new file mode 100644 index 0000000000..35ebd0aaf0 --- /dev/null +++ b/mycore-solr/src/main/java/org/mycore/solr/MCRConfigurableIndexManager.java @@ -0,0 +1,93 @@ +/* + * This file is part of *** M y C o R e *** + * See https://www.mycore.de/ for details. + * + * MyCoRe is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * MyCoRe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with MyCoRe. If not, see . + */ + +package org.mycore.solr; + +import static org.mycore.solr.MCRSolrConstants.SOLR_COLLECTION_MANAGER_INDEX_PREFIX; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Supplier; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.mycore.common.config.annotation.MCRConfigurationProxy; +import org.mycore.common.config.annotation.MCRInstanceMap; +import org.mycore.common.events.MCRShutdownHandler; +import org.mycore.solr.MCRConfigurableIndexManager.ConfigAdapter; + +@MCRConfigurationProxy( + proxyClass = ConfigAdapter.class) +public class MCRConfigurableIndexManager implements MCRSolrIndexManager { + + private final Map configuredCollections; + private static final Logger LOGGER = LogManager.getLogger(); + + public MCRConfigurableIndexManager(Map configuredCollections) { + this.configuredCollections = configuredCollections; + + MCRShutdownHandler shutdownHandler = MCRShutdownHandler.getInstance(); + if (shutdownHandler != null) { + shutdownHandler.addCloseable(this::closeIndexes); + } + } + + @Override + public Optional getIndex(String indexId) { + return Optional.ofNullable(configuredCollections.get(indexId)); + } + + @Override + public List getIndexWithType(MCRIndexType type) { + return configuredCollections.values() + .stream() + .filter(col -> col.getCoreTypes().contains(type)) + .toList(); + } + + public void closeIndexes() { + configuredCollections.values().forEach(col -> { + try { + col.close(); + } catch (IOException e) { + LOGGER.error("Error closing Solr index {}", col::getName, () -> e); + } + }); + } + + public static class ConfigAdapter implements Supplier { + + private Map collections; + + public Map getCollections() { + return collections; + } + + @MCRInstanceMap(name = SOLR_COLLECTION_MANAGER_INDEX_PREFIX, valueClass = MCRSolrIndex.class) + public void setCollections(Map collections) { + this.collections = collections; + } + + @Override + public MCRConfigurableIndexManager get() { + return new MCRConfigurableIndexManager(this.getCollections()); + } + } +} diff --git a/mycore-solr/src/main/java/org/mycore/solr/MCRSolrCoreType.java b/mycore-solr/src/main/java/org/mycore/solr/MCRIndexType.java similarity index 77% rename from mycore-solr/src/main/java/org/mycore/solr/MCRSolrCoreType.java rename to mycore-solr/src/main/java/org/mycore/solr/MCRIndexType.java index f49eca8f86..fee278870c 100644 --- a/mycore-solr/src/main/java/org/mycore/solr/MCRSolrCoreType.java +++ b/mycore-solr/src/main/java/org/mycore/solr/MCRIndexType.java @@ -18,10 +18,10 @@ package org.mycore.solr; -public record MCRSolrCoreType(String name) { +public record MCRIndexType(String name) { - public static final MCRSolrCoreType MAIN = new MCRSolrCoreType("main"); + public static final MCRIndexType MAIN = new MCRIndexType("main"); - public static final MCRSolrCoreType CLASSIFICATION = new MCRSolrCoreType("classification"); + public static final MCRIndexType CLASSIFICATION = new MCRIndexType("classification"); } diff --git a/mycore-solr/src/main/java/org/mycore/solr/MCRModifiableSolrIndexManager.java b/mycore-solr/src/main/java/org/mycore/solr/MCRModifiableSolrIndexManager.java new file mode 100644 index 0000000000..f54e57e8bb --- /dev/null +++ b/mycore-solr/src/main/java/org/mycore/solr/MCRModifiableSolrIndexManager.java @@ -0,0 +1,25 @@ +package org.mycore.solr; + +/** + * An extension of {@link MCRSolrIndexManager} which allows to add and remove indices at runtime. + */ +public interface MCRModifiableSolrIndexManager extends MCRSolrIndexManager { + + /** + * Adds a new index to the manager. If an index with the same id already exists, it will + * be replaced. + * The index will not be created in solr. It will be returned by the manager after adding. + * @param indexId the id of the index, must not be null + * @param index the index to add, must not be null + */ + void addIndex(String indexId, MCRSolrIndex index); + + /** + * Removes the index with the given id from the manager. If no index with the given id exists, + * this method does nothing. + * The index will not be deleted from solr, but will not be returned by the manager anymore. + * @param indexId the id of the index to remove, must not be null + */ + void removeIndex(String indexId); + +} diff --git a/mycore-solr/src/main/java/org/mycore/solr/MCRSolrConstants.java b/mycore-solr/src/main/java/org/mycore/solr/MCRSolrConstants.java index 5aecf8d2f6..0d812236a8 100644 --- a/mycore-solr/src/main/java/org/mycore/solr/MCRSolrConstants.java +++ b/mycore-solr/src/main/java/org/mycore/solr/MCRSolrConstants.java @@ -28,22 +28,16 @@ public class MCRSolrConstants { public static final String SOLR_CONFIG_PREFIX = "MCR.Solr."; - // public static final String SOLR_SERVER_URL; - - public static final String DEFAULT_SOLR_SERVER_URL = MCRConfiguration2 - .getStringOrThrow(SOLR_CONFIG_PREFIX + "ServerURL"); - public static final String SOLR_CORE_PREFIX = SOLR_CONFIG_PREFIX + "Core."; - public static final String SOLR_CORE_NAME_SUFFIX = ".Name"; + public static final String SOLR_COLLECTION_MANAGER_PROPERTY_PREFIX = SOLR_CONFIG_PREFIX + "IndexManager."; - public static final String SOLR_CORE_SERVER_SUFFIX = ".ServerURL"; + public static final String SOLR_COLLECTION_MANAGER_PROPERTY = SOLR_COLLECTION_MANAGER_PROPERTY_PREFIX + + "Class"; - public static final String SOLR_CORE_CONFIGSET_TEMPLATE_SUFFIX = ".ConfigSetTemplate"; + public static final String SOLR_COLLECTION_MANAGER_INDEX_PREFIX = "Index"; - public static final String SOLR_CORE_SHARD_COUNT_SUFFIX = ".ShardCount"; - - public static final String SOLR_CORE_TYPE_SUFFIX = ".Type"; + public static final String SOLR_CORE_NAME_SUFFIX = ".Name"; public static final String SOLR_QUERY_XML_PROTOCOL_VERSION = MCRConfiguration2 .getStringOrThrow(SOLR_CONFIG_PREFIX + "XMLProtocolVersion"); diff --git a/mycore-solr/src/main/java/org/mycore/solr/MCRSolrCore.java b/mycore-solr/src/main/java/org/mycore/solr/MCRSolrCore.java deleted file mode 100644 index 3b0256bc9a..0000000000 --- a/mycore-solr/src/main/java/org/mycore/solr/MCRSolrCore.java +++ /dev/null @@ -1,279 +0,0 @@ -/* - * This file is part of *** M y C o R e *** - * See https://www.mycore.de/ for details. - * - * MyCoRe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * MyCoRe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with MyCoRe. If not, see . - */ - -package org.mycore.solr; - -import static org.mycore.solr.MCRSolrConstants.SOLR_CONFIG_PREFIX; -import static org.mycore.solr.MCRSolrUtils.USE_HTTP_1_1_PROPERTY; - -import java.io.IOException; -import java.util.LinkedHashSet; -import java.util.Objects; -import java.util.Set; -import java.util.concurrent.TimeUnit; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.impl.ConcurrentUpdateBaseSolrClient; -import org.apache.solr.client.solrj.impl.HttpJdkSolrClient; -import org.apache.solr.client.solrj.impl.HttpSolrClientBuilderBase; -import org.apache.solr.client.solrj.jetty.ConcurrentUpdateJettySolrClient; -import org.apache.solr.client.solrj.jetty.HttpJettySolrClient; -import org.apache.solr.client.solrj.request.JavaBinRequestWriter; -import org.mycore.common.config.MCRConfiguration2; -import org.mycore.common.events.MCRShutdownHandler; - -/** - * Core instance of a solr server. - * - * @author Matthias Eichner - */ -public class MCRSolrCore { - - private static final Logger LOGGER = LogManager.getLogger(); - - public static final int DEFAULT_SHARD_COUNT = 1; - - private static final boolean USE_CONCURRENT_SERVER = MCRConfiguration2 - .getOrThrow(SOLR_CONFIG_PREFIX + "ConcurrentUpdateSolrClient.Enabled", Boolean::parseBoolean); - - private static final boolean USE_JETTY_HTTP_CLIENT = MCRConfiguration2 - .getOrThrow(SOLR_CONFIG_PREFIX + "SolrClient.JettyHttpClient.Enabled", Boolean::parseBoolean); - - protected String serverURL; - - protected String name; - - protected String configSet; - - protected Integer shardCount; - - // todo: maybe add support for replicaCount and compositeId if required - - protected SolrClient solrClient; - - protected SolrClient baseSolrClient; - - protected ConcurrentUpdateBaseSolrClient concurrentClient; - - private Set types; - - /** - * Creates a new solr server core instance. - * - * @param serverURL - * base url of the solr server e.g. http://localhost:8296 - * @param name - * name of the core e.g. docportal - * @param configSet - * name of the config set - * @param shardCount - * number of shards - */ - public MCRSolrCore(String serverURL, String name, String configSet, Integer shardCount, - Set types) { - setup(serverURL, name, configSet, shardCount, types); - } - - protected void setup(String serverURL, String name, String configSet, - Integer shardCount, Set types) { - - this.serverURL = serverURL.endsWith("/") ? serverURL : serverURL + "/"; - this.name = name; - this.configSet = configSet; - this.shardCount = Objects.requireNonNull(shardCount, "shardCount must not be null"); - this.types = new LinkedHashSet<>(Objects.requireNonNull(types, "type must not be null")); - String coreURL = getV1CoreURL(); - int connectionTimeout = MCRConfiguration2 - .getOrThrow(SOLR_CONFIG_PREFIX + "SolrClient.ConnectionTimeout", Integer::parseInt); - int socketTimeout = MCRConfiguration2 - .getOrThrow(SOLR_CONFIG_PREFIX + "SolrClient.SocketTimeout", Integer::parseInt); - - // default server - solrClient = getSolrClientInstance(coreURL, connectionTimeout, socketTimeout); - - baseSolrClient = getSolrClientInstance(getServerURL() + "solr/", connectionTimeout, socketTimeout); - - // concurrent server - if (USE_CONCURRENT_SERVER) { - int queueSize = MCRConfiguration2 - .getOrThrow(SOLR_CONFIG_PREFIX + "ConcurrentUpdateSolrClient.QueueSize", Integer::parseInt); - int threadCount = MCRConfiguration2 - .getOrThrow(SOLR_CONFIG_PREFIX + "ConcurrentUpdateSolrClient.ThreadCount", Integer::parseInt); - concurrentClient = new ConcurrentUpdateJettySolrClient.Builder(coreURL, (HttpJettySolrClient) solrClient) - .withQueueSize(queueSize) - .withThreadCount(threadCount) - .build(); - } - // shutdown handler - MCRShutdownHandler.getInstance().addCloseable(new MCRShutdownHandler.Closeable() { - - @Override - public int getPriority() { - return Integer.MIN_VALUE + 5; - } - - @Override - public void close() { - shutdown(); - } - }); - } - - private static SolrClient getSolrClientInstance(String baseSolrUrl, int connectionTimeout, - int socketTimeout) { - HttpSolrClientBuilderBase baseBuilder = useJettyHttpClient() ? new HttpJettySolrClient.Builder(baseSolrUrl) - : new HttpJdkSolrClient.Builder(baseSolrUrl); - - MCRConfiguration2.getBoolean(USE_HTTP_1_1_PROPERTY) - .filter(useHttp11 -> useHttp11) - .ifPresent(useHttp11 -> { - baseBuilder.useHttp1_1(true); - }); - - return baseBuilder - .withConnectionTimeout(connectionTimeout, TimeUnit.MILLISECONDS) - .withIdleTimeout(socketTimeout, TimeUnit.MILLISECONDS) - .withRequestTimeout(socketTimeout, TimeUnit.MILLISECONDS) - .withRequestWriter(new JavaBinRequestWriter()) - .build(); - } - - private static boolean useJettyHttpClient() { - return USE_CONCURRENT_SERVER || USE_JETTY_HTTP_CLIENT; - } - - public String getV1CoreURL() { - return this.serverURL + "solr/" + this.name; - } - - public synchronized void shutdown() { - try { - shutdownGracefully(solrClient); - solrClient = null; - } catch (SolrServerException | IOException e) { - LOGGER.error("Error while shutting down SOLR client.", e); - } - try { - shutdownGracefully(baseSolrClient); - baseSolrClient = null; - } catch (SolrServerException | IOException e) { - LOGGER.error("Error while shutting down SOLR client.", e); - } - try { - shutdownGracefully(concurrentClient); - concurrentClient = null; - } catch (SolrServerException | IOException e) { - LOGGER.error("Error while shutting down SOLR client.", e); - } - LOGGER.info("Solr shutdown process completed."); - } - - private void shutdownGracefully(SolrClient client) throws SolrServerException, IOException { - if (client != null) { - LOGGER.info("Shutting down solr client: {}", client); - client.close(); - } - } - - /** - * Returns the name of the core. - */ - public String getName() { - return name; - } - - public String getServerURL() { - return serverURL; - } - - /** - * Returns the default solr client instance. Use this for queries. - */ - public SolrClient getClient() { - return solrClient; - } - - /** - * Returns the base solr client instance, without core information. Use this for admin operations. - */ - public SolrClient getBaseClient() { - return baseSolrClient; - } - - public String buildRemoteConfigSetName() { - return this.getName() + "_" + this.getConfigSet(); - } - - /** - * Returns the concurrent solr client instance. Use this for indexing. - */ - public SolrClient getConcurrentClient() { - return concurrentClient != null ? concurrentClient : solrClient; - } - - /** - * Returns the ConfigSet assigned in the properties - * @return the ConfigSet - */ - public String getConfigSet() { - return configSet; - } - - /** - * Returns the shard count assigned in the properties - * @return the shard count - */ - public Integer getShardCount() { - return shardCount; - } - - /** - * Sets the shard count, it does not change an existing core, but is used for creating a new core. - * @param shardCount the new shard count - */ - public void setShardCount(Integer shardCount) { - this.shardCount = shardCount; - } - - /** - * Sets the ConfigSet, it does not change an existing core, but is used for creating a new core. - * @param configSet the new ConfigSet - */ - public void setConfigSet(String configSet) { - this.configSet = configSet; - } - - /** - * Sets the server URL, it does not change an existing core, but is used for creating a new core. - * @param serverURL the new server URL - */ - public void setServerURL(String serverURL) { - this.serverURL = serverURL; - } - - /** - * Returns which type of core this is - * @return the type of core - */ - public Set getTypes() { - return types; - } -} diff --git a/mycore-solr/src/main/java/org/mycore/solr/MCRSolrCoreManager.java b/mycore-solr/src/main/java/org/mycore/solr/MCRSolrCoreManager.java deleted file mode 100644 index e8e8c6a037..0000000000 --- a/mycore-solr/src/main/java/org/mycore/solr/MCRSolrCoreManager.java +++ /dev/null @@ -1,185 +0,0 @@ -/* - * This file is part of *** M y C o R e *** - * See https://www.mycore.de/ for details. - * - * MyCoRe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * MyCoRe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with MyCoRe. If not, see . - */ - -package org.mycore.solr; - -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.solr.client.solrj.SolrClient; -import org.mycore.common.config.MCRConfiguration2; -import org.mycore.common.config.MCRConfigurationException; - -/** - * @author shermann - * @author Thomas Scheffler (yagee) - * @author Matthias Eichner - * @author Jens Kupferschmidt - */ -public final class MCRSolrCoreManager { - - private static final Logger LOGGER = LogManager.getLogger(); - - private static Map coreMap; - - // if any of these properties are present for a core, we have to initialize it - private static final Set INITIALIZATION_RELEVANT_PROPERTY_SUFFIXES = Set.of( - MCRSolrConstants.SOLR_CORE_NAME_SUFFIX, - MCRSolrConstants.SOLR_CORE_SERVER_SUFFIX); - - static { - try { - coreMap = Collections.synchronizedMap(loadCoresFromProperties()); - } catch (Exception e) { - LOGGER.error("Exception creating solr client object", e); - } - } - - private MCRSolrCoreManager() { - } - - /** - * MCR.Solr.Core.Main.Name=cmo - * MCR.Solr.Core.Classfication.Name=cmo-classification - * @return a map of all cores defined in the properties. - */ - private static Map loadCoresFromProperties() { - return MCRConfiguration2 - .getPropertiesMap() - .keySet() - .stream() - .filter(p -> p.startsWith(MCRSolrConstants.SOLR_CORE_PREFIX)) - .map(cp -> cp.substring(MCRSolrConstants.SOLR_CORE_PREFIX.length())) - .map(prop -> { - int indexOfDot = prop.indexOf('.'); - if (indexOfDot == -1) { - // only the core id without suffix -> skip - return null; - } - String coreId = prop.substring(0, indexOfDot); - String suffix = prop.substring(indexOfDot); - return INITIALIZATION_RELEVANT_PROPERTY_SUFFIXES.contains(suffix) ? coreId : null; - }) - .filter(Objects::nonNull) - .distinct() - .collect(Collectors.toMap(coreID -> coreID, MCRSolrCoreManager::initializeSolrCore)); - } - - private static MCRSolrCore initializeSolrCore(String coreID) { - final String coreNameKey = MCRSolrConstants.SOLR_CORE_PREFIX + coreID + MCRSolrConstants.SOLR_CORE_NAME_SUFFIX; - final String coreServerKey = MCRSolrConstants.SOLR_CORE_PREFIX + coreID - + MCRSolrConstants.SOLR_CORE_SERVER_SUFFIX; - - final String configSetTemplateKey = MCRSolrConstants.SOLR_CORE_PREFIX + coreID - + MCRSolrConstants.SOLR_CORE_CONFIGSET_TEMPLATE_SUFFIX; - - final String shardCountKey = MCRSolrConstants.SOLR_CORE_PREFIX + coreID - + MCRSolrConstants.SOLR_CORE_SHARD_COUNT_SUFFIX; - - String coreName = MCRConfiguration2.getString(coreNameKey) - .orElseThrow(() -> new MCRConfigurationException("Missing property " + coreNameKey)); - - String coreServer = MCRConfiguration2.getString(coreServerKey) - .orElse(MCRSolrConstants.DEFAULT_SOLR_SERVER_URL); - - String configSetTemplate = MCRConfiguration2.getString(configSetTemplateKey).orElse(null); - - Integer shardCount = MCRConfiguration2.getInt(shardCountKey).orElse(MCRSolrCore.DEFAULT_SHARD_COUNT); - - String coreTypeKey = MCRSolrConstants.SOLR_CORE_PREFIX + coreID + MCRSolrConstants.SOLR_CORE_TYPE_SUFFIX; - Set coreTypes = MCRConfiguration2.getOrThrow(coreTypeKey, MCRConfiguration2::splitValue) - .map(MCRSolrCoreType::new) - .collect(Collectors.toSet()); - - return new MCRSolrCore(coreServer, coreName, configSetTemplate, shardCount, coreTypes); - } - - public static MCRSolrCore addCore(String coreID, MCRSolrCore core) { - coreMap.put(coreID, core); - return core; - } - - /** - * Add a SOLR core instance to the list - * - * @param core the MCRSolrCore instance - */ - public static void add(String coreID, MCRSolrCore core) { - coreMap.put(coreID, core); - } - - /** - * Remove a SOLR core instance from the list - * - * @param coreID the name of the MCRSolrCore instance - */ - public static Optional remove(String coreID) { - return Optional.ofNullable(coreMap.remove(coreID)); - } - - /** - * @param coreID the id of the core - * @return a core with a specific id - */ - public static Optional get(String coreID) { - return Optional.ofNullable(coreMap.get(coreID)); - } - - public static MCRSolrCore getMainSolrCore() { - return get(MCRSolrConstants.MAIN_CORE_TYPE) - .orElseThrow(() -> new MCRConfigurationException("The core main is not configured!")); - } - - /** - * Returns the solr cores which should be used for a specific type. - * @param type the type of the core - * @return a list of cores - */ - public static List getCoresForType(MCRSolrCoreType type) { - return coreMap.values().stream().filter(c -> c.getTypes().contains(type)).collect(Collectors.toList()); - } - - /** - * Returns the solr client of the default core. - */ - public static SolrClient getMainSolrClient() { - return getMainSolrCore().getClient(); - } - - /** - * Returns the concurrent solr client of the default core. - */ - public static SolrClient getMainConcurrentSolrClient() { - return getMainSolrCore().getConcurrentClient(); - } - - /** - * @return the read only core map wich contains the coreId and the core - */ - public static Map getCoreMap() { - return Collections.unmodifiableMap(coreMap); - } - -} diff --git a/mycore-solr/src/main/java/org/mycore/solr/MCRSolrDefaultPropertyConstants.java b/mycore-solr/src/main/java/org/mycore/solr/MCRSolrDefaultPropertyConstants.java new file mode 100644 index 0000000000..e0f67a1294 --- /dev/null +++ b/mycore-solr/src/main/java/org/mycore/solr/MCRSolrDefaultPropertyConstants.java @@ -0,0 +1,37 @@ +/* + * This file is part of *** M y C o R e *** + * See https://www.mycore.de/ for details. + * + * MyCoRe is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * MyCoRe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with MyCoRe. If not, see . + */ + +package org.mycore.solr; + +public class MCRSolrDefaultPropertyConstants { + + public static final String USE_JETTY_HTTP_CLIENT = "MCR.Solr.Default.UseJettyHttpClient"; + + public static final String CLIENT_IDLE_TIMEOUT = "MCR.Solr.Default.Client.IdleTimeout"; + public static final String CLIENT_IDLE_TIMEOUT_UNIT = "MCR.Solr.Default.Client.IdleTimeout.Unit"; + + public static final String CLIENT_CONNECTION_TIMEOUT = "MCR.Solr.Default.Client.ConnectionTimeout"; + public static final String CLIENT_CONNECTION_TIMEOUT_UNIT = "MCR.Solr.Default.Client.ConnectionTimeout.Unit"; + + public static final String CLIENT_REQUEST_TIMEOUT = "MCR.Solr.Default.Client.RequestTimeout"; + public static final String CLIENT_REQUEST_TIMEOUT_UNIT = "MCR.Solr.Default.Client.RequestTimeout.Unit"; + + public static final String DEFAULT_SHARD_COUNT = "MCR.Solr.Default.ShardCount"; + public static final String DEFAULT_REPLICA_COUNT = "MCR.Solr.Default.ReplicaCount"; + +} diff --git a/mycore-solr/src/main/java/org/mycore/solr/MCRSolrIndex.java b/mycore-solr/src/main/java/org/mycore/solr/MCRSolrIndex.java new file mode 100644 index 0000000000..2189c36070 --- /dev/null +++ b/mycore-solr/src/main/java/org/mycore/solr/MCRSolrIndex.java @@ -0,0 +1,73 @@ +/* + * This file is part of *** M y C o R e *** + * See https://www.mycore.de/ for details. + * + * MyCoRe is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * MyCoRe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with MyCoRe. If not, see . + */ + +package org.mycore.solr; + +import java.io.IOException; +import java.util.Optional; +import java.util.Set; + +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.impl.ConcurrentUpdateBaseSolrClient; + +/** + * Describes a Collection or Core on a SolrServer + */ +public interface MCRSolrIndex { + + /** + * The name of the collection/core on the servers + * @return the name + */ + String getName(); + + /** + * Returns a {@link SolrClient} which has this {@link MCRSolrIndex} set as default + * collection/core. + * @return the SolrClient + */ + SolrClient getClient(); + + /** + * Returns a {@link SolrClient} without a default collection/core + * @return the SolrClient + */ + SolrClient getBaseClient(); + + /** + * Returns an optional {@link ConcurrentUpdateBaseSolrClient} if the implementation provides one. + * This client can be used for concurrent updates to the index, which can improve performance in + * certain scenarios. If the implementation does not provide a concurrent client, this method + * will return an empty Optional. + * @return an Optional containing the ConcurrentUpdateBaseSolrClient if available, + * or an empty Optional if not a + */ + Optional getConcurrentClient(); + + /** + * Returns a set of {@link MCRIndexType} which describes the Content stored in this + * collection/core + * @return a set + */ + Set getCoreTypes(); + + /** + * Closes the underlying SolrClient(s) and releases any resources. + */ + void close() throws IOException; +} diff --git a/mycore-solr/src/main/java/org/mycore/solr/MCRSolrIndexManager.java b/mycore-solr/src/main/java/org/mycore/solr/MCRSolrIndexManager.java new file mode 100644 index 0000000000..681415e351 --- /dev/null +++ b/mycore-solr/src/main/java/org/mycore/solr/MCRSolrIndexManager.java @@ -0,0 +1,73 @@ +/* + * This file is part of *** M y C o R e *** + * See https://www.mycore.de/ for details. + * + * MyCoRe is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * MyCoRe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with MyCoRe. If not, see . + */ + +package org.mycore.solr; + +import java.util.List; +import java.util.Optional; + +import org.mycore.common.config.MCRConfiguration2; +import org.mycore.common.config.MCRConfigurationException; + +public interface MCRSolrIndexManager { + + Optional getIndex(String indexId); + + List getIndexWithType(MCRIndexType type); + + default Optional getMainIndex() { + return getIndex(MCRSolrConstants.MAIN_CORE_TYPE); + } + + default MCRSolrIndex requireMainIndex() { + return getMainIndex() + .orElseThrow(() -> new MCRConfigurationException("No main index configured")); + } + + default MCRSolrIndex requireIndex(String indexId) { + return getIndex(indexId) + .orElseThrow(() -> new MCRConfigurationException("No index with id " + indexId + " configured")); + } + + static MCRSolrIndexManager obtainInstance() { + MCRSolrIndexManager result = InstanceHolder.instance; + if (result == null) { + synchronized (InstanceHolder.class) { + result = InstanceHolder.instance; + if (result == null) { + InstanceHolder.instance = MCRConfiguration2.getInstanceOf(MCRSolrIndexManager.class, + MCRSolrConstants.SOLR_COLLECTION_MANAGER_PROPERTY).orElseThrow(); + result = InstanceHolder.instance; + } + } + } + return result; + } + + static void reloadInstance() { + synchronized (InstanceHolder.class) { + + InstanceHolder.instance = MCRConfiguration2.getInstanceOf(MCRSolrIndexManager.class, + MCRSolrConstants.SOLR_COLLECTION_MANAGER_PROPERTY).orElseThrow(); + } + } + + final class InstanceHolder { + private static volatile MCRSolrIndexManager instance; + } +} diff --git a/mycore-solr/src/main/java/org/mycore/solr/MCRSolrUtils.java b/mycore-solr/src/main/java/org/mycore/solr/MCRSolrUtils.java index d7f99daeaf..b9d9b06f59 100644 --- a/mycore-solr/src/main/java/org/mycore/solr/MCRSolrUtils.java +++ b/mycore-solr/src/main/java/org/mycore/solr/MCRSolrUtils.java @@ -20,12 +20,16 @@ import static org.mycore.solr.MCRSolrConstants.SOLR_CONFIG_PREFIX; +import java.io.IOException; import java.net.URLDecoder; import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.nio.charset.StandardCharsets; import java.util.regex.Pattern; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.common.params.ModifiableSolrParams; import org.mycore.common.config.MCRConfiguration2; import org.mycore.common.config.MCRConfigurationException; @@ -37,6 +41,8 @@ */ public class MCRSolrUtils { + private static final Logger LOGGER = LogManager.getLogger(); + public static final String USE_HTTP_1_1_PROPERTY = "MCR.Solr.UseHttp_1_1"; /** @@ -134,4 +140,11 @@ public static ModifiableSolrParams parseQueryString(String query) { return params; } + public static void shutdownSolrClient(SolrClient client) throws IOException { + if (client != null) { + LOGGER.info("Shutting down solr client: {}", client); + client.close(); + } + } + } diff --git a/mycore-solr/src/main/java/org/mycore/solr/MCRXMLFunctions.java b/mycore-solr/src/main/java/org/mycore/solr/MCRXMLFunctions.java index 3deb3928f1..3be9d2b9cc 100644 --- a/mycore-solr/src/main/java/org/mycore/solr/MCRXMLFunctions.java +++ b/mycore-solr/src/main/java/org/mycore/solr/MCRXMLFunctions.java @@ -39,9 +39,14 @@ public class MCRXMLFunctions { * @param q the query to execute (in solr syntax) * * @return the amount of documents matching the given query + * @deprecated the method lacks support for core selection and is used as xalan extension, + * which is deprecated as well. It will be removed in a future release, so please use the + * SolrJ API directly instead or implement a XSLT function which uses uri resolver to do the + * request to the correct core. */ + @Deprecated(forRemoval = true, since = "2026.06.0") public static long getNumFound(String q) throws SolrServerException, IOException { - if (q == null || q.length() == 0) { + if (q == null || q.isEmpty()) { throw new IllegalArgumentException("The query string must not be null"); } SolrQuery solrQuery = new SolrQuery(q); @@ -49,16 +54,22 @@ public static long getNumFound(String q) throws SolrServerException, IOException QueryRequest queryRequest = new QueryRequest(solrQuery); MCRSolrAuthenticationManager.obtainInstance().applyAuthentication(queryRequest, MCRSolrAuthenticationLevel.SEARCH); - QueryResponse queryResponse = queryRequest.process(MCRSolrCoreManager.getMainSolrClient()); + QueryResponse queryResponse = queryRequest.process(MCRSolrIndexManager.obtainInstance() + .requireMainIndex().getClient()); return queryResponse.getResults().getNumFound(); } /** * @param q the query to execute (in solr syntax) * @return the identifier of the first document matching the query + * @deprecated tje method lacks support for core selection and is used as xalan extension, + * which is deprecated as well. It will be removed in a future release, so please use the + * SolrJ API directly instead or implement a XSLT function which uses uri resolver to do the + * request to the correct core. */ + @Deprecated(forRemoval = true, since = "2026.06.0") public static String getIdentifierOfFirst(String q) throws SolrServerException, IOException { - if (q == null || q.length() == 0) { + if (q == null || q.isEmpty()) { throw new IllegalArgumentException("The query string must not be null"); } SolrQuery solrQuery = new SolrQuery(q); @@ -67,7 +78,8 @@ public static String getIdentifierOfFirst(String q) throws SolrServerException, QueryRequest queryRequest = new QueryRequest(solrQuery); MCRSolrAuthenticationManager.obtainInstance().applyAuthentication(queryRequest, MCRSolrAuthenticationLevel.SEARCH); - queryResponse = queryRequest.process(MCRSolrCoreManager.getMainSolrClient()); + queryResponse = queryRequest.process(MCRSolrIndexManager.obtainInstance() + .requireMainIndex().getClient()); if (queryResponse.getResults().getNumFound() == 0) { return null; diff --git a/mycore-solr/src/main/java/org/mycore/solr/auth/MCRSolrAuthenticationManager.java b/mycore-solr/src/main/java/org/mycore/solr/auth/MCRSolrAuthenticationManager.java index 6df4a1ab6c..968f294175 100644 --- a/mycore-solr/src/main/java/org/mycore/solr/auth/MCRSolrAuthenticationManager.java +++ b/mycore-solr/src/main/java/org/mycore/solr/auth/MCRSolrAuthenticationManager.java @@ -25,6 +25,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.solr.client.solrj.SolrRequest; +import org.apache.solr.client.solrj.impl.HttpSolrClientBuilderBase; import org.mycore.common.config.MCRConfiguration2; import org.mycore.common.config.MCRConfigurationException; import org.mycore.solr.MCRSolrConstants; @@ -96,6 +97,16 @@ public void applyAuthentication(HttpRequest.Builder request, MCRSolrAuthenticati authenticators.get(level).applyAuthentication(request); } + /** + * Add authentication to a Solr client builder. + * @param clientBuilder the client builder to add the authentication to + * @param level the level of authentication to add + */ + public void applyAuthentication(HttpSolrClientBuilderBase clientBuilder, MCRSolrAuthenticationLevel level) { + authenticators.get(level).applyAuthentication(clientBuilder); + } + + public static MCRSolrAuthenticationManager obtainInstance() { return LazyInstanceHolder.SINGLETON_INSTANCE; } diff --git a/mycore-solr/src/main/java/org/mycore/solr/auth/MCRSolrAuthenticator.java b/mycore-solr/src/main/java/org/mycore/solr/auth/MCRSolrAuthenticator.java index 8b4a822d06..53d9365fb5 100644 --- a/mycore-solr/src/main/java/org/mycore/solr/auth/MCRSolrAuthenticator.java +++ b/mycore-solr/src/main/java/org/mycore/solr/auth/MCRSolrAuthenticator.java @@ -21,6 +21,7 @@ import org.apache.solr.client.solrj.SolrRequest; import java.net.http.HttpRequest; +import org.apache.solr.client.solrj.impl.HttpSolrClientBuilderBase; /** * Interface for adding authentication to Solr requests. @@ -41,4 +42,10 @@ public interface MCRSolrAuthenticator { */ void applyAuthentication(HttpRequest.Builder request); + /** + * Add authentication to a Solr client. + * @param clientBuilder the client builder to add the authentication to + */ + void applyAuthentication(HttpSolrClientBuilderBase clientBuilder); + } diff --git a/mycore-solr/src/main/java/org/mycore/solr/auth/MCRSolrBasicAuthenticatorBase.java b/mycore-solr/src/main/java/org/mycore/solr/auth/MCRSolrBasicAuthenticatorBase.java index 9698105abc..00e537ff2e 100644 --- a/mycore-solr/src/main/java/org/mycore/solr/auth/MCRSolrBasicAuthenticatorBase.java +++ b/mycore-solr/src/main/java/org/mycore/solr/auth/MCRSolrBasicAuthenticatorBase.java @@ -23,6 +23,7 @@ import java.net.http.HttpRequest; import java.nio.charset.StandardCharsets; import java.util.Base64; +import org.apache.solr.client.solrj.impl.HttpSolrClientBuilderBase; /** * Authentication implementation that adds Authorization headers with @@ -40,6 +41,11 @@ public final void applyAuthentication(HttpRequest.Builder request) { request.header("Authorization", "Basic " + getEncodedAuthString()); } + @Override + public void applyAuthentication(HttpSolrClientBuilderBase clientBuilder) { + clientBuilder.withBasicAuthCredentials(getUsername(), getPassword()); + } + private String getEncodedAuthString() { String authString = getUsername() + ":" + getPassword(); return Base64.getEncoder().encodeToString(authString.getBytes(StandardCharsets.UTF_8)); diff --git a/mycore-solr/src/main/java/org/mycore/solr/auth/MCRSolrNoOpAuthenticator.java b/mycore-solr/src/main/java/org/mycore/solr/auth/MCRSolrNoOpAuthenticator.java index 2ca5fcbe48..f6f8cdba23 100644 --- a/mycore-solr/src/main/java/org/mycore/solr/auth/MCRSolrNoOpAuthenticator.java +++ b/mycore-solr/src/main/java/org/mycore/solr/auth/MCRSolrNoOpAuthenticator.java @@ -21,6 +21,8 @@ import org.apache.solr.client.solrj.SolrRequest; import java.net.http.HttpRequest; +import org.apache.solr.client.solrj.impl.HttpSolrClientBuilderBase; + /** * Authentication implementation that doesn't add any authentication to Solr requests. @@ -35,4 +37,8 @@ public final void applyAuthentication(SolrRequest request) { public final void applyAuthentication(HttpRequest.Builder request) { } + @Override + public void applyAuthentication(HttpSolrClientBuilderBase clientBuilder) { + } + } diff --git a/mycore-solr/src/main/java/org/mycore/solr/classification/MCRSolrCategLinkService.java b/mycore-solr/src/main/java/org/mycore/solr/classification/MCRSolrCategLinkService.java index e94aea486b..adf4edbf08 100644 --- a/mycore-solr/src/main/java/org/mycore/solr/classification/MCRSolrCategLinkService.java +++ b/mycore-solr/src/main/java/org/mycore/solr/classification/MCRSolrCategLinkService.java @@ -31,9 +31,9 @@ import org.mycore.datamodel.classifications2.MCRCategLinkReference; import org.mycore.datamodel.classifications2.MCRCategoryID; import org.mycore.datamodel.classifications2.impl.MCRCategLinkServiceImpl; -import org.mycore.solr.MCRSolrCore; -import org.mycore.solr.MCRSolrCoreManager; -import org.mycore.solr.MCRSolrCoreType; +import org.mycore.solr.MCRIndexType; +import org.mycore.solr.MCRSolrIndex; +import org.mycore.solr.MCRSolrIndexManager; import org.mycore.solr.auth.MCRSolrAuthenticationLevel; import org.mycore.solr.auth.MCRSolrAuthenticationManager; @@ -52,37 +52,45 @@ public class MCRSolrCategLinkService extends MCRCategLinkServiceImpl { public void setLinks(MCRCategLinkReference objectReference, Collection categories) { super.setLinks(objectReference, categories); // solr - List cores = MCRSolrCoreManager.getCoresForType(MCRSolrCoreType.CLASSIFICATION); + List indexList = + MCRSolrIndexManager.obtainInstance().getIndexWithType(MCRIndexType.CLASSIFICATION); List solrDocumentList = MCRSolrClassificationUtil .toSolrDocument(objectReference, categories); - MCRSolrClassificationUtil.bulkIndex(cores, solrDocumentList); + MCRSolrClassificationUtil.bulkIndex(indexList, solrDocumentList); } @Override public void deleteLink(MCRCategLinkReference reference) { super.deleteLink(reference); // solr - try { - SolrClient solrClient = MCRSolrClassificationUtil.getCore().getClient(); - delete(solrClient, reference); - } catch (Exception exc) { - LOGGER.error(() -> "Unable to delete links of object " + reference.getObjectID(), exc); - } + MCRSolrClassificationUtil.getIndexList().forEach(index -> { + try { + SolrClient solrClient = index.getClient(); + delete(solrClient, reference); + } catch (Exception exc) { + LOGGER.error(() -> "Unable to delete links of object " + reference.getObjectID(), exc); + } + }); + } @Override public void deleteLinks(Collection references) { super.deleteLinks(references); // solr - SolrClient solrClient = MCRSolrClassificationUtil.getCore().getClient(); - for (MCRCategLinkReference reference : references) { - try { - delete(solrClient, reference); - } catch (Exception exc) { - LOGGER.error(() -> "Unable to delete links of object " + reference.getObjectID(), exc); + MCRSolrClassificationUtil.getIndexList().forEach(index -> { + SolrClient solrClient = index.getClient(); + for (MCRCategLinkReference reference : references) { + try { + delete(solrClient, reference); + } catch (Exception exc) { + LOGGER.error( + () -> "Unable to delete links of object " + reference.getObjectID(), + exc); + } } - } + }); } /** diff --git a/mycore-solr/src/main/java/org/mycore/solr/classification/MCRSolrClassificationUtil.java b/mycore-solr/src/main/java/org/mycore/solr/classification/MCRSolrClassificationUtil.java index 43ff21d04e..5791aa5389 100644 --- a/mycore-solr/src/main/java/org/mycore/solr/classification/MCRSolrClassificationUtil.java +++ b/mycore-solr/src/main/java/org/mycore/solr/classification/MCRSolrClassificationUtil.java @@ -24,12 +24,10 @@ import java.util.Collection; import java.util.Deque; import java.util.List; -import java.util.Optional; import java.util.stream.Collectors; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.request.UpdateRequest; import org.apache.solr.common.SolrInputDocument; @@ -41,9 +39,9 @@ import org.mycore.datamodel.classifications2.MCRCategoryDAO; import org.mycore.datamodel.classifications2.MCRCategoryDAOFactory; import org.mycore.datamodel.classifications2.MCRCategoryID; -import org.mycore.solr.MCRSolrCore; -import org.mycore.solr.MCRSolrCoreManager; -import org.mycore.solr.MCRSolrUtils; +import org.mycore.solr.MCRIndexType; +import org.mycore.solr.MCRSolrIndex; +import org.mycore.solr.MCRSolrIndexManager; import org.mycore.solr.auth.MCRSolrAuthenticationLevel; import org.mycore.solr.auth.MCRSolrAuthenticationManager; import org.mycore.solr.search.MCRSolrSearchUtils; @@ -59,7 +57,7 @@ public final class MCRSolrClassificationUtil { private static final Logger LOGGER = LogManager.getLogger(); - private static final String CLASSIFICATION_CORE_TYPE = "classification"; + private static final MCRIndexType CLASSIFICATION_INDEX_TYPE = MCRIndexType.CLASSIFICATION; private MCRSolrClassificationUtil() { } @@ -68,15 +66,15 @@ private MCRSolrClassificationUtil() { * Reindex the whole classification system with the default classification solr core. */ public static void rebuildIndex() { - rebuildIndex(List.of(getCore())); + rebuildIndex(getIndexList()); } /** * Reindex the whole classification system. * - * @param cores the solr cores to use + * @param indexList the solr cores to use */ - public static void rebuildIndex(List cores) { + public static void rebuildIndex(List indexList) { LOGGER.info("rebuild classification index..."); // categories MCRCategoryDAO categoryDAO = MCRCategoryDAOFactory.obtainInstance(); @@ -88,14 +86,14 @@ public static void rebuildIndex(List cores) { categoryList.add(rootCategory); List solrDocumentList = toSolrDocument(categoryList); - bulkIndex(cores, solrDocumentList); + bulkIndex(indexList, solrDocumentList); } // links MCRCategLinkService linkService = MCRCategLinkServiceFactory.obtainInstance(); Collection linkTypes = linkService.getTypes(); for (String linkType : linkTypes) { LOGGER.info("rebuild '{}' links...", linkType); - bulkIndex(cores, linkService.getLinks(linkType) + bulkIndex(indexList, linkService.getLinks(linkType) .stream() .map(link -> new MCRSolrCategoryLink(link.getCategory().getId(), link.getObjectReference())) @@ -109,7 +107,7 @@ public static void rebuildIndex(List cores) { * * @param solrDocumentList the list to index */ - public static void bulkIndex(final List client, List solrDocumentList) { + public static void bulkIndex(final List indexList, List solrDocumentList) { MCRSessionMgr.getCurrentSession().onCommit(() -> { List> partitionList = Lists.partition(solrDocumentList, 1000); int docNum = solrDocumentList.size(); @@ -121,8 +119,8 @@ public static void bulkIndex(final List client, List client, List { + try { + UpdateRequest req = new UpdateRequest(); + MCRSolrAuthenticationManager.obtainInstance().applyAuthentication(req, + MCRSolrAuthenticationLevel.INDEX); + req.deleteByQuery("*:*"); + req.process(solrClient); + } catch (Exception exc) { + LOGGER.error("Unable to drop solr classification index", exc); + } + }); } /** @@ -210,31 +210,33 @@ public static List toSolrDocument(MCRCategLinkReference linkR * @param categories the categories to reindex */ public static void reindex(MCRCategory... categories) { - SolrClient solrClient = getCore().getClient(); - for (MCRCategory category : categories) { - if (category == null) { - continue; + // remove all descendants and itself + getIndexList().stream().map(MCRSolrIndex::getClient).forEach(solrClient -> { + for (MCRCategory category : categories) { + if (category == null) { + continue; + } + MCRSolrCategory solrCategory = new MCRSolrCategory(category); + try { + UpdateRequest req = new UpdateRequest(); + req.add(solrCategory.toSolrDocument()); + MCRSolrAuthenticationManager.obtainInstance().applyAuthentication(req, + MCRSolrAuthenticationLevel.INDEX); + req.process(solrClient); + } catch (Exception exc) { + LOGGER.error(() -> "Unable to reindex " + category.getId(), exc); + } } - MCRSolrCategory solrCategory = new MCRSolrCategory(category); try { - UpdateRequest req = new UpdateRequest(); - req.add(solrCategory.toSolrDocument()); - MCRSolrAuthenticationManager.obtainInstance().applyAuthentication(req, - MCRSolrAuthenticationLevel.INDEX); - req.process(solrClient); + UpdateRequest commitRequest = new UpdateRequest(); + commitRequest.setAction(UpdateRequest.ACTION.COMMIT, true, true); + MCRSolrAuthenticationManager.obtainInstance().applyAuthentication(commitRequest, + MCRSolrAuthenticationLevel.ADMIN); + commitRequest.process(solrClient); } catch (Exception exc) { - LOGGER.error(() -> "Unable to reindex " + category.getId(), exc); + LOGGER.error("Unable to commit reindexed categories", exc); } - } - try { - UpdateRequest commitRequest = new UpdateRequest(); - commitRequest.setAction(UpdateRequest.ACTION.COMMIT, true, true); - MCRSolrAuthenticationManager.obtainInstance().applyAuthentication(commitRequest, - MCRSolrAuthenticationLevel.INDEX); - commitRequest.process(solrClient); - } catch (Exception exc) { - LOGGER.error("Unable to commit reindexed categories", exc); - } + }); } /** @@ -263,9 +265,8 @@ public static void reindex(Collection categoryIds) { /** * Returns the solr classification core. */ - public static MCRSolrCore getCore() { - Optional classCore = MCRSolrCoreManager.get(CLASSIFICATION_CORE_TYPE); - return classCore.orElseThrow(() -> MCRSolrUtils.getCoreConfigMissingException(CLASSIFICATION_CORE_TYPE)); + public static List getIndexList() { + return MCRSolrIndexManager.obtainInstance().getIndexWithType(CLASSIFICATION_INDEX_TYPE); } /** @@ -278,36 +279,40 @@ public static String encodeCategoryId(MCRCategoryID classId) { } static void solrDelete(MCRCategoryID id, MCRCategory parent) { - try { - // remove all descendants and itself - SolrClient solrClient = getCore().getClient(); - List toDelete = MCRSolrSearchUtils.listIDs(solrClient, - "ancestor:" + encodeCategoryId(id)); - toDelete.add(id.toString()); - UpdateRequest req = new UpdateRequest(); - req.deleteById(toDelete); - MCRSolrAuthenticationManager.obtainInstance().applyAuthentication(req, - MCRSolrAuthenticationLevel.INDEX); - req.process(solrClient); - // reindex parent - if (parent != null) { - reindex(parent); + // remove all descendants and itself + getIndexList().stream().map(MCRSolrIndex::getClient).forEach(solrClient -> { + try { + List toDelete = MCRSolrSearchUtils.listIDs(solrClient, + "ancestor:" + encodeCategoryId(id)); + toDelete.add(id.toString()); + UpdateRequest req = new UpdateRequest(); + req.deleteById(toDelete); + MCRSolrAuthenticationManager.obtainInstance().applyAuthentication(req, + MCRSolrAuthenticationLevel.INDEX); + req.process(solrClient); + // reindex parent + if (parent != null) { + reindex(parent); + } + } catch (Exception exc) { + LOGGER.error("Solr: unable to delete categories of parent {}", id); } - } catch (Exception exc) { - LOGGER.error("Solr: unable to delete categories of parent {}", id); - } + }); } static void solrMove(MCRCategoryID id, MCRCategoryID newParentID) { - try { - SolrClient solrClient = getCore().getClient(); - List reindexList = MCRSolrSearchUtils.listIDs(solrClient, - "ancestor:" + encodeCategoryId(id)); - reindexList.add(id.toString()); - reindexList.add(newParentID.toString()); - reindex(fromString(reindexList)); - } catch (Exception exc) { - LOGGER.error("Solr: unable to move categories of category {} to {}", id, newParentID); - } + // remove all descendants and itself + getIndexList().stream().map(MCRSolrIndex::getClient).forEach(solrClient -> { + try { + List reindexList = MCRSolrSearchUtils.listIDs(solrClient, + "ancestor:" + encodeCategoryId(id)); + reindexList.add(id.toString()); + reindexList.add(newParentID.toString()); + reindex(fromString(reindexList)); + } catch (Exception exc) { + LOGGER.error("Solr: unable to move categories of category {} to {}", id, + newParentID); + } + }); } } diff --git a/mycore-solr/src/main/java/org/mycore/solr/cloud/collection/MCRConfigurableSolrCloudCollection.java b/mycore-solr/src/main/java/org/mycore/solr/cloud/collection/MCRConfigurableSolrCloudCollection.java new file mode 100644 index 0000000000..3c91a4dd3d --- /dev/null +++ b/mycore-solr/src/main/java/org/mycore/solr/cloud/collection/MCRConfigurableSolrCloudCollection.java @@ -0,0 +1,296 @@ +/* + * This file is part of *** M y C o R e *** + * See https://www.mycore.de/ for details. + * + * MyCoRe is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * MyCoRe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with MyCoRe. If not, see . + */ + +package org.mycore.solr.cloud.collection; + +import static org.mycore.solr.MCRSolrDefaultPropertyConstants.DEFAULT_REPLICA_COUNT; +import static org.mycore.solr.MCRSolrDefaultPropertyConstants.DEFAULT_SHARD_COUNT; + +import java.io.IOException; +import java.util.List; +import java.util.Optional; +import java.util.Set; + +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.impl.CloudHttp2SolrClient; +import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.client.solrj.impl.ConcurrentUpdateBaseSolrClient; +import org.apache.solr.client.solrj.impl.HttpClusterStateProvider; +import org.apache.solr.client.solrj.impl.HttpSolrClientBuilderBase; +import org.apache.solr.client.solrj.jetty.HttpJettySolrClient; +import org.mycore.common.MCRException; +import org.mycore.common.config.MCRConfiguration2; +import org.mycore.common.config.annotation.MCRConfigurationProxy; +import org.mycore.common.config.annotation.MCRProperty; +import org.mycore.solr.MCRAbstractHttpBasedIndexConfigAdapter; +import org.mycore.solr.MCRIndexType; +import org.mycore.solr.MCRSolrUtils; +import org.mycore.solr.auth.MCRSolrAuthenticationLevel; +import org.mycore.solr.auth.MCRSolrAuthenticationManager; +import org.mycore.solr.cloud.collection.MCRConfigurableSolrCloudCollection.ConfigAdapter; + +@MCRConfigurationProxy(proxyClass = ConfigAdapter.class) +public class MCRConfigurableSolrCloudCollection implements MCRSolrCloudCollection { + + private final CloudSolrClient client; + private final CloudSolrClient baseClient; + private final String collectionName; + private final Set coreTypes; + + private final Integer numShards; + private final Integer numNrtReplicas; + private final Integer numTlogReplicas; + private final Integer numPullReplicas; + + private final String configSetTemplate; + + public MCRConfigurableSolrCloudCollection(CloudSolrClient client, CloudSolrClient baseClient, + String collectionName, Set coreTypes, Integer numShards, + Integer numNrtReplicas, Integer numTlogReplicas, Integer numPullReplicas, + String configSetTemplate) { + this.client = client; + this.baseClient = baseClient; + this.collectionName = collectionName; + this.coreTypes = coreTypes; + this.numShards = numShards; + this.numNrtReplicas = numNrtReplicas; + this.numTlogReplicas = numTlogReplicas; + this.numPullReplicas = numPullReplicas; + this.configSetTemplate = configSetTemplate; + } + + @Override + public String getName() { + return this.collectionName; + } + + @Override + public SolrClient getClient() { + return client; + } + + @Override + public SolrClient getBaseClient() { + return baseClient; + } + + @Override + public Optional getConcurrentClient() { + // You should use the normal Cloud Client, when using SolrCloud + return Optional.empty(); + } + + @Override + public Set getCoreTypes() { + return coreTypes; + } + + @Override + public void close() throws IOException { + MCRSolrUtils.shutdownSolrClient(this.client); + MCRSolrUtils.shutdownSolrClient(this.baseClient); + } + + @Override + public Integer getNumShards() { + return numShards; + } + + @Override + public Integer getNumNrtReplicas() { + return numNrtReplicas; + } + + @Override + public Integer getNumTlogReplicas() { + return numTlogReplicas; + } + + @Override + public Integer getNumPullReplicas() { + return numPullReplicas; + } + + @Override + public String getConfigSetTemplate() { + return configSetTemplate; + } + + public static class ConfigAdapter extends + MCRAbstractHttpBasedIndexConfigAdapter { + + private String solrUrls; + private String zkUrls; + private String zkChroot; + private String collectionName; + + private String numShards; + private String numNrtReplicas; + private String numTlogReplicas; + private String numPullReplicas; + private String configSetTemplate; + + public String getNumShards() { + return numShards; + } + + @MCRProperty(name = "NumShards", defaultName = DEFAULT_SHARD_COUNT) + public void setNumShards(String numShards) { + this.numShards = numShards; + } + + public String getNumNrtReplicas() { + return numNrtReplicas; + } + + @MCRProperty(name = "NumNrtReplicas", defaultName = DEFAULT_REPLICA_COUNT) + public void setNumNrtReplicas(String numNrtReplicas) { + this.numNrtReplicas = numNrtReplicas; + } + + public String getNumTlogReplicas() { + return numTlogReplicas; + } + + @MCRProperty(name = "NumTlogReplicas", required = false) + public void setNumTlogReplicas(String numTlogReplicas) { + this.numTlogReplicas = numTlogReplicas; + } + + public String getNumPullReplicas() { + return numPullReplicas; + } + + @MCRProperty(name = "NumPullReplicas", required = false) + public void setNumPullReplicas(String numPullReplicas) { + this.numPullReplicas = numPullReplicas; + } + + public String getCollectionName() { + return collectionName; + } + + @MCRProperty(name = "CollectionName") + public void setCollectionName(String collectionName) { + this.collectionName = collectionName; + } + + public String getZkChroot() { + return zkChroot; + } + + @MCRProperty(name = "ZkChroot", required = false) + public void setZkChroot(String zkChroot) { + this.zkChroot = zkChroot; + } + + public String getZkUrl() { + return zkUrls; + } + + @MCRProperty(name = "ZkUrls", required = false) + public void setZkUrl(String zkUrl) { + this.zkUrls = zkUrl; + } + + public String getSolrUrls() { + return solrUrls; + } + + @MCRProperty(name = "SolrUrls", required = false) + public void setSolrUrls(String solrUrls) { + this.solrUrls = solrUrls; + } + + private ClientPair buildClient() { + CloudHttp2SolrClient.Builder builder; + if (zkUrls != null && !zkUrls.isEmpty()) { + List zkUrlList = MCRConfiguration2.splitValue(zkUrls).toList(); + builder = new CloudHttp2SolrClient.Builder(zkUrlList, Optional.ofNullable(zkChroot)); + } else if (solrUrls != null && !solrUrls.isEmpty()) { + builder = getURLBaseCloudHttp2SolrClientBuilder(); + } else { + throw new IllegalStateException( + "No Solr URLs or Zookeeper URL configured for SolrCloud collection " + collectionName); + } + + HttpSolrClientBuilderBase baseClientBuilder = getBuilder(); + + applySettings(baseClientBuilder); + + if (!(baseClientBuilder instanceof HttpJettySolrClient.Builder http2Builder)) { + throw new IllegalStateException( + "Only HttpJettySolrClient.Builder is supported for SolrCloud collections"); + } + builder.withHttpClientBuilder(http2Builder); + + CloudHttp2SolrClient baseClient = builder.build(); + CloudHttp2SolrClient client = builder.withDefaultCollection(getCollectionName()).build(); + + return new ClientPair(client, baseClient); + } + + private CloudHttp2SolrClient.Builder getURLBaseCloudHttp2SolrClientBuilder() { + // the Http2ClusterStateProvider requires a Http2SolrClient to fetch the cluster state, + // the Client needs admin level authentication to fetch the cluster state + try { + List solrUrlList = MCRConfiguration2.splitValue(solrUrls) + .map(String::trim) + .map(p -> p.endsWith("solr/") ? p : p + "solr/") + .toList(); + HttpJettySolrClient.Builder clusterStateHttpClientBuilder = new HttpJettySolrClient.Builder(); + MCRSolrAuthenticationManager.obtainInstance().applyAuthentication(clusterStateHttpClientBuilder, + MCRSolrAuthenticationLevel.ADMIN); + HttpJettySolrClient clusterStateHttpClient = clusterStateHttpClientBuilder.build(); + HttpClusterStateProvider clusterStateProvider = + new HttpClusterStateProvider<>(solrUrlList, clusterStateHttpClient); + return new CloudHttp2SolrClient.Builder(clusterStateProvider); + } catch (Exception e) { + throw new MCRException("Error building SolrCloud client for collection " + collectionName, e); + } + } + + @Override + public MCRConfigurableSolrCloudCollection get() { + ClientPair clientPair = buildClient(); + + return new MCRConfigurableSolrCloudCollection( + clientPair.client, + clientPair.baseClient, + getCollectionName(), + buildCoreTypes(), + numShards != null ? Integer.parseInt(numShards) : null, + numNrtReplicas != null ? Integer.parseInt(numNrtReplicas) : null, + numTlogReplicas != null ? Integer.parseInt(numTlogReplicas) : null, + numPullReplicas != null ? Integer.parseInt(numPullReplicas) : null, + getConfigSetTemplate()); + } + + public String getConfigSetTemplate() { + return configSetTemplate; + } + + @MCRProperty(name = "ConfigSetTemplate") + public void setConfigSetTemplate(String configSetTemplate) { + this.configSetTemplate = configSetTemplate; + } + + private record ClientPair(CloudHttp2SolrClient client, CloudHttp2SolrClient baseClient) { + } + } +} diff --git a/mycore-solr/src/main/java/org/mycore/solr/cloud/collection/MCRSolrCloudCollection.java b/mycore-solr/src/main/java/org/mycore/solr/cloud/collection/MCRSolrCloudCollection.java new file mode 100644 index 0000000000..a3348ec8ec --- /dev/null +++ b/mycore-solr/src/main/java/org/mycore/solr/cloud/collection/MCRSolrCloudCollection.java @@ -0,0 +1,34 @@ +/* + * This file is part of *** M y C o R e *** + * See https://www.mycore.de/ for details. + * + * MyCoRe is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * MyCoRe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with MyCoRe. If not, see . + */ + +package org.mycore.solr.cloud.collection; + +import org.mycore.solr.MCRSolrIndex; + +public interface MCRSolrCloudCollection extends MCRSolrIndex { + + Integer getNumShards(); + + Integer getNumNrtReplicas(); + + Integer getNumTlogReplicas(); + + Integer getNumPullReplicas(); + + String getConfigSetTemplate(); +} diff --git a/mycore-solr/src/main/java/org/mycore/solr/cloud/collection/MCRSolrCollectionHelper.java b/mycore-solr/src/main/java/org/mycore/solr/cloud/collection/MCRSolrCollectionHelper.java index ce8584a025..3bcf4f8e49 100644 --- a/mycore-solr/src/main/java/org/mycore/solr/cloud/collection/MCRSolrCollectionHelper.java +++ b/mycore-solr/src/main/java/org/mycore/solr/cloud/collection/MCRSolrCollectionHelper.java @@ -26,47 +26,78 @@ import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.response.CollectionAdminResponse; import org.mycore.common.MCRException; -import org.mycore.solr.MCRSolrCore; import org.mycore.solr.auth.MCRSolrAuthenticationLevel; import org.mycore.solr.auth.MCRSolrAuthenticationManager; /** - * Provides helper methods for working with Solr collections (cores). + * Provides helper methods for working with Solr collections. */ public class MCRSolrCollectionHelper { private static final Logger LOGGER = LogManager.getLogger(); - public static void createCollection(MCRSolrCore core) throws SolrServerException, IOException { + /** + * Creates a new collection on the Solr server based on the given collection configuration. + * The collection will be created with the name and shard count specified in the collection + * configuration, and it will use the remote config set specified in the collection + * configuration. + * + * @param collection The collection configuration based on which the collection should be + * created. + * @throws SolrServerException If an error occurs while communicating with the Solr server. + * @throws IOException If an error occurs while communicating with the Solr server. + */ + public static void createCollection(MCRSolrCloudCollection collection) throws SolrServerException, IOException { CollectionAdminRequest.Create collectionCreateRequest = CollectionAdminRequest - .createCollection(core.getName(), core.buildRemoteConfigSetName(), core.getShardCount(), null, null, null); + .createCollection(collection.getName(), buildRemoteConfigSetName(collection), collection.getNumShards(), + collection.getNumNrtReplicas(), collection.getNumTlogReplicas(), collection.getNumPullReplicas()); + MCRSolrAuthenticationManager.obtainInstance().applyAuthentication(collectionCreateRequest, MCRSolrAuthenticationLevel.ADMIN); CollectionAdminResponse collectionAdminResponse = collectionCreateRequest - .process(core.getBaseClient()); + .process(collection.getBaseClient()); if (!collectionAdminResponse.isSuccess()) { - throw new MCRException("Error creating collection " + core.getName() + ": " + + throw new MCRException("Error creating collection " + collection.getName() + ": " + collectionAdminResponse.getErrorMessages()); } - LOGGER.info("Collection {} created.", core::getName); + LOGGER.info("Collection {} created.", collection::getName); } - public static void removeCollection(MCRSolrCore core) throws SolrServerException, + /** + * Removes the given collection from the Solr server. This will delete all data in the + * collection, so use with caution. + * @param collection The collection to be removed. + * @throws SolrServerException If an error occurs while communicating with the Solr server. + * @throws IOException If an error occurs while communicating with the Solr server. + */ + public static void removeCollection(MCRSolrCloudCollection collection) throws SolrServerException, IOException { - CollectionAdminRequest.Delete collectionDeleteReq = CollectionAdminRequest.deleteCollection(core.getName()); + CollectionAdminRequest.Delete collectionDeleteReq = + CollectionAdminRequest.deleteCollection(collection.getName()); MCRSolrAuthenticationManager.obtainInstance().applyAuthentication(collectionDeleteReq, MCRSolrAuthenticationLevel.ADMIN); CollectionAdminResponse collectionAdminResponse = collectionDeleteReq - .process(core.getBaseClient()); + .process(collection.getBaseClient()); if (!collectionAdminResponse.isSuccess()) { - throw new MCRException("Error creating collection " + core.getName() + ": " + + throw new MCRException("Error creating collection " + collection.getName() + ": " + collectionAdminResponse.getErrorMessages()); } - LOGGER.info("Collection {} deleted.", core::getName); + LOGGER.info("Collection {} deleted.", collection::getName); } + /** + * Builds the name of the remote config set for the given collection. It is assumed that the + * remote config set has the same name as the collection, followed by an underscore and the + * config set template name. + * + * @param collection The collection for which the config set name should be built. + * @return The name of the remote config set. + */ + public static String buildRemoteConfigSetName(MCRSolrCloudCollection collection) { + return collection.getName() + "_" + collection.getConfigSetTemplate(); + } } diff --git a/mycore-solr/src/main/java/org/mycore/solr/cloud/configsets/MCRSolrConfigSetHelper.java b/mycore-solr/src/main/java/org/mycore/solr/cloud/configsets/MCRSolrConfigSetHelper.java index daf13b0462..946787e2cb 100644 --- a/mycore-solr/src/main/java/org/mycore/solr/cloud/configsets/MCRSolrConfigSetHelper.java +++ b/mycore-solr/src/main/java/org/mycore/solr/cloud/configsets/MCRSolrConfigSetHelper.java @@ -35,10 +35,11 @@ import org.apache.solr.common.util.ContentStreamBase; import org.mycore.common.config.MCRConfiguration2; import org.mycore.common.config.MCRConfigurationException; +import org.mycore.solr.cloud.collection.MCRSolrCloudCollection; import org.mycore.solr.MCRSolrConstants; -import org.mycore.solr.MCRSolrCore; import org.mycore.solr.auth.MCRSolrAuthenticationLevel; import org.mycore.solr.auth.MCRSolrAuthenticationManager; +import org.mycore.solr.cloud.collection.MCRSolrCollectionHelper; /** * Provides helper methods for working with Solr configuration sets. @@ -58,7 +59,8 @@ public class MCRSolrConfigSetHelper { * @throws URISyntaxException If the URL is invalid. * @throws IOException If an error occurs while fetching the config sets. */ - public static List getRemoteConfigSetNames(MCRSolrCore core) throws URISyntaxException, IOException, + public static List getRemoteConfigSetNames(MCRSolrCloudCollection core) + throws URISyntaxException, IOException, SolrServerException { ConfigSetAdminRequest.List listRequest = new ConfigSetAdminRequest.List(); MCRSolrAuthenticationManager.obtainInstance().applyAuthentication(listRequest, @@ -91,13 +93,13 @@ public static Map getLocalConfigSets() { /** * Transfers a config set to the remote Solr server. - * @param core The core for which the config set should be transferred. + * @param collection The index for which the config set should be transferred. */ - public static void transferConfigSetToRemoteSolrServer(MCRSolrCore core) throws SolrServerException, IOException { - String remoteName = core.buildRemoteConfigSetName(); - + public static void transferConfigSetToRemoteSolrServer(MCRSolrCloudCollection collection) + throws SolrServerException, IOException { + String remoteName = MCRSolrCollectionHelper.buildRemoteConfigSetName(collection); try { - List remoteConfigSetNames = getRemoteConfigSetNames(core); + List remoteConfigSetNames = getRemoteConfigSetNames(collection); if (remoteConfigSetNames.contains(remoteName)) { throw new MCRConfigurationException("Config set " + remoteName + " already exists on the " + "remote Solr server."); @@ -111,7 +113,7 @@ public static void transferConfigSetToRemoteSolrServer(MCRSolrCore core) throws MCRSolrAuthenticationManager.obtainInstance().applyAuthentication(request, MCRSolrAuthenticationLevel.ADMIN); request.setConfigSetName(remoteName); - MCRSolrConfigSetProvider configSetProvider = getLocalConfigSets().get(core.getConfigSet()); + MCRSolrConfigSetProvider configSetProvider = getLocalConfigSets().get(collection.getConfigSetTemplate()); request.setUploadStream(new ContentStreamBase() { @Override @@ -125,7 +127,7 @@ public InputStream getStream() { } }); - ConfigSetAdminResponse uploadResponse = request.process(core.getBaseClient()); + ConfigSetAdminResponse uploadResponse = request.process(collection.getBaseClient()); if (uploadResponse.getStatus() != 0) { throw new MCRConfigurationException("Error while transferring config set to remote Solr server. " + @@ -138,17 +140,17 @@ public InputStream getStream() { /** * Deletes a config set from the remote Solr server. - * @param core The core for which the config set should be deleted. + * @param collection The index for which the config set should be deleted. */ - public static void deleteConfigSetFromRemoteSolrServer(MCRSolrCore core) { + public static void deleteConfigSetFromRemoteSolrServer(MCRSolrCloudCollection collection) { ConfigSetAdminRequest.Delete request = new ConfigSetAdminRequest.Delete(); MCRSolrAuthenticationManager.obtainInstance().applyAuthentication(request, MCRSolrAuthenticationLevel.ADMIN); - request.setConfigSetName(core.buildRemoteConfigSetName()); + request.setConfigSetName(MCRSolrCollectionHelper.buildRemoteConfigSetName(collection)); try { - ConfigSetAdminResponse deleteResponse = request.process(core.getBaseClient()); + ConfigSetAdminResponse deleteResponse = request.process(collection.getBaseClient()); if (deleteResponse.getStatus() != 0) { throw new MCRConfigurationException("Error while deleting config set from remote Solr server. " + "Status code: " + deleteResponse.getStatus() + "\n " + deleteResponse.getErrorMessages()); @@ -157,7 +159,7 @@ public static void deleteConfigSetFromRemoteSolrServer(MCRSolrCore core) { throw new MCRConfigurationException("Error while deleting config set from remote Solr server.", e); } - LOGGER.info("Config set {} deleted from remote Solr server.", core::getConfigSet); + LOGGER.info("Config set {} deleted from remote Solr server.", collection::getConfigSetTemplate); } } diff --git a/mycore-solr/src/main/java/org/mycore/solr/commands/MCRSolrCloudCommands.java b/mycore-solr/src/main/java/org/mycore/solr/commands/MCRSolrCloudCommands.java index 29a872fe98..58110ffe85 100644 --- a/mycore-solr/src/main/java/org/mycore/solr/commands/MCRSolrCloudCommands.java +++ b/mycore-solr/src/main/java/org/mycore/solr/commands/MCRSolrCloudCommands.java @@ -23,14 +23,15 @@ import java.util.List; import java.util.Map; +import java.util.Optional; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.solr.client.solrj.SolrServerException; import org.mycore.frontend.cli.annotation.MCRCommand; import org.mycore.frontend.cli.annotation.MCRCommandGroup; -import org.mycore.solr.MCRSolrCore; -import org.mycore.solr.MCRSolrCoreManager; -import org.mycore.solr.MCRSolrUtils; +import org.mycore.solr.cloud.collection.MCRSolrCloudCollection; +import org.mycore.solr.MCRSolrIndex; +import org.mycore.solr.MCRSolrIndexManager; import org.mycore.solr.cloud.collection.MCRSolrCollectionHelper; import org.mycore.solr.cloud.configsets.MCRSolrConfigSetHelper; import org.mycore.solr.cloud.configsets.MCRSolrConfigSetProvider; @@ -47,12 +48,16 @@ public class MCRSolrCloudCommands { private static final Logger LOGGER = LogManager.getLogger(); @MCRCommand( - syntax = "show remote config sets", - help = "displays the names of the config sets on the remote Solr server", + syntax = "show remote config sets of {0}", + help = "displays the names of the config sets on the remote Solr server were the collection" + + " for the core {0} is located", order = 10) - public static void listRemoteConfig() throws URISyntaxException, IOException, SolrServerException { + public static void listRemoteConfig(String collectionName) throws URISyntaxException, + IOException, SolrServerException { + + MCRSolrCloudCollection collection = obtainCollection(collectionName); List configSetNames = MCRSolrConfigSetHelper - .getRemoteConfigSetNames(MCRSolrCoreManager.getMainSolrCore()); + .getRemoteConfigSetNames(collection); for (String configSetName : configSetNames) { LOGGER.info("Remote ConfigSet: {}", configSetName); @@ -82,9 +87,8 @@ public static void listLocalConfig() { "remote Solr server", order = 30) public static void deleteRemoteConfig(String localCoreName) throws URISyntaxException { - MCRSolrCore core = MCRSolrCoreManager.get(localCoreName) - .orElseThrow(() -> MCRSolrUtils.getCoreConfigMissingException(localCoreName)); - MCRSolrConfigSetHelper.deleteConfigSetFromRemoteSolrServer(core); + MCRSolrCloudCollection collection = obtainCollection(localCoreName); + MCRSolrConfigSetHelper.deleteConfigSetFromRemoteSolrServer(collection); } @MCRCommand( @@ -92,10 +96,10 @@ public static void deleteRemoteConfig(String localCoreName) throws URISyntaxExce help = "looks up in the mycore.properties which config set is used for the core {0} and uploads it to the" + "remote Solr server", order = 40) - public static void uploadLocalConfig(String coreName) throws URISyntaxException, SolrServerException, IOException { - MCRSolrCore core = MCRSolrCoreManager.get(coreName) - .orElseThrow(() -> MCRSolrUtils.getCoreConfigMissingException(coreName)); - String localConfigSetName = core.getConfigSet(); + public static void uploadLocalConfig(String collectionName) + throws URISyntaxException, SolrServerException, IOException { + MCRSolrCloudCollection collection = obtainCollection(collectionName); + String localConfigSetName = collection.getConfigSetTemplate(); MCRSolrConfigSetProvider configSet = MCRSolrConfigSetHelper.getLocalConfigSets().get(localConfigSetName); if (configSet == null) { @@ -103,7 +107,7 @@ public static void uploadLocalConfig(String coreName) throws URISyntaxException, return; } - MCRSolrConfigSetHelper.transferConfigSetToRemoteSolrServer(core); + MCRSolrConfigSetHelper.transferConfigSetToRemoteSolrServer(collection); } @MCRCommand( @@ -111,12 +115,10 @@ public static void uploadLocalConfig(String coreName) throws URISyntaxException, help = "creates a new collection in the remote Solr server using the config set defined in the " + "mycore.properties", order = 60) - public static void createCollection(String localCoreName) - throws URISyntaxException, SolrServerException, IOException { - MCRSolrCore core = MCRSolrCoreManager.get(localCoreName) - .orElseThrow(() -> MCRSolrUtils.getCoreConfigMissingException(localCoreName)); - - MCRSolrCollectionHelper.createCollection(core); + public static void createCollection(String collectionName) + throws SolrServerException, IOException { + MCRSolrCloudCollection collection = obtainCollection(collectionName); + MCRSolrCollectionHelper.createCollection(collection); } @MCRCommand( @@ -124,11 +126,35 @@ public static void createCollection(String localCoreName) help = "remove a collection in the remote Solr server using the config set defined in the " + "mycore.properties", order = 70) - public static void removeCollection(String collectionName) throws URISyntaxException, SolrServerException, + public static void removeCollection(String collectionName) throws SolrServerException, IOException { - MCRSolrCore core = MCRSolrCoreManager.get(collectionName) - .orElseThrow(() -> MCRSolrUtils.getCoreConfigMissingException(collectionName)); - MCRSolrCollectionHelper.removeCollection(core); + MCRSolrCloudCollection collection = obtainCollection(collectionName); + MCRSolrCollectionHelper.removeCollection(collection); + } + + /** + * Helper method to obtain a {@link MCRSolrCloudCollection} for a given collection name. + * The collection name is looked up in the {@link MCRSolrIndexManager} and checked if it is an + * instance of {@link MCRSolrCloudCollection}. + * + * @param collectionName the name of the collection to obtain + * @return the {@link MCRSolrCloudCollection} for the given collection name + * @throws IllegalArgumentException if no index is found for the given collection name or + * if the index is not a SolrCloud collection + */ + private static MCRSolrCloudCollection obtainCollection( + String collectionName) { + Optional optionalIndex = MCRSolrIndexManager.obtainInstance().getIndex(collectionName); + + if (optionalIndex.isEmpty()) { + throw new IllegalArgumentException("No index found for collection name " + collectionName); + } + + MCRSolrIndex index = optionalIndex.get(); + if (!(index instanceof MCRSolrCloudCollection collection)) { + throw new IllegalArgumentException("Index " + collectionName + " is not a SolrCloud collection."); + } + return collection; } } diff --git a/mycore-solr/src/main/java/org/mycore/solr/commands/MCRSolrCommands.java b/mycore-solr/src/main/java/org/mycore/solr/commands/MCRSolrCommands.java index 141053edea..87d930ea84 100644 --- a/mycore-solr/src/main/java/org/mycore/solr/commands/MCRSolrCommands.java +++ b/mycore-solr/src/main/java/org/mycore/solr/commands/MCRSolrCommands.java @@ -26,9 +26,8 @@ import org.mycore.frontend.cli.MCRBasicCommands; import org.mycore.frontend.cli.annotation.MCRCommand; import org.mycore.frontend.cli.annotation.MCRCommandGroup; -import org.mycore.solr.MCRSolrCore; -import org.mycore.solr.MCRSolrCoreManager; -import org.mycore.solr.MCRSolrUtils; +import org.mycore.solr.MCRSolrIndex; +import org.mycore.solr.MCRSolrIndexManager; import org.mycore.solr.classification.MCRSolrClassificationUtil; import org.mycore.solr.index.MCRSolrIndexer; import org.mycore.solr.search.MCRSolrSearchUtils; @@ -48,7 +47,7 @@ public class MCRSolrCommands extends MCRAbstractCommands { help = "rebuilds metadata and content index in Solr for core with the id {0}", order = 110) public static void rebuildMetadataAndContentIndex(String coreIDs) { - List cores = getCoreList(coreIDs); + List cores = getIndexList(coreIDs); MCRSolrIndexer.rebuildMetadataIndex(cores); MCRSolrIndexer.rebuildContentIndex(cores); MCRSolrIndexer.optimize(cores); @@ -59,7 +58,7 @@ public static void rebuildMetadataAndContentIndex(String coreIDs) { help = "rebuilds the metadata index in Solr for all objects of type {0} in core with the id {1}", order = 130) public static void rebuildMetadataIndexType(String type, String coreIDs) { - List cores = getCoreList(coreIDs); + List cores = getIndexList(coreIDs); MCRSolrIndexer.rebuildMetadataIndex(type, cores); } @@ -67,8 +66,8 @@ public static void rebuildMetadataIndexType(String type, String coreIDs) { syntax = "rebuild solr metadata index for all objects of base {0} in core {1}", help = "rebuilds the metadata index in Solr for all objects of base {0} in core with the id {1}", order = 130) - public static void rebuildMetadataIndexBase(String base, String coreIDs) throws Exception { - List cores = getCoreList(coreIDs); + public static void rebuildMetadataIndexBase(String base, String indexName) throws Exception { + List cores = getIndexList(indexName); MCRSolrIndexer.rebuildMetadataIndexForObjectBase(base, cores); } @@ -76,8 +75,8 @@ public static void rebuildMetadataIndexBase(String base, String coreIDs) throws syntax = "rebuild solr metadata index for object {0} in core {1}", help = "rebuilds metadata index in Solr for the given object in core with the id {1}", order = 120) - public static void rebuildMetadataIndexObject(String object, String coreIDs) { - List cores = getCoreList(coreIDs); + public static void rebuildMetadataIndexObject(String object, String indexNames) { + List cores = getIndexList(indexNames); MCRSolrIndexer.rebuildMetadataIndex(Stream.of(object).collect(Collectors.toList()), cores); } @@ -85,8 +84,8 @@ public static void rebuildMetadataIndexObject(String object, String coreIDs) { syntax = "rebuild solr metadata index for selected in core {0}", help = "rebuilds content index in Solr for selected objects and or derivates in core with the id {0}", order = 140) - public static void rebuildMetadataIndexForSelected(String coreIDs) { - List cores = getCoreList(coreIDs); + public static void rebuildMetadataIndexForSelected(String indexName) { + List cores = getIndexList(indexName); List selectedObjects = MCRBasicCommands.getSelectedValues(); MCRSolrIndexer.rebuildMetadataIndex(selectedObjects, cores); } @@ -95,8 +94,8 @@ public static void rebuildMetadataIndexForSelected(String coreIDs) { syntax = "rebuild solr metadata index in core {0}", help = "rebuilds metadata index in Solr in core with the id {0}", order = 150) - public static void rebuildMetadataIndex(String coreIDs) { - List cores = getCoreList(coreIDs); + public static void rebuildMetadataIndex(String indexName) { + List cores = getIndexList(indexName); MCRSolrIndexer.rebuildMetadataIndex(cores); } @@ -105,8 +104,8 @@ public static void rebuildMetadataIndex(String coreIDs) { help = "rebuilds content index in Solr for the all derivates of object with the id {0} " + "in core with the id {1}", order = 160) - public static void rebuildContentIndexObject(String objectID, String coreIDs) { - List cores = getCoreList(coreIDs); + public static void rebuildContentIndexObject(String objectID, String indexName) { + List cores = getIndexList(indexName); MCRSolrIndexer.rebuildContentIndex(Stream.of(objectID).collect(Collectors.toList()), cores); } @@ -114,8 +113,8 @@ public static void rebuildContentIndexObject(String objectID, String coreIDs) { syntax = "rebuild solr content index for selected in core {0}", help = "rebuilds content index in Solr for alll derivates of selected objects in core with the id {0}", order = 170) - public static void rebuildContentIndexForSelected(String coreIDs) { - List cores = getCoreList(coreIDs); + public static void rebuildContentIndexForSelected(String indexName) { + List cores = getIndexList(indexName); List selectedObjects = MCRBasicCommands.getSelectedValues(); MCRSolrIndexer.rebuildContentIndex(selectedObjects, cores); } @@ -124,8 +123,8 @@ public static void rebuildContentIndexForSelected(String coreIDs) { syntax = "rebuild solr content index in core {0}", help = "rebuilds content index in Solr in core with the id {0}", order = 180) - public static void rebuildContentIndex(String coreIDs) { - List cores = getCoreList(coreIDs); + public static void rebuildContentIndex(String indexName) { + List cores = getIndexList(indexName); MCRSolrIndexer.rebuildContentIndex(cores); } @@ -133,17 +132,17 @@ public static void rebuildContentIndex(String coreIDs) { syntax = "rebuild solr classification index in core {0}", help = "rebuilds classification index in Solr in the core with the id {0}", order = 190) - public static void rebuildClassificationIndex(String coreIDs) { - List core = getCoreList(coreIDs); - MCRSolrClassificationUtil.rebuildIndex(core); + public static void rebuildClassificationIndex(String indexName) { + MCRSolrIndex index = MCRSolrIndexManager.obtainInstance().requireIndex(indexName); + MCRSolrClassificationUtil.rebuildIndex(List.of(index)); } @MCRCommand( syntax = "clear solr index in core {0}", help = "deletes all entries from index in Solr in core with the id {0}", order = 210) - public static void dropIndex(String coreID) throws Exception { - MCRSolrCore core = getCore(coreID); + public static void dropIndex(String indexName) throws Exception { + MCRSolrIndex core = getIndex(indexName); MCRSolrIndexer.dropIndex(core.getClient()); } @@ -151,8 +150,8 @@ public static void dropIndex(String coreID) throws Exception { syntax = "delete from solr index all objects of type {0} in core {1}", help = "deletes all objects of type {0} from index in Solr in core with the id {1}", order = 220) - public static void dropIndexByType(String type, String coreID) throws Exception { - MCRSolrCore core = getCore(coreID); + public static void dropIndexByType(String type, String indexName) throws Exception { + MCRSolrIndex core = getIndex(indexName); MCRSolrIndexer.dropIndexByType(type, core.getClient()); } @@ -160,8 +159,8 @@ public static void dropIndexByType(String type, String coreID) throws Exception syntax = "delete from solr index all objects of base {0} in core {1}", help = "deletes all objects of base {0} from index in Solr in core with the id {1}", order = 220) - public static void dropIndexByBase(String base, String coreID) throws Exception { - MCRSolrCore core = getCore(coreID); + public static void dropIndexByBase(String base, String indexName) throws Exception { + MCRSolrIndex core = getIndex(indexName); MCRSolrIndexer.dropIndexByBase(base, core.getClient()); } @@ -169,8 +168,8 @@ public static void dropIndexByBase(String base, String coreID) throws Exception syntax = "delete from solr index object {0} in core {1}", help = "deletes an object with id {0} from index in Solr in core with the id {1}", order = 230) - public static void deleteByIdFromSolr(String objectID, String coreID) { - MCRSolrCore core = getCore(coreID); + public static void deleteByIdFromSolr(String objectID, String indexName) { + MCRSolrIndex core = getIndex(indexName); MCRSolrIndexer.deleteById(core.getClient(), objectID); } @@ -178,8 +177,8 @@ public static void deleteByIdFromSolr(String objectID, String coreID) { syntax = "select objects with solr query {0} in core {1}", help = "selects mcr objects with a solr query {0} in core with the id {1}", order = 310) - public static void selectObjectsWithSolrQuery(String query, String coreID) { - MCRSolrCore core = getCore(coreID); + public static void selectObjectsWithSolrQuery(String query, String indexName) { + MCRSolrIndex core = getIndex(indexName); MCRBasicCommands.setSelectedValues(MCRSolrSearchUtils.listIDs(core.getClient(), query)); } @@ -199,13 +198,12 @@ public static void selectObjectsWithSolrQuery(String query, String coreID) { + "since it is very expensive and involves reading and re-writing the entire index", order = 410) public static void optimize(String coreID) { - List cores = getCoreList(coreID); + List cores = getIndexList(coreID); MCRSolrIndexer.optimize(cores); } - private static MCRSolrCore getCore(String coreID) { - return MCRSolrCoreManager.get(coreID) - .orElseThrow(() -> MCRSolrUtils.getCoreConfigMissingException(coreID)); + private static MCRSolrIndex getIndex(String indexName) { + return MCRSolrIndexManager.obtainInstance().requireIndex(indexName); } @MCRCommand( @@ -213,7 +211,7 @@ private static MCRSolrCore getCore(String coreID) { help = "synchronizes the MyCoRe store and index in Solr in core with the id {1} for objects of type {0}", order = 420) public static void synchronizeMetadataIndex(String objectType, String coreID) throws Exception { - List cores = getCoreList(coreID); + List cores = getIndexList(coreID); MCRSolrIndexer.synchronizeMetadataIndex(cores, objectType); } @@ -222,7 +220,7 @@ public static void synchronizeMetadataIndex(String objectType, String coreID) th help = "synchronizes the MyCoRe store and index in Solr in core with the id {1} for objects of base {0}", order = 420) public static void synchronizeMetadataIndexForObjectBase(String objectBase, String coreIDs) throws Exception { - List core = getCoreList(coreIDs); + List core = getIndexList(coreIDs); MCRSolrIndexer.synchronizeMetadataIndexForObjectBase(core, objectBase); } @@ -231,12 +229,12 @@ public static void synchronizeMetadataIndexForObjectBase(String objectBase, Stri help = "synchronizes the MyCoRe store and index in Solr in core with the id {0}", order = 430) public static void synchronizeMetadataIndex(String coreIDs) throws Exception { - List cores = getCoreList(coreIDs); + List cores = getIndexList(coreIDs); MCRSolrIndexer.synchronizeMetadataIndex(cores); } - private static List getCoreList(String coreIDs) { - return Stream.of(coreIDs.split("[, ]")).map(MCRSolrCommands::getCore) + private static List getIndexList(String indexNames) { + return Stream.of(indexNames.split("[, ]")).map(MCRSolrCommands::getIndex) .toList(); } } diff --git a/mycore-solr/src/main/java/org/mycore/solr/commands/MCRSolrCoreAdminCommands.java b/mycore-solr/src/main/java/org/mycore/solr/commands/MCRSolrCoreAdminCommands.java index b4cfa3e72e..293683485b 100644 --- a/mycore-solr/src/main/java/org/mycore/solr/commands/MCRSolrCoreAdminCommands.java +++ b/mycore-solr/src/main/java/org/mycore/solr/commands/MCRSolrCoreAdminCommands.java @@ -18,32 +18,17 @@ package org.mycore.solr.commands; -import static org.mycore.solr.MCRSolrConstants.DEFAULT_SOLR_SERVER_URL; -import static org.mycore.solr.MCRSolrConstants.SOLR_CORE_CONFIGSET_TEMPLATE_SUFFIX; -import static org.mycore.solr.MCRSolrConstants.SOLR_CORE_NAME_SUFFIX; -import static org.mycore.solr.MCRSolrConstants.SOLR_CORE_PREFIX; -import static org.mycore.solr.MCRSolrConstants.SOLR_CORE_SERVER_SUFFIX; -import static org.mycore.solr.MCRSolrConstants.SOLR_CORE_SHARD_COUNT_SUFFIX; -import static org.mycore.solr.MCRSolrConstants.SOLR_CORE_TYPE_SUFFIX; - import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardCopyOption; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.Locale; import java.util.Map; -import java.util.stream.Collectors; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.mycore.frontend.cli.annotation.MCRCommand; import org.mycore.frontend.cli.annotation.MCRCommandGroup; -import org.mycore.solr.MCRSolrCore; -import org.mycore.solr.MCRSolrCoreManager; -import org.mycore.solr.MCRSolrCoreType; import org.mycore.solr.cloud.configsets.MCRSolrConfigSetHelper; import org.mycore.solr.cloud.configsets.MCRSolrConfigSetProvider; import org.mycore.solr.schema.MCRSolrConfigReloader; @@ -55,124 +40,6 @@ public class MCRSolrCoreAdminCommands { private static final Logger LOGGER = LogManager.getLogger(); - /** - * This command displays the MyCoRe properties, - * which have to be set, to reload the current Solr / Solr Core configuration - * on the next start of the MyCoRe application. - */ - @MCRCommand( - syntax = "show solr configuration", - help = "displays MyCoRe properties for the current Solr configuration", - order = 10) - public static void listConfig() { - LOGGER.info("List core configuration: {}", () -> getSolrConfiguration().entrySet().stream() - .map(entry -> String.format(Locale.ROOT, "%s=%s", entry.getKey(), entry.getValue())) - .collect(Collectors.joining("\n"))); - } - - @MCRCommand( - syntax = "register solr core with name {0} as core {1}", - help = "registers a Solr core on the configured default Solr server within MyCoRe", - order = 50) - public static void registerSolrCore(String remoteCoreName, String coreID) { - MCRSolrCore core = new MCRSolrCore(DEFAULT_SOLR_SERVER_URL, remoteCoreName, null, 1, Collections.emptySet()); - MCRSolrCoreManager.addCore(coreID, core); - } - - @MCRCommand( - syntax = "add type {0} to solr core {1}", - help = "Adds a type to a solr core, so that documents of this type are indexed in this core. " + - "Type may be main or classification.", - order = 65) - public static void addTypeToSolrCore(String type, String coreID) { - MCRSolrCore core = MCRSolrCoreManager.get(coreID).orElseThrow(); - core.getTypes().add(new MCRSolrCoreType(type)); - } - - @MCRCommand( - syntax = "switch solr core {0} with core {1}", - help = "switches between two Solr core configurations", - order = 60) - public static void switchSolrCore(String coreID1, String coreID2) { - - MCRSolrCore core1 = MCRSolrCoreManager.get(coreID1).orElseThrow(); - MCRSolrCore core2 = MCRSolrCoreManager.get(coreID2).orElseThrow(); - - MCRSolrCoreManager.add(coreID1, core2); - MCRSolrCoreManager.add(coreID2, core1); - } - - @MCRCommand( - syntax = "remove type {0} from solr core {1}", - help = "Removes a type from a solr core, so that documents of this type are not indexed in this core" + - "anymore. Type may be main or classification.", - order = 66) - public static void removeTypeFromSolrCore(String type, String coreID) { - MCRSolrCore core = MCRSolrCoreManager.get(coreID).orElseThrow(); - core.getTypes().remove(new MCRSolrCoreType(type)); - } - - @MCRCommand( - syntax = "set shard count {0} for solr core {1}", - help = "Sets the number of shards for a solr core, the shard count is not updated in the solr server, but " + - "used if the core is created", - order = 70) - public static void setShardCountForSolrCore(int shardCount, String coreID) { - MCRSolrCore core = MCRSolrCoreManager.get(coreID).orElseThrow(); - core.setShardCount(shardCount); - } - - @MCRCommand(syntax = "set configset {0} for solr core {1}", - help = "Sets the configset for a solr core, the configset is not updated in the solr server, but " + - "used if the core is created", - order = 75) - public static void setConfigSetForSolrCore(String configSet, String coreID) { - MCRSolrCore core = MCRSolrCoreManager.get(coreID).orElseThrow(); - core.setConfigSet(configSet); - } - - @MCRCommand( - syntax = "set server {0} for solr core {1}", - help = "Sets the server for a solr core, the server is not updated in the solr server, but " + - "used if the core is created", - order = 80) - public static void setServerForSolrCore(String server, String coreID) { - MCRSolrCore core = MCRSolrCoreManager.get(coreID).orElseThrow(); - core.setServerURL(server); - } - - public static Map getSolrConfiguration() { - Map solrConfiguration = new LinkedHashMap<>(); - - MCRSolrCoreManager.getCoreMap().entrySet().stream().forEach(entry -> { - String coreID = entry.getKey(); - MCRSolrCore core = entry.getValue(); - solrConfiguration.put(SOLR_CORE_PREFIX + coreID + SOLR_CORE_NAME_SUFFIX, core.getName()); - - if (!DEFAULT_SOLR_SERVER_URL.equals(core.getServerURL())) { - solrConfiguration.put(SOLR_CORE_PREFIX + coreID + SOLR_CORE_SERVER_SUFFIX, core.getServerURL()); - } - - if (core.getConfigSet() != null) { - solrConfiguration.put(SOLR_CORE_PREFIX + coreID + SOLR_CORE_CONFIGSET_TEMPLATE_SUFFIX, - core.getConfigSet()); - } - - if (core.getShardCount() > 1) { - solrConfiguration.put(SOLR_CORE_PREFIX + coreID + SOLR_CORE_SHARD_COUNT_SUFFIX, - Integer.toString(core.getShardCount())); - } - - if (!core.getTypes().isEmpty()) { - String coreTypes = core.getTypes().stream().map(MCRSolrCoreType::name) - .collect(Collectors.joining(",")); - solrConfiguration.put(SOLR_CORE_PREFIX + coreID + SOLR_CORE_TYPE_SUFFIX, coreTypes); - } - }); - - return solrConfiguration; - } - /** * This command recreates the managed-schema.xml and solrconfig.xml files. First it removes all * schema definitions, except for some MyCoRe default types and fields. Second it parses the available diff --git a/mycore-solr/src/main/java/org/mycore/solr/common/xml/MCRSolrFieldsResolver.java b/mycore-solr/src/main/java/org/mycore/solr/common/xml/MCRSolrFieldsResolver.java index 453839551f..54e6b9f3aa 100644 --- a/mycore-solr/src/main/java/org/mycore/solr/common/xml/MCRSolrFieldsResolver.java +++ b/mycore-solr/src/main/java/org/mycore/solr/common/xml/MCRSolrFieldsResolver.java @@ -17,8 +17,8 @@ import org.jdom2.Document; import org.jdom2.Element; import org.jdom2.transform.JDOMSource; -import org.mycore.solr.MCRSolrCore; -import org.mycore.solr.MCRSolrCoreManager; +import org.mycore.solr.MCRSolrIndex; +import org.mycore.solr.MCRSolrIndexManager; import org.mycore.solr.auth.MCRSolrAuthenticationLevel; import org.mycore.solr.auth.MCRSolrAuthenticationManager; @@ -72,16 +72,15 @@ public Source resolve(String href, String base) throws TransformerException { } String solrCore = extractSolrCore(href); - - Optional mayCore = MCRSolrCoreManager.get(solrCore); + Optional mayCore = MCRSolrIndexManager.obtainInstance().getIndex(solrCore); if (mayCore.isEmpty()) { throw new TransformerException("MCRSolrCore not found: " + solrCore); } - MCRSolrCore core = mayCore.get(); + MCRSolrIndex index = mayCore.get(); - SolrClient client = core.getClient(); + SolrClient client = index.getClient(); LukeRequest lukeRequest = new LukeRequest(); MCRSolrAuthenticationManager.obtainInstance().applyAuthentication(lukeRequest, diff --git a/mycore-solr/src/main/java/org/mycore/solr/common/xml/MCRSolrQueryResolver.java b/mycore-solr/src/main/java/org/mycore/solr/common/xml/MCRSolrQueryResolver.java index ee64200ea6..79ef932018 100644 --- a/mycore-solr/src/main/java/org/mycore/solr/common/xml/MCRSolrQueryResolver.java +++ b/mycore-solr/src/main/java/org/mycore/solr/common/xml/MCRSolrQueryResolver.java @@ -32,8 +32,8 @@ import org.apache.solr.common.params.ModifiableSolrParams; import org.mycore.common.MCRException; import org.mycore.common.content.MCRByteContent; -import org.mycore.solr.MCRSolrCore; -import org.mycore.solr.MCRSolrCoreManager; +import org.mycore.solr.MCRSolrIndex; +import org.mycore.solr.MCRSolrIndexManager; import org.mycore.solr.MCRSolrUtils; import org.mycore.solr.search.MCRSolrSearchUtils; @@ -85,9 +85,10 @@ public Source resolve(String href, String base) { throw new IllegalArgumentException("Empty query for: " + href); } - SolrClient client = core.flatMap(MCRSolrCoreManager::get) - .map(MCRSolrCore::getClient) - .orElse(MCRSolrCoreManager.getMainSolrClient()); + MCRSolrIndexManager im = MCRSolrIndexManager.obtainInstance(); + SolrClient client = core.flatMap(im::getIndex) + .map(MCRSolrIndex::getClient) + .orElseGet(() -> im.requireMainIndex().getClient()); ModifiableSolrParams params = MCRSolrUtils.parseQueryString(query.get()); try { diff --git a/mycore-solr/src/main/java/org/mycore/solr/idmapper/MCRSolrIDMapper.java b/mycore-solr/src/main/java/org/mycore/solr/idmapper/MCRSolrIDMapper.java index bd89e09750..f37ba535ab 100644 --- a/mycore-solr/src/main/java/org/mycore/solr/idmapper/MCRSolrIDMapper.java +++ b/mycore-solr/src/main/java/org/mycore/solr/idmapper/MCRSolrIDMapper.java @@ -25,6 +25,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.request.QueryRequest; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.client.solrj.util.ClientUtils; @@ -35,7 +36,7 @@ import org.mycore.datamodel.metadata.MCRObjectID; import org.mycore.frontend.idmapper.MCRDefaultIDMapper; import org.mycore.frontend.idmapper.MCRIDMapper; -import org.mycore.solr.MCRSolrCoreManager; +import org.mycore.solr.MCRSolrIndexManager; import org.mycore.solr.auth.MCRSolrAuthenticationLevel; import org.mycore.solr.auth.MCRSolrAuthenticationManager; @@ -101,7 +102,9 @@ private String retrieveMCRObjectIDfromSOLR(String mcrid) { QueryRequest queryRequest = new QueryRequest(params); MCRSolrAuthenticationManager.obtainInstance().applyAuthentication(queryRequest, MCRSolrAuthenticationLevel.SEARCH); - solrResponse = queryRequest.process(MCRSolrCoreManager.getMainSolrClient()); + SolrClient client = MCRSolrIndexManager.obtainInstance().requireMainIndex() + .getClient(); + solrResponse = queryRequest.process(client); } catch (Exception e) { LOGGER.error("Error retrieving object id from SOLR", e); } @@ -146,7 +149,9 @@ private String retrieveMCRDerivateIDfromSOLR(MCRObjectID mcrObjId, String derid) QueryRequest queryRequest = new QueryRequest(params); MCRSolrAuthenticationManager.obtainInstance().applyAuthentication(queryRequest, MCRSolrAuthenticationLevel.SEARCH); - solrResponse = queryRequest.process(MCRSolrCoreManager.getMainSolrClient()); + SolrClient client = MCRSolrIndexManager.obtainInstance().requireMainIndex() + .getClient(); + solrResponse = queryRequest.process(client); } catch (Exception e) { LOGGER.error("Error retrieving derivate id from SOLR", e); } diff --git a/mycore-solr/src/main/java/org/mycore/solr/index/MCRSolrIndexEventHandler.java b/mycore-solr/src/main/java/org/mycore/solr/index/MCRSolrIndexEventHandler.java index 2e4d11bc40..605a3eb319 100644 --- a/mycore-solr/src/main/java/org/mycore/solr/index/MCRSolrIndexEventHandler.java +++ b/mycore-solr/src/main/java/org/mycore/solr/index/MCRSolrIndexEventHandler.java @@ -47,9 +47,9 @@ import org.mycore.datamodel.metadata.MCRObject; import org.mycore.datamodel.metadata.MCRObjectID; import org.mycore.datamodel.niofs.MCRPath; -import org.mycore.solr.MCRSolrCore; -import org.mycore.solr.MCRSolrCoreManager; -import org.mycore.solr.MCRSolrCoreType; +import org.mycore.solr.MCRIndexType; +import org.mycore.solr.MCRSolrIndex; +import org.mycore.solr.MCRSolrIndexManager; import org.mycore.solr.index.handlers.MCRSolrIndexHandlerFactory; import org.mycore.util.concurrent.MCRDelayedRunnable; import org.mycore.util.concurrent.MCRTransactionableRunnable; @@ -232,7 +232,8 @@ protected void handlePathDeleted(MCREvent evt, Path file, BasicFileAttributes at protected void updateDerivateFileIndex(MCREvent evt, MCRDerivate derivate) { MCRSessionMgr.getCurrentSession().onCommit(() -> { //MCR-2349 initialize solr client early enough - final List mainSolrCores = MCRSolrCoreManager.getCoresForType(MCRSolrCoreType.MAIN); + final List mainSolrCores = + MCRSolrIndexManager.obtainInstance().getIndexWithType(MCRIndexType.MAIN); putIntoTaskQueue(new MCRDelayedRunnable("updateDerivateFileIndex_" + derivate.getId().toString(), DELAY_IN_MS, new MCRTransactionableRunnable( @@ -284,7 +285,8 @@ protected synchronized void solrDelete(MCRObjectID id) { LOGGER.debug("Solr: submitting data of \"{}\" for deleting", id); MCRSessionMgr.getCurrentSession().onCommit(() -> { //MCR-2349 initialize solr client early enough - final SolrClient mainSolrClient = MCRSolrCoreManager.getMainSolrClient(); + SolrClient mainSolrClient = MCRSolrIndexManager.obtainInstance().requireMainIndex() + .getClient(); putIntoTaskQueue(new MCRDelayedRunnable(id.toString(), DELAY_IN_MS, new MCRTransactionableRunnable(() -> MCRSolrIndexer.deleteById(mainSolrClient, id.toString())))); }); @@ -294,7 +296,8 @@ protected synchronized void deleteDerivate(MCRDerivate derivate) { LOGGER.debug("Solr: submitting data of \"{}\" for derivate", derivate::getId); MCRSessionMgr.getCurrentSession().onCommit(() -> { //MCR-2349 initialize solr client early enough - final SolrClient mainSolrClient = MCRSolrCoreManager.getMainSolrClient(); + SolrClient mainSolrClient = MCRSolrIndexManager.obtainInstance().requireMainIndex() + .getClient(); putIntoTaskQueue(new MCRDelayedRunnable(derivate.getId().toString(), DELAY_IN_MS, new MCRTransactionableRunnable( () -> MCRSolrIndexer.deleteDerivate(mainSolrClient, derivate.getId().toString())))); @@ -334,7 +337,8 @@ protected synchronized void removeFile(Path file) { } MCRSessionMgr.getCurrentSession().onCommit(() -> { //MCR-2349 initialize solr client early enough - final SolrClient mainSolrClient = MCRSolrCoreManager.getMainSolrClient(); + SolrClient mainSolrClient = MCRSolrIndexManager.obtainInstance().requireMainIndex() + .getClient(); putIntoTaskQueue( new MCRDelayedRunnable(file.toUri().toString(), DELAY_IN_MS, new MCRTransactionableRunnable(() -> { UpdateResponse updateResponse = MCRSolrIndexer diff --git a/mycore-solr/src/main/java/org/mycore/solr/index/MCRSolrIndexHandler.java b/mycore-solr/src/main/java/org/mycore/solr/index/MCRSolrIndexHandler.java index e2c931caf6..a8245569ca 100644 --- a/mycore-solr/src/main/java/org/mycore/solr/index/MCRSolrIndexHandler.java +++ b/mycore-solr/src/main/java/org/mycore/solr/index/MCRSolrIndexHandler.java @@ -22,7 +22,7 @@ import java.util.List; import org.apache.solr.client.solrj.SolrServerException; -import org.mycore.solr.MCRSolrCoreType; +import org.mycore.solr.MCRIndexType; import org.mycore.solr.index.statistic.MCRSolrIndexStatistic; /** @@ -54,7 +54,7 @@ public interface MCRSolrIndexHandler { int getCommitWithin(); - void setCoreType(MCRSolrCoreType coreType); + void setIndexType(MCRIndexType indexType); MCRSolrIndexStatistic getStatistic(); diff --git a/mycore-solr/src/main/java/org/mycore/solr/index/MCRSolrIndexer.java b/mycore-solr/src/main/java/org/mycore/solr/index/MCRSolrIndexer.java index f1c5d263a3..58a5aaac38 100644 --- a/mycore-solr/src/main/java/org/mycore/solr/index/MCRSolrIndexer.java +++ b/mycore-solr/src/main/java/org/mycore/solr/index/MCRSolrIndexer.java @@ -60,7 +60,7 @@ import org.mycore.datamodel.metadata.MCRExpandedObject; import org.mycore.datamodel.metadata.MCRMetadataManager; import org.mycore.datamodel.metadata.MCRObjectID; -import org.mycore.solr.MCRSolrCore; +import org.mycore.solr.MCRSolrIndex; import org.mycore.solr.MCRSolrUtils; import org.mycore.solr.auth.MCRSolrAuthenticationLevel; import org.mycore.solr.auth.MCRSolrAuthenticationManager; @@ -212,7 +212,7 @@ private void waitForShutdown(ExecutorService service) { * @throws SolrServerException solr server exception * @throws IOException io exception */ - public static void deleteOrphanedNestedDocuments(List solrCores) + public static void deleteOrphanedNestedDocuments(List indexList) throws SolrServerException, IOException { if (!MCRSolrUtils.useNestedDocuments()) { return; @@ -221,8 +221,8 @@ public static void deleteOrphanedNestedDocuments(List solrCores) req.deleteByQuery("-({!join from=id to=_root_ score=none}_root_:*) +_root_:*"); req.setCommitWithin(0); SOLR_AUTHENTICATION_MANAGER.applyAuthentication(req, MCRSolrAuthenticationLevel.INDEX); - for (MCRSolrCore solrCore : solrCores) { - req.process(solrCore.getClient()); + for (MCRSolrIndex solrIndex : indexList) { + req.process(solrIndex.getClient()); } } @@ -319,7 +319,7 @@ private static void commit(SolrClient solrClient) throws SolrServerException, IO UpdateRequest commitRequest = new UpdateRequest(); commitRequest.setAction(UpdateRequest.ACTION.COMMIT, true, true); SOLR_AUTHENTICATION_MANAGER.applyAuthentication(commitRequest, - MCRSolrAuthenticationLevel.INDEX); + MCRSolrAuthenticationLevel.ADMIN); commitRequest.process(solrClient); } @@ -336,8 +336,8 @@ protected static boolean useNestedDocuments() { /** * Rebuilds solr's metadata index. */ - public static void rebuildMetadataIndex(List solrCores) { - rebuildMetadataIndex(MCRXMLMetadataManager.obtainInstance().listIDs(), solrCores); + public static void rebuildMetadataIndex(List indexList) { + rebuildMetadataIndex(MCRXMLMetadataManager.obtainInstance().listIDs(), indexList); } /** @@ -346,9 +346,9 @@ public static void rebuildMetadataIndex(List solrCores) { * @param type * of the objects to index */ - public static void rebuildMetadataIndex(String type, List solrCores) { + public static void rebuildMetadataIndex(String type, List indexList) { List identfiersOfType = MCRXMLMetadataManager.obtainInstance().listIDsOfType(type); - rebuildMetadataIndex(identfiersOfType, solrCores); + rebuildMetadataIndex(identfiersOfType, indexList); } /** @@ -357,9 +357,9 @@ public static void rebuildMetadataIndex(String type, List solrCores * @param base * of the objects to index */ - public static void rebuildMetadataIndexForObjectBase(String base, List solrCores) { + public static void rebuildMetadataIndexForObjectBase(String base, List indexList) { List identfiersOfBase = MCRXMLMetadataManager.obtainInstance().listIDsForBase(base); - rebuildMetadataIndex(identfiersOfBase, solrCores); + rebuildMetadataIndex(identfiersOfBase, indexList); } /** @@ -367,10 +367,10 @@ public static void rebuildMetadataIndexForObjectBase(String base, List list, List solrCores) { + public static void rebuildMetadataIndex(List list, List indexList) { LOGGER.info("Re-building Metadata Index"); if (list.isEmpty()) { LOGGER.info("Sorry, no documents to index"); @@ -398,7 +398,7 @@ public static void rebuildMetadataIndex(List list, List sol MCRSolrIndexHandler indexHandler = MCRSolrIndexHandlerFactory.obtainInstance() .getIndexHandler(contentMap); if (indexHandler instanceof MCRSolrAbstractIndexHandler aie) { - aie.setDestinationCores(solrCores); + aie.setDestinationIndex(indexList); } else { throw new MCRException("indexHandler " + indexHandler.getClass().getName() + " does not support indexing with cores"); @@ -421,33 +421,33 @@ public static void rebuildMetadataIndex(List list, List sol /** * Rebuilds solr's content index. */ - public static void rebuildContentIndex(List cores) { - rebuildContentIndex(MCRXMLMetadataManager.obtainInstance().listIDsOfType(MCRDerivate.OBJECT_TYPE), cores); + public static void rebuildContentIndex(List indexList) { + rebuildContentIndex(MCRXMLMetadataManager.obtainInstance().listIDsOfType(MCRDerivate.OBJECT_TYPE), indexList); } /** * Rebuilds solr's content index. * - * @param cores + * @param indexList * the solr cores to send the objects to * @param list * list of mycore object id's */ - public static void rebuildContentIndex(List list, List cores) { - rebuildContentIndex(list, cores, LOW_PRIORITY); + public static void rebuildContentIndex(List list, List indexList) { + rebuildContentIndex(list, indexList, LOW_PRIORITY); } /** * Rebuilds solr's content index. * - * @param cores + * @param indexList * the solr cores to send the objects to * @param list * list of mycore object id's * @param priority * higher priority means earlier execution */ - public static void rebuildContentIndex(List list, List cores, int priority) { + public static void rebuildContentIndex(List list, List indexList, int priority) { LOGGER.info("Re-building Content Index"); if (list.isEmpty()) { @@ -461,7 +461,7 @@ public static void rebuildContentIndex(List list, List core for (String id : list) { MCRSolrFilesIndexHandler indexHandler = new MCRSolrFilesIndexHandler(id); indexHandler.setCommitWithin(BATCH_AUTO_COMMIT_WITHIN_MS); - indexHandler.setDestinationCores(cores); + indexHandler.setDestinationIndex(indexList); submitIndexHandler(indexHandler, priority); } long tStop = System.currentTimeMillis(); @@ -561,10 +561,10 @@ public static void dropIndexByBase(String base, SolrClient client) throws Except /** * Sends a signal to the remote solr server to optimize its index. */ - public static void optimize(List cores) { + public static void optimize(List indexList) { try { MCRSolrOptimizeIndexHandler indexHandler = new MCRSolrOptimizeIndexHandler(); - indexHandler.setDestinationCores(cores); + indexHandler.setDestinationIndex(indexList); indexHandler.setCommitWithin(BATCH_AUTO_COMMIT_WITHIN_MS); submitIndexHandler(indexHandler); } catch (Exception ex) { @@ -576,10 +576,10 @@ public static void optimize(List cores) { * Synchronizes the solr server with the database. As a result the solr server contains the same documents as the * database. All solr zombie documents will be removed, and all not indexed mycore objects will be indexed. */ - public static void synchronizeMetadataIndex(List client) throws IOException, SolrServerException { + public static void synchronizeMetadataIndex(List indexList) throws IOException, SolrServerException { Collection objectTypes = MCRXMLMetadataManager.obtainInstance().getObjectTypes(); for (String objectType : objectTypes) { - synchronizeMetadataIndex(client, objectType); + synchronizeMetadataIndex(indexList, objectType); } } @@ -588,20 +588,21 @@ public static void synchronizeMetadataIndex(List client) throws IOE * the same documents as the store. All solr zombie documents will be removed, and all not indexed mycore objects * will be indexed. */ - public static void synchronizeMetadataIndex(List cores, String objectType) + public static void synchronizeMetadataIndex(List indexList, String objectType) throws IOException, SolrServerException { - synchronizeMetadataIndex(cores, objectType, - () -> MCRXMLMetadataManager.obtainInstance().listIDsOfType(objectType), "objectType:" + objectType); + synchronizeMetadataIndex(indexList, objectType, + () -> MCRXMLMetadataManager.obtainInstance().listIDsOfType(objectType), + "objectType:" + objectType); } - public static void synchronizeMetadataIndexForObjectBase(List cores, String objectBase) + public static void synchronizeMetadataIndexForObjectBase(List indexList, String objectBase) throws IOException, SolrServerException { final String solrQuery = "objectType:" + objectBase.split("_")[1] + " _root_:" + objectBase + "_*"; - synchronizeMetadataIndex(cores, objectBase, () -> MCRXMLMetadataManager.obtainInstance() + synchronizeMetadataIndex(indexList, objectBase, () -> MCRXMLMetadataManager.obtainInstance() .listIDsForBase(objectBase), solrQuery); } - private static void synchronizeMetadataIndex(List cores, String synchBase, + private static void synchronizeMetadataIndex(List cores, String synchBase, Supplier> localIDListSupplier, String query) throws SolrServerException, IOException { LOGGER.info("synchronize {}", synchBase); @@ -611,7 +612,7 @@ private static void synchronizeMetadataIndex(List cores, String syn LOGGER.info("there are {} mycore objects", storeList::size); // get ids from solr LOGGER.info("fetching solr..."); - for (MCRSolrCore core : cores) { + for (MCRSolrIndex core : cores) { List solrList = MCRSolrSearchUtils.listIDs(core.getClient(), query); LOGGER.info("there are {} solr objects", solrList::size); // documents to remove diff --git a/mycore-solr/src/main/java/org/mycore/solr/index/file/MCRSolrFileIndexBaseAccumulator.java b/mycore-solr/src/main/java/org/mycore/solr/index/file/MCRSolrFileIndexBaseAccumulator.java index 36e7dbd080..d1a3df9386 100644 --- a/mycore-solr/src/main/java/org/mycore/solr/index/file/MCRSolrFileIndexBaseAccumulator.java +++ b/mycore-solr/src/main/java/org/mycore/solr/index/file/MCRSolrFileIndexBaseAccumulator.java @@ -95,7 +95,7 @@ public void accumulate(SolrInputDocument doc, Path input, BasicFileAttributes at doc.setField("stream_size", attr.size()); doc.setField("stream_name", absolutePath); doc.setField("stream_source_info", input.toString()); - doc.setField("stream_content_type", MCRContentTypes.probeContentType(input)); + doc.setField("IN", MCRContentTypes.probeContentType(input)); doc.setField("extension", Files.getFileExtension(input.getFileName().toString())); MCRISO8601Date iDate = new MCRISO8601Date(); iDate.setDate(new Date(attr.lastModifiedTime().toMillis())); diff --git a/mycore-solr/src/main/java/org/mycore/solr/index/file/tika/MCRSimpleTikaMapper.java b/mycore-solr/src/main/java/org/mycore/solr/index/file/tika/MCRSimpleTikaMapper.java index 954a5cabe2..751cf28d71 100644 --- a/mycore-solr/src/main/java/org/mycore/solr/index/file/tika/MCRSimpleTikaMapper.java +++ b/mycore-solr/src/main/java/org/mycore/solr/index/file/tika/MCRSimpleTikaMapper.java @@ -34,7 +34,7 @@ *

* If the property "StripNamespace" is set to true, the namespace of the key is removed before mapping. * If the property "MultiValueField" is set to true, the values are added as multi value fields, otherwise the values - * will be concatenated to a single string with newlines. + * will be concatenated to a single stringc with newlines. * * @author Sebastian Hofmann */ diff --git a/mycore-solr/src/main/java/org/mycore/solr/index/handlers/MCRSolrAbstractIndexHandler.java b/mycore-solr/src/main/java/org/mycore/solr/index/handlers/MCRSolrAbstractIndexHandler.java index 7de1a8143b..f6b6680a7f 100644 --- a/mycore-solr/src/main/java/org/mycore/solr/index/handlers/MCRSolrAbstractIndexHandler.java +++ b/mycore-solr/src/main/java/org/mycore/solr/index/handlers/MCRSolrAbstractIndexHandler.java @@ -27,9 +27,9 @@ import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.request.UpdateRequest; import org.mycore.common.config.MCRConfiguration2; -import org.mycore.solr.MCRSolrCore; -import org.mycore.solr.MCRSolrCoreManager; -import org.mycore.solr.MCRSolrCoreType; +import org.mycore.solr.MCRIndexType; +import org.mycore.solr.MCRSolrIndex; +import org.mycore.solr.MCRSolrIndexManager; import org.mycore.solr.auth.MCRSolrAuthenticationLevel; import org.mycore.solr.auth.MCRSolrAuthenticationManager; import org.mycore.solr.index.MCRSolrIndexHandler; @@ -38,11 +38,11 @@ public abstract class MCRSolrAbstractIndexHandler implements MCRSolrIndexHandler protected final MCRSolrAuthenticationManager solrAuthenticationFactory; - private List destinationCores; + private List destinationIndexList; protected int commitWithin; - private MCRSolrCoreType coreType; + private MCRIndexType indexType; public MCRSolrAbstractIndexHandler() { this.commitWithin = MCRConfiguration2.getInt("MCR.Solr.commitWithIn").orElseThrow(); @@ -71,16 +71,16 @@ public int getCommitWithin() { return commitWithin; } - public List getDestinationCores() { - if (destinationCores != null) { - return destinationCores; + public List getDestinationIndex() { + if (destinationIndexList != null) { + return destinationIndexList; } else { - return MCRSolrCoreManager.getCoresForType(this.coreType); + return MCRSolrIndexManager.obtainInstance().getIndexWithType(this.indexType); } } - public void setDestinationCores(List destinationCores) { - this.destinationCores = destinationCores; + public void setDestinationIndex(List destinationIndexList) { + this.destinationIndexList = destinationIndexList; } /** @@ -88,7 +88,7 @@ public void setDestinationCores(List destinationCores) { * @return set of solr clients */ protected List getClients() { - return getDestinationCores().stream().map(MCRSolrCore::getClient).collect(Collectors.toList()); + return getDestinationIndex().stream().map(MCRSolrIndex::getClient).collect(Collectors.toList()); } @Override @@ -108,11 +108,11 @@ public MCRSolrAuthenticationManager getSolrAuthenticationFactory() { } @Override - public void setCoreType(MCRSolrCoreType coreType) { - this.coreType = coreType; + public void setIndexType(MCRIndexType indexType) { + this.indexType = indexType; } - public MCRSolrCoreType getCoreType() { - return coreType; + public MCRIndexType getIndexType() { + return indexType; } } diff --git a/mycore-solr/src/main/java/org/mycore/solr/index/handlers/MCRSolrIndexHandlerFactory.java b/mycore-solr/src/main/java/org/mycore/solr/index/handlers/MCRSolrIndexHandlerFactory.java index e44d3e9ef4..98148c6a06 100644 --- a/mycore-solr/src/main/java/org/mycore/solr/index/handlers/MCRSolrIndexHandlerFactory.java +++ b/mycore-solr/src/main/java/org/mycore/solr/index/handlers/MCRSolrIndexHandlerFactory.java @@ -32,7 +32,7 @@ import org.mycore.datamodel.common.MCRXMLMetadataManager; import org.mycore.datamodel.metadata.MCRBase; import org.mycore.datamodel.metadata.MCRObjectID; -import org.mycore.solr.MCRSolrCoreType; +import org.mycore.solr.MCRIndexType; import org.mycore.solr.index.MCRSolrIndexHandler; import org.mycore.solr.index.file.MCRSolrPathDocumentFactory; import org.mycore.solr.index.handlers.document.MCRSolrInputDocumentHandler; @@ -106,8 +106,8 @@ public MCRSolrIndexHandler getIndexHandler(Path file, BasicFileAttributes attrs, } else { indexHandler = new MCRSolrInputDocumentHandler( () -> MCRSolrPathDocumentFactory.obtainInstance().getDocument(file, attrs), file.toString(), - MCRSolrCoreType.MAIN); - indexHandler.setCoreType(MCRSolrCoreType.MAIN); + MCRIndexType.MAIN); + indexHandler.setIndexType(MCRIndexType.MAIN); } long end = System.currentTimeMillis(); indexHandler.getStatistic().addTime(end - start); diff --git a/mycore-solr/src/main/java/org/mycore/solr/index/handlers/MCRSolrLazyInputDocumentHandlerFactory.java b/mycore-solr/src/main/java/org/mycore/solr/index/handlers/MCRSolrLazyInputDocumentHandlerFactory.java index a0ed690d4b..6a3b5ca70e 100644 --- a/mycore-solr/src/main/java/org/mycore/solr/index/handlers/MCRSolrLazyInputDocumentHandlerFactory.java +++ b/mycore-solr/src/main/java/org/mycore/solr/index/handlers/MCRSolrLazyInputDocumentHandlerFactory.java @@ -23,7 +23,7 @@ import org.mycore.common.content.MCRContent; import org.mycore.datamodel.metadata.MCRObjectID; -import org.mycore.solr.MCRSolrCoreType; +import org.mycore.solr.MCRIndexType; import org.mycore.solr.index.MCRSolrIndexHandler; import org.mycore.solr.index.handlers.content.MCRSolrMCRContentIndexHandler; import org.mycore.solr.index.handlers.content.MCRSolrMCRContentMapIndexHandler; @@ -36,14 +36,14 @@ public class MCRSolrLazyInputDocumentHandlerFactory extends MCRSolrIndexHandlerF @Override public MCRSolrIndexHandler getIndexHandler(MCRContent content, MCRObjectID id) { - return new MCRSolrMCRContentIndexHandler(id, content, MCRSolrCoreType.MAIN); + return new MCRSolrMCRContentIndexHandler(id, content, MCRIndexType.MAIN); } @Override public MCRSolrIndexHandler getIndexHandler(Map contentMap) { //contentMap is reused in different threads Map copyMap = new HashMap<>(contentMap); - return new MCRSolrMCRContentMapIndexHandler(copyMap, MCRSolrCoreType.MAIN); + return new MCRSolrMCRContentMapIndexHandler(copyMap, MCRIndexType.MAIN); } } diff --git a/mycore-solr/src/main/java/org/mycore/solr/index/handlers/content/MCRSolrMCRContentIndexHandler.java b/mycore-solr/src/main/java/org/mycore/solr/index/handlers/content/MCRSolrMCRContentIndexHandler.java index 6247609760..3355c4512b 100644 --- a/mycore-solr/src/main/java/org/mycore/solr/index/handlers/content/MCRSolrMCRContentIndexHandler.java +++ b/mycore-solr/src/main/java/org/mycore/solr/index/handlers/content/MCRSolrMCRContentIndexHandler.java @@ -25,7 +25,7 @@ import org.apache.solr.common.SolrInputDocument; import org.mycore.common.content.MCRContent; import org.mycore.datamodel.metadata.MCRObjectID; -import org.mycore.solr.MCRSolrCoreType; +import org.mycore.solr.MCRIndexType; import org.mycore.solr.index.MCRSolrIndexHandler; import org.mycore.solr.index.document.MCRSolrInputDocumentFactory; import org.mycore.solr.index.handlers.MCRSolrAbstractIndexHandler; @@ -46,10 +46,10 @@ public class MCRSolrMCRContentIndexHandler extends MCRSolrAbstractIndexHandler { private SolrInputDocument document; - public MCRSolrMCRContentIndexHandler(MCRObjectID id, MCRContent content, MCRSolrCoreType type) { + public MCRSolrMCRContentIndexHandler(MCRObjectID id, MCRContent content, MCRIndexType type) { this.id = id; this.content = content; - this.setCoreType(type); + this.setIndexType(type); } @Override @@ -72,7 +72,7 @@ public void index() throws IOException { @Override public List getSubHandlers() { MCRSolrIndexHandler mcrSolrIndexHandler = new MCRSolrInputDocumentHandler(() -> document, id.toString(), - this.getCoreType()); + this.getIndexType()); mcrSolrIndexHandler.setCommitWithin(getCommitWithin()); return Collections.singletonList(mcrSolrIndexHandler); } diff --git a/mycore-solr/src/main/java/org/mycore/solr/index/handlers/content/MCRSolrMCRContentMapIndexHandler.java b/mycore-solr/src/main/java/org/mycore/solr/index/handlers/content/MCRSolrMCRContentMapIndexHandler.java index 100587625f..a73b13ca77 100644 --- a/mycore-solr/src/main/java/org/mycore/solr/index/handlers/content/MCRSolrMCRContentMapIndexHandler.java +++ b/mycore-solr/src/main/java/org/mycore/solr/index/handlers/content/MCRSolrMCRContentMapIndexHandler.java @@ -34,8 +34,8 @@ import org.apache.solr.common.SolrInputDocument; import org.mycore.common.content.MCRContent; import org.mycore.datamodel.metadata.MCRObjectID; -import org.mycore.solr.MCRSolrCore; -import org.mycore.solr.MCRSolrCoreType; +import org.mycore.solr.MCRIndexType; +import org.mycore.solr.MCRSolrIndex; import org.mycore.solr.auth.MCRSolrAuthenticationLevel; import org.mycore.solr.index.MCRSolrIndexHandler; import org.mycore.solr.index.document.MCRSolrInputDocumentFactory; @@ -57,11 +57,11 @@ public class MCRSolrMCRContentMapIndexHandler extends MCRSolrAbstractIndexHandle private Map contentMap; - public MCRSolrMCRContentMapIndexHandler(Map contentMap, MCRSolrCoreType type) { + public MCRSolrMCRContentMapIndexHandler(Map contentMap, MCRIndexType type) { super(); this.contentMap = contentMap; this.subhandlers = new ArrayList<>(contentMap.size()); - this.setCoreType(type); + this.setIndexType(type); } @Override @@ -104,25 +104,25 @@ public void index() { } req.add(docs); - for (MCRSolrCore destinationCore : getDestinationCores()) { + for (MCRSolrIndex destinationIndex : getDestinationIndex()) { try { - UpdateResponse updateResponse = req.process(destinationCore.getClient()); + UpdateResponse updateResponse = req.process(destinationIndex.getClient()); if (updateResponse != null && updateResponse.getStatus() != 0) { LOGGER.error("Error while indexing document collection. Split and retry: {}", updateResponse::getResponse); - splitup(List.of(destinationCore)); + splitup(List.of(destinationIndex)); } else { LOGGER.info("Sending {} documents was successful in {} ms.", () -> totalCount, updateResponse::getElapsedTime); } } catch (SolrServerException | IOException e) { LOGGER.warn("Error while indexing document collection. Split and retry.", e); - splitup(List.of(destinationCore)); + splitup(List.of(destinationIndex)); return; } } } catch (SAXException | IOException e) { - splitup(getDestinationCores()); + splitup(getDestinationIndex()); } finally { contentMap.clear(); } @@ -132,19 +132,19 @@ private void makeConcurrent(Iterator documents) { while (documents.hasNext()) { SolrInputDocument nextDocument = documents.next(); MCRSolrInputDocumentHandler subhandler = new MCRSolrInputDocumentHandler(() -> nextDocument, - nextDocument.get("id").toString(), getCoreType()); + nextDocument.get("id").toString(), getIndexType()); subhandler.setCommitWithin(getCommitWithin()); subhandlers.add(subhandler); } contentMap.clear(); } - private void splitup(List cores) { + private void splitup(List indexList) { for (Map.Entry entry : contentMap.entrySet()) { MCRSolrMCRContentIndexHandler subHandler = new MCRSolrMCRContentIndexHandler(entry.getKey(), - entry.getValue(), getCoreType()); + entry.getValue(), getIndexType()); subHandler.setCommitWithin(getCommitWithin()); - subHandler.setDestinationCores(cores); + subHandler.setDestinationIndex(indexList); subhandlers.add(subHandler); } } diff --git a/mycore-solr/src/main/java/org/mycore/solr/index/handlers/document/MCRSolrInputDocumentHandler.java b/mycore-solr/src/main/java/org/mycore/solr/index/handlers/document/MCRSolrInputDocumentHandler.java index 8b083b9441..b2e4da87d0 100644 --- a/mycore-solr/src/main/java/org/mycore/solr/index/handlers/document/MCRSolrInputDocumentHandler.java +++ b/mycore-solr/src/main/java/org/mycore/solr/index/handlers/document/MCRSolrInputDocumentHandler.java @@ -30,7 +30,7 @@ import org.apache.solr.client.solrj.request.UpdateRequest; import org.apache.solr.common.SolrInputDocument; import org.mycore.solr.MCRSolrConstants; -import org.mycore.solr.MCRSolrCoreType; +import org.mycore.solr.MCRIndexType; import org.mycore.solr.MCRSolrUtils; import org.mycore.solr.index.MCRSolrIndexHandler; import org.mycore.solr.index.MCRSolrIndexer; @@ -71,11 +71,11 @@ public class MCRSolrInputDocumentHandler extends MCRSolrAbstractIndexHandler { * @param coreType The type of Solr core this document should be sent to (e.g., main). */ public MCRSolrInputDocumentHandler(Supplier documentSupplier, String id, - MCRSolrCoreType coreType) { + MCRIndexType coreType) { this.id = id; this.documentSupplier = documentSupplier; this.subHandlers = new ArrayList<>(); - this.setCoreType(coreType); + this.setIndexType(coreType); } /** diff --git a/mycore-solr/src/main/java/org/mycore/solr/index/handlers/document/MCRSolrInputDocumentsHandler.java b/mycore-solr/src/main/java/org/mycore/solr/index/handlers/document/MCRSolrInputDocumentsHandler.java index d6e7a12dec..64cc366c0a 100644 --- a/mycore-solr/src/main/java/org/mycore/solr/index/handlers/document/MCRSolrInputDocumentsHandler.java +++ b/mycore-solr/src/main/java/org/mycore/solr/index/handlers/document/MCRSolrInputDocumentsHandler.java @@ -30,7 +30,7 @@ import org.apache.solr.client.solrj.response.UpdateResponse; import org.apache.solr.common.SolrInputDocument; import org.mycore.solr.MCRSolrConstants; -import org.mycore.solr.MCRSolrCoreType; +import org.mycore.solr.MCRIndexType; import org.mycore.solr.index.MCRSolrIndexHandler; import org.mycore.solr.index.handlers.MCRSolrAbstractIndexHandler; import org.mycore.solr.index.statistic.MCRSolrIndexStatistic; @@ -47,9 +47,9 @@ public class MCRSolrInputDocumentsHandler extends MCRSolrAbstractIndexHandler { private static final Logger LOGGER = LogManager.getLogger(); - public MCRSolrInputDocumentsHandler(Collection documents, MCRSolrCoreType coreType) { + public MCRSolrInputDocumentsHandler(Collection documents, MCRIndexType coreType) { this.documents = documents; - setCoreType(coreType); + setIndexType(coreType); } /* (non-Javadoc) @@ -106,7 +106,7 @@ private void splitDocuments() { subHandlerList = new ArrayList<>(documents.size()); for (SolrInputDocument document : documents) { MCRSolrInputDocumentHandler subHandler = new MCRSolrInputDocumentHandler(() -> document, - String.valueOf(document.getFieldValue("id")), getCoreType()); + String.valueOf(document.getFieldValue("id")), getIndexType()); subHandler.setCommitWithin(getCommitWithin()); this.subHandlerList.add(subHandler); } diff --git a/mycore-solr/src/main/java/org/mycore/solr/index/handlers/stream/MCRSolrFileIndexHandler.java b/mycore-solr/src/main/java/org/mycore/solr/index/handlers/stream/MCRSolrFileIndexHandler.java index d3c696eb5b..26772a4ba0 100644 --- a/mycore-solr/src/main/java/org/mycore/solr/index/handlers/stream/MCRSolrFileIndexHandler.java +++ b/mycore-solr/src/main/java/org/mycore/solr/index/handlers/stream/MCRSolrFileIndexHandler.java @@ -36,7 +36,7 @@ import org.apache.solr.common.SolrInputField; import org.apache.solr.common.params.ModifiableSolrParams; import org.mycore.common.MCRUtils; -import org.mycore.solr.MCRSolrCoreType; +import org.mycore.solr.MCRIndexType; import org.mycore.solr.auth.MCRSolrAuthenticationLevel; import org.mycore.solr.index.cs.MCRSolrPathContentStream; import org.mycore.solr.index.file.MCRSolrPathDocumentFactory; @@ -54,7 +54,7 @@ public class MCRSolrFileIndexHandler extends MCRSolrAbstractStreamIndexHandler { public MCRSolrFileIndexHandler(Path file, BasicFileAttributes attrs) { this.file = file; this.attrs = attrs; - this.setCoreType(MCRSolrCoreType.MAIN); + this.setIndexType(MCRIndexType.MAIN); } @Override diff --git a/mycore-solr/src/main/java/org/mycore/solr/index/handlers/stream/MCRSolrFilesIndexHandler.java b/mycore-solr/src/main/java/org/mycore/solr/index/handlers/stream/MCRSolrFilesIndexHandler.java index 3b831239e7..0dec990d3c 100644 --- a/mycore-solr/src/main/java/org/mycore/solr/index/handlers/stream/MCRSolrFilesIndexHandler.java +++ b/mycore-solr/src/main/java/org/mycore/solr/index/handlers/stream/MCRSolrFilesIndexHandler.java @@ -33,7 +33,7 @@ import org.mycore.datamodel.metadata.MCRMetadataManager; import org.mycore.datamodel.metadata.MCRObjectID; import org.mycore.datamodel.niofs.MCRPath; -import org.mycore.solr.MCRSolrCoreType; +import org.mycore.solr.MCRIndexType; import org.mycore.solr.index.MCRSolrIndexHandler; import org.mycore.solr.index.handlers.MCRSolrAbstractIndexHandler; import org.mycore.solr.index.handlers.MCRSolrIndexHandlerFactory; @@ -63,7 +63,7 @@ public MCRSolrFilesIndexHandler(String mcrID) { super(); this.mcrID = mcrID; this.subHandlerList = new ArrayList<>(); - setCoreType(MCRSolrCoreType.MAIN); + setIndexType(MCRIndexType.MAIN); } @Override diff --git a/mycore-solr/src/main/java/org/mycore/solr/proxy/MCRSolrProxyServlet.java b/mycore-solr/src/main/java/org/mycore/solr/proxy/MCRSolrProxyServlet.java index 190bc6bb40..a2e6774afb 100644 --- a/mycore-solr/src/main/java/org/mycore/solr/proxy/MCRSolrProxyServlet.java +++ b/mycore-solr/src/main/java/org/mycore/solr/proxy/MCRSolrProxyServlet.java @@ -28,10 +28,7 @@ import java.io.OutputStream; import java.io.PrintWriter; import java.io.Serial; -import java.net.URI; import java.net.http.HttpClient; -import java.net.http.HttpRequest; -import java.net.http.HttpResponse; import java.text.MessageFormat; import java.util.Collections; import java.util.HashMap; @@ -42,7 +39,6 @@ import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.TreeMap; import java.util.stream.Collectors; import javax.xml.transform.TransformerException; @@ -50,6 +46,9 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.solr.client.solrj.request.SolrQuery; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.response.InputStreamResponseParser; +import org.apache.solr.client.solrj.request.QueryRequest; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.params.MultiMapSolrParams; import org.apache.solr.common.util.NamedList; @@ -64,8 +63,8 @@ import org.mycore.frontend.servlets.MCRServletJob; import org.mycore.services.http.MCRHttpUtils; import org.mycore.solr.MCRSolrConstants; -import org.mycore.solr.MCRSolrCoreManager; -import org.mycore.solr.MCRSolrUtils; +import org.mycore.solr.MCRSolrIndex; +import org.mycore.solr.MCRSolrIndexManager; import org.mycore.solr.auth.MCRSolrAuthenticationLevel; import org.mycore.solr.auth.MCRSolrAuthenticationManager; import org.xml.sax.SAXException; @@ -90,7 +89,7 @@ public class MCRSolrProxyServlet extends MCRServlet { @Serial private static final long serialVersionUID = 1L; - + private static final Logger LOGGER = LogManager.getLogger(); /** @@ -112,14 +111,6 @@ public class MCRSolrProxyServlet extends MCRServlet { public static final MCRSolrAuthenticationManager SOLR_AUTHENTICATION_MANAGER = MCRSolrAuthenticationManager.obtainInstance(); - private static final Map NEW_HTTP_RESPONSE_HEADER; - - static { - Map newRespHeader = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); - newRespHeader.putAll(MCRConfiguration2.getSubPropertiesMap(SOLR_CONFIG_PREFIX + "HTTPResponseHeader.")); - NEW_HTTP_RESPONSE_HEADER = Collections.unmodifiableMap(newRespHeader); - } - private HttpClient httpClient; private Set queryHandlerWhitelist; @@ -230,55 +221,52 @@ private void handleQuery(String queryHandlerPath, HttpServletRequest request, Ht throws IOException, TransformerException, SAXException { ModifiableSolrParams solrParameter = getSolrQueryParameter(request); filterParams(solrParameter); - HttpRequest.Builder solrHttpMethod = getSolrHttpMethod(queryHandlerPath, solrParameter, - Optional.ofNullable(request.getParameter(QUERY_CORE_PARAMETER)).orElse(MCRSolrConstants.MAIN_CORE_TYPE)); - SOLR_AUTHENTICATION_MANAGER.applyAuthentication(solrHttpMethod, MCRSolrAuthenticationLevel.SEARCH); + QueryRequest queryRequest = new QueryRequest(solrParameter); + queryRequest.setPath(queryHandlerPath); + SOLR_AUTHENTICATION_MANAGER.applyAuthentication(queryRequest, MCRSolrAuthenticationLevel.SEARCH); + + String core = Optional.ofNullable(request.getParameter(QUERY_CORE_PARAMETER)) + .orElse(MCRSolrConstants.MAIN_CORE_TYPE); + + Optional optionalIndex = MCRSolrIndexManager.obtainInstance().getIndex(core); + if (optionalIndex.isEmpty()) { + resp.sendError(HttpServletResponse.SC_BAD_REQUEST, "No such core: " + core); + return; + } + MCRSolrIndex solrIndex = optionalIndex.get(); + String writerType = getWriterType(request); + queryRequest.setResponseParser(new InputStreamResponseParser(writerType)); - HttpRequest solrRequest; try { - solrRequest = solrHttpMethod.build(); - LOGGER.info("Sending Request: {}", solrRequest::uri); - HttpResponse response = - httpClient.send(solrRequest, HttpResponse.BodyHandlers.ofInputStream()); - int statusCode = response.statusCode(); - - // set status code - resp.setStatus(statusCode); - - boolean isXML = response.headers().firstValue("Content-Type").orElse("").contains("/xml"); - boolean justCopyInput = !isXML; - - // set all headers - MCRHttpUtils.filterHopByHop(response.headers()).map().forEach((headerName, headerValues) -> { - LOGGER.debug("SOLR response header: {} - {}", headerName, headerValues); - if (NEW_HTTP_RESPONSE_HEADER.containsKey(headerName)) { - String headerValue = NEW_HTTP_RESPONSE_HEADER.get(headerName); - if (headerValue != null && !headerValue.isEmpty()) { - resp.setHeader(headerName, headerValue); - } - } else { - headerValues.forEach(headerValue -> resp.setHeader(headerName, headerValue)); - } - }); - - try (InputStream solrResponseStream = response.body()) { - if (justCopyInput) { - // copy solr response to servlet outputstream - OutputStream servletOutput = resp.getOutputStream(); - solrResponseStream.transferTo(servletOutput); - } else { - MCRStreamContent solrResponse = new MCRStreamContent(solrResponseStream, - solrRequest.uri().toString(), - "response"); - MCRLayoutService.obtainInstance().doLayout(request, resp, solrResponse); - } + NamedList solrResponse = solrIndex.getClient().request(queryRequest); + InputStream is = (InputStream) solrResponse.get("stream"); + + if(solrResponse.get("responseStatus") != null) { + Integer responseStatus = (Integer) solrResponse.get("responseStatus"); + resp.setStatus(responseStatus); } - } catch (InterruptedException ex) { - throw new IOException(ex); + + if (!writerType.equals("xml")) { + // copy solr response to servlet outputstream + OutputStream servletOutput = resp.getOutputStream(); + is.transferTo(servletOutput); + } else { + MCRStreamContent solrResponseContent = new MCRStreamContent(is, + request.getRequestURI(), + "response"); + MCRLayoutService.obtainInstance().doLayout(request, resp, solrResponseContent); + } + } catch (SolrServerException e) { + throw new IOException("Error while processing query request", e); } } + private String getWriterType(HttpServletRequest request) { + return Optional.ofNullable(request.getParameter("wt")) + .orElse("xml"); + } + private void filterParams(ModifiableSolrParams solrParameter) { MCRConfiguration2.getString("MCR.Solr.Disallowed.Facets") .ifPresent(disallowedFacets -> MCRConfiguration2.splitValue(disallowedFacets) @@ -303,23 +291,6 @@ private void updateQueryHandlerMap() { this.queryHandlerWhitelist = new HashSet<>(whitelistPropertyList); } - /** - * Gets a HttpGet to make a request to the Solr-Server. - * - * @param queryHandlerPath - * The query handler path - * @param params - * Parameters to use with the Request - * @return a method to make the request - */ - private static HttpRequest.Builder getSolrHttpMethod(String queryHandlerPath, ModifiableSolrParams params, - String type) { - String serverURL = MCRSolrCoreManager.get(type).get().getV1CoreURL(); - - return MCRSolrUtils.getRequestBuilder() - .uri(URI.create(serverURL + queryHandlerPath + params.toQueryString())); - } - private static ModifiableSolrParams getSolrQueryParameter(HttpServletRequest request) { SolrQuery query = (SolrQuery) request.getAttribute(QUERY_KEY); if (query != null) { diff --git a/mycore-solr/src/main/java/org/mycore/solr/schema/MCRSolrConfigReloader.java b/mycore-solr/src/main/java/org/mycore/solr/schema/MCRSolrConfigReloader.java index ae79dd0666..9793798e71 100644 --- a/mycore-solr/src/main/java/org/mycore/solr/schema/MCRSolrConfigReloader.java +++ b/mycore-solr/src/main/java/org/mycore/solr/schema/MCRSolrConfigReloader.java @@ -21,11 +21,8 @@ import static java.util.Map.entry; import java.io.IOException; -import java.io.UnsupportedEncodingException; -import java.net.URI; -import java.net.http.HttpClient; -import java.net.http.HttpRequest; -import java.net.http.HttpResponse; +import java.io.InputStream; +import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Locale; @@ -38,14 +35,17 @@ import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.request.GenericSolrRequest; import org.mycore.common.config.MCRConfiguration2; import org.mycore.common.config.MCRConfigurationInputStream; -import org.mycore.services.http.MCRHttpUtils; -import org.mycore.solr.MCRSolrCore; -import org.mycore.solr.MCRSolrCoreManager; +import org.mycore.solr.MCRSolrIndex; +import org.mycore.solr.MCRSolrIndexManager; import org.mycore.solr.MCRSolrUtils; import org.mycore.solr.auth.MCRSolrAuthenticationLevel; import org.mycore.solr.auth.MCRSolrAuthenticationManager; +import org.mycore.solr.search.MCRSolrSearchUtils; import com.google.gson.Gson; import com.google.gson.JsonArray; @@ -97,10 +97,12 @@ public class MCRSolrConfigReloader { */ public static void reset(String configType, String coreID) { LOGGER.info(() -> "Resetting config definitions for core " + coreID + " using configuration " + configType); - String coreURL = MCRSolrCoreManager.get(coreID) - .map(MCRSolrCore::getV1CoreURL) + + MCRSolrIndex index = MCRSolrIndexManager.obtainInstance() + .getIndex(coreID) .orElseThrow(() -> MCRSolrUtils.getCoreConfigMissingException(coreID)); - JsonObject currentSolrConfig = retrieveCurrentSolrConfigOverlay(coreURL); + + JsonObject currentSolrConfig = retrieveCurrentSolrConfigOverlay(index); JsonObject configPart = currentSolrConfig.getAsJsonObject("overlay"); for (String observedType : getObserverConfigTypes()) { @@ -117,11 +119,7 @@ public static void reset(String configType, String coreID) { final JsonObject deleteCommand = new JsonObject(); deleteCommand.addProperty(deleteSectionCommand, configuredComponent.getKey()); LOGGER.debug(deleteCommand); - try { - executeSolrCommand(coreURL, deleteCommand); - } catch (IOException e) { - LOGGER.error(() -> "Exception while executing '" + deleteCommand + "'.", e); - } + executeSolrCommand(index, deleteCommand); } } } @@ -136,10 +134,10 @@ public static void reset(String configType, String coreID) { public static void processConfigFiles(String configType, String coreID) { LOGGER.info(() -> "Load config definitions for core " + coreID + " using configuration " + configType); try { - String coreURL = MCRSolrCoreManager.get(coreID) - .orElseThrow(() -> MCRSolrUtils.getCoreConfigMissingException(coreID)).getV1CoreURL(); + MCRSolrIndex index = MCRSolrIndexManager.obtainInstance().requireIndex(coreID); + List observedTypes = getObserverConfigTypes(); - JsonObject currentSolrConfig = retrieveCurrentSolrConfig(coreURL); + JsonObject currentSolrConfig = retrieveCurrentSolrConfig(index); Map configFileContents = MCRConfigurationInputStream.getConfigFileContents( "solr/" + configType + "/" + SOLR_CONFIG_UPDATE_FILE_NAME); @@ -154,7 +152,7 @@ public static void processConfigFiles(String configType, String coreID) { for (JsonElement command : json.getAsJsonArray()) { LOGGER.debug(command); - processConfigCommand(coreURL, command, currentSolrConfig, observedTypes); + processConfigCommand(index, command, currentSolrConfig, observedTypes); } } } catch (IOException e) { @@ -179,33 +177,29 @@ private static List getObserverConfigTypes() { * @param coreURL - the URL of the core * @param command - the command in JSON syntax */ - private static void processConfigCommand(String coreURL, JsonElement command, JsonObject currentSolrConfig, + private static void processConfigCommand(MCRSolrIndex coreURL, JsonElement command, JsonObject currentSolrConfig, List observedTypes) { if (command.isJsonObject()) { - try { - //get first and only? property of the command object - final JsonObject commandJsonObject = command.getAsJsonObject(); - Entry commandObject = commandJsonObject.entrySet().iterator().next(); - final String configCommand = commandObject.getKey(); - final String configType = StringUtils.substringAfter(configCommand, "add-"); - - if (isKnownSolrConfigCommmand(configCommand)) { - - if (observedTypes.contains(configType) && configCommand.startsWith("add-") && - commandObject.getValue() instanceof JsonObject) { - final JsonElement configCommandName = commandObject.getValue().getAsJsonObject().get("name"); - if (isConfigTypeAlreadyAdded(configType, configCommandName, currentSolrConfig)) { - LOGGER.info(() -> "Current configuration has already an " + configCommand - + " with name " + configCommandName.getAsString() - + ". Rewrite config command as update-" + configType); - commandJsonObject.add("update-" + configType, commandJsonObject.get(configCommand)); - commandJsonObject.remove(configCommand); - } + //get first and only? property of the command object + final JsonObject commandJsonObject = command.getAsJsonObject(); + Entry commandObject = commandJsonObject.entrySet().iterator().next(); + final String configCommand = commandObject.getKey(); + final String configType = StringUtils.substringAfter(configCommand, "add-"); + + if (isKnownSolrConfigCommmand(configCommand)) { + + if (observedTypes.contains(configType) && configCommand.startsWith("add-") && + commandObject.getValue() instanceof JsonObject) { + final JsonElement configCommandName = commandObject.getValue().getAsJsonObject().get("name"); + if (isConfigTypeAlreadyAdded(configType, configCommandName, currentSolrConfig)) { + LOGGER.info(() -> "Current configuration has already an " + configCommand + + " with name " + configCommandName.getAsString() + + ". Rewrite config command as update-" + configType); + commandJsonObject.add("update-" + configType, commandJsonObject.get(configCommand)); + commandJsonObject.remove(configCommand); } - executeSolrCommand(coreURL, commandJsonObject); } - } catch (IOException e) { - LOGGER.error(e); + executeSolrCommand(coreURL, commandJsonObject); } } @@ -213,32 +207,26 @@ private static void processConfigCommand(String coreURL, JsonElement command, Js /** * Sends a command to SOLR server - * @param coreURL to which the command will be send + * @param index to which the command will be send * @param command the command - * @throws UnsupportedEncodingException if command encoding is not supported */ - private static void executeSolrCommand(String coreURL, JsonObject command) throws UnsupportedEncodingException { - HttpRequest.Builder requestBuilder = MCRSolrUtils.getRequestBuilder() - .uri(URI.create(coreURL + "/config")) - .header("Content-Type", "application/json") - .POST(HttpRequest.BodyPublishers.ofString(command.toString())); + private static void executeSolrCommand(MCRSolrIndex index, JsonObject command) { + GenericSolrRequest request = new GenericSolrRequest(CollectionAdminRequest.METHOD.POST, + "/config"); + request.setRequiresCollection(true); + request.withContent(command.toString().getBytes(StandardCharsets.UTF_8), "application/json"); - MCRSolrAuthenticationManager.obtainInstance().applyAuthentication(requestBuilder, + MCRSolrAuthenticationManager.obtainInstance().applyAuthentication(request, MCRSolrAuthenticationLevel.ADMIN); - String commandprefix = command.keySet().stream().findFirst().orElse("unknown command"); - try (HttpClient httpClient = MCRHttpUtils.getHttpClient()) { - HttpResponse response = - httpClient.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofString()); - if (response.statusCode() == 200) { - LOGGER.debug(() -> "SOLR config " + commandprefix + " command was successful \n" + response.body()); - } else { - LOGGER - .error(() -> "SOLR config " + commandprefix + " error: " + response.statusCode() + " " - + MCRHttpUtils.getReasonPhrase(response.statusCode()) + "\n" - + response.body()); - } - - } catch (InterruptedException | IOException e) { + String commandPrefix = command.keySet().stream().findFirst().orElse("unknown command"); + + try (InputStream is = MCRSolrSearchUtils.streamRequest(index.getClient(), request, "json")) { + String response = new String(is.readAllBytes(), StandardCharsets.UTF_8); + LOGGER.debug(() -> "SOLR config " + commandPrefix + " command was successful \n" + response); + } catch (SolrServerException e) { + LOGGER + .error(() -> "SOLR config " + commandPrefix + " error: " + e.getMessage() + "\n" + command, e); + } catch (IOException e) { LOGGER.error(() -> "Could not execute the following Solr config command:\n" + command, e); } } @@ -260,42 +248,38 @@ private static boolean isConfigTypeAlreadyAdded(String configType, JsonElement n /** * retrieves the current SOLR configuration for the given core - * @param coreURL from which the current SOLR configuration will be load + * @param solrIndex from which the current SOLR configuration will be load * @return the configuration as JSON object */ - private static JsonObject retrieveCurrentSolrConfig(String coreURL) { - HttpRequest.Builder solrRequestBuilder = MCRSolrUtils.getRequestBuilder() - .uri(URI.create(coreURL + "/config")); - MCRSolrAuthenticationManager.obtainInstance().applyAuthentication(solrRequestBuilder, + private static JsonObject retrieveCurrentSolrConfig(MCRSolrIndex solrIndex) { + GenericSolrRequest request = new GenericSolrRequest(CollectionAdminRequest.METHOD.GET, + "/config"); + request.setRequiresCollection(true); + MCRSolrAuthenticationManager.obtainInstance().applyAuthentication(request, MCRSolrAuthenticationLevel.ADMIN); - return getJSON(solrRequestBuilder.build()); + return getJSON(solrIndex, request); } /** * retrieves the current SOLR configuration overlay for the given core - * @param coreURL from which the current SOLR configuration will be load + * @param solrIndex from which the current SOLR configuration will be load * @return the configuration as JSON object */ - private static JsonObject retrieveCurrentSolrConfigOverlay(String coreURL) { - HttpRequest.Builder solrRequestBuilder = MCRSolrUtils.getRequestBuilder() - .uri(URI.create(coreURL + "/config/overlay")); - MCRSolrAuthenticationManager.obtainInstance().applyAuthentication(solrRequestBuilder, + private static JsonObject retrieveCurrentSolrConfigOverlay(MCRSolrIndex solrIndex) { + GenericSolrRequest request = new GenericSolrRequest(CollectionAdminRequest.METHOD.GET, + "/config/overlay"); + request.setRequiresCollection(true); + MCRSolrAuthenticationManager.obtainInstance().applyAuthentication(request, MCRSolrAuthenticationLevel.ADMIN); - return getJSON(solrRequestBuilder.build()); + return getJSON(solrIndex, request); } - private static JsonObject getJSON(HttpRequest getConfig) { + private static JsonObject getJSON(MCRSolrIndex solrIndex, GenericSolrRequest request) { JsonObject convertedObject = null; - try (HttpClient httpClient = MCRHttpUtils.getHttpClient()) { - HttpResponse response = httpClient.send(getConfig, HttpResponse.BodyHandlers.ofString()); - if (response.statusCode() == 200) { - convertedObject = new Gson().fromJson(response.body(), JsonObject.class); - } else { - LOGGER.error(() -> "Could not retrieve current Solr configuration from solr server. Http Status: " - + response.statusCode() + " " + MCRHttpUtils.getReasonPhrase(response.statusCode())); - } - - } catch (IOException | InterruptedException e) { + try (InputStream jsonIS = MCRSolrSearchUtils.streamRequest(solrIndex.getClient(), request, + "json"); InputStreamReader reader = new InputStreamReader(jsonIS, StandardCharsets.UTF_8)) { + return new Gson().fromJson(reader, JsonObject.class); + } catch (IOException | SolrServerException e) { LOGGER.error("Could not read current Solr configuration", e); } catch (JsonSyntaxException e) { LOGGER.error("Current json configuration is not a valid json", e); diff --git a/mycore-solr/src/main/java/org/mycore/solr/schema/MCRSolrSchemaReloader.java b/mycore-solr/src/main/java/org/mycore/solr/schema/MCRSolrSchemaReloader.java index 6240c6c725..429f1dec25 100644 --- a/mycore-solr/src/main/java/org/mycore/solr/schema/MCRSolrSchemaReloader.java +++ b/mycore-solr/src/main/java/org/mycore/solr/schema/MCRSolrSchemaReloader.java @@ -20,11 +20,8 @@ import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.InputStream; import java.io.InputStreamReader; -import java.net.URI; -import java.net.http.HttpClient; -import java.net.http.HttpRequest; -import java.net.http.HttpResponse; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; @@ -36,19 +33,22 @@ import org.apache.logging.log4j.Logger; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.request.GenericSolrRequest; import org.apache.solr.client.solrj.request.schema.SchemaRequest; import org.apache.solr.client.solrj.response.schema.FieldTypeRepresentation; -import org.mycore.common.config.MCRConfigurationException; +import org.mycore.common.MCRException; import org.mycore.common.config.MCRConfigurationInputStream; -import org.mycore.services.http.MCRHttpUtils; -import org.mycore.solr.MCRSolrCore; -import org.mycore.solr.MCRSolrCoreManager; +import org.mycore.solr.MCRSolrIndex; +import org.mycore.solr.MCRSolrIndexManager; import org.mycore.solr.MCRSolrUtils; import org.mycore.solr.auth.MCRSolrAuthenticationLevel; import org.mycore.solr.auth.MCRSolrAuthenticationManager; +import org.mycore.solr.search.MCRSolrSearchUtils; import com.google.gson.JsonArray; import com.google.gson.JsonElement; +import com.google.gson.JsonObject; import com.google.gson.JsonParser; /** @@ -82,8 +82,11 @@ public class MCRSolrSchemaReloader { public static void reset(String configType, String coreID) { LOGGER.info(() -> "Resetting SOLR schema for core " + coreID + " using configuration " + configType); try { - SolrClient solrClient = MCRSolrCoreManager.get(coreID).map(MCRSolrCore::getClient) - .orElseThrow(() -> new MCRConfigurationException("The core " + coreID + " is not configured!")); + MCRSolrIndex index = MCRSolrIndexManager.obtainInstance() + .getIndex(coreID) + .orElseThrow(() -> MCRSolrUtils.getCoreConfigMissingException(coreID)); + + SolrClient solrClient = index.getClient(); deleteCopyFields(solrClient); LOGGER.debug(() -> "CopyFields cleaned for core " + coreID + " for configuration " + configType); @@ -176,51 +179,59 @@ private static void deleteCopyFields(SolrClient solrClient) * @param coreID the ID of the core, which the configuration should be applied to */ public static void processSchemaFiles(String configType, String coreID) { - MCRSolrCore solrCore = MCRSolrCoreManager.get(coreID) + MCRSolrIndex index = MCRSolrIndexManager.obtainInstance() + .getIndex(coreID) .orElseThrow(() -> MCRSolrUtils.getCoreConfigMissingException(coreID)); LOGGER.info(() -> "Load schema definitions for core " + coreID + " using configuration " + configType); - try (HttpClient httpClient = MCRHttpUtils.getHttpClient()) { - Collection schemaFileContents = MCRConfigurationInputStream.getConfigFileContents( + Collection schemaFileContents; + try { + schemaFileContents = MCRConfigurationInputStream.getConfigFileContents( "solr/" + configType + "/" + SOLR_SCHEMA_UPDATE_FILE_NAME).values(); - for (byte[] schemaFileData : schemaFileContents) { - InputStreamReader schemaReader = new InputStreamReader(new ByteArrayInputStream(schemaFileData), - StandardCharsets.UTF_8); - JsonElement json = JsonParser.parseReader(schemaReader); - if (!json.isJsonArray()) { - JsonElement e = json; - json = new JsonArray(); - json.getAsJsonArray().add(e); - } - for (JsonElement e : json.getAsJsonArray()) { - LOGGER.debug(e); - String command = e.toString(); - - HttpRequest.Builder solrRequestBuilder = MCRSolrUtils.getRequestBuilder() - .uri(URI.create(solrCore.getV1CoreURL() + "/schema")) - .header("Content-type", "application/json") - .POST(HttpRequest.BodyPublishers.ofString(command)); - SOLR_AUTHENTICATION_MANAGER.applyAuthentication(solrRequestBuilder, - MCRSolrAuthenticationLevel.ADMIN); - String commandPrefix = command.indexOf('-') != -1 ? command.substring(2, command.indexOf('-')) - : "unknown command"; - - HttpResponse response = httpClient.send(solrRequestBuilder.build(), - HttpResponse.BodyHandlers.ofString()); - - if (response.statusCode() == 200) { - LOGGER.debug("SOLR schema {} successful \n{}", () -> commandPrefix, response::body); - } else { - - LOGGER - .error("SOLR schema {} error: {} {}\n{}", () -> commandPrefix, response::statusCode, - () -> MCRHttpUtils.getReasonPhrase(response.statusCode()), response::body); - } + } catch (IOException e) { + throw new MCRException(e); + } + for (byte[] schemaFileData : schemaFileContents) { + InputStreamReader schemaReader = new InputStreamReader(new ByteArrayInputStream(schemaFileData), + StandardCharsets.UTF_8); + JsonElement json = JsonParser.parseReader(schemaReader); + if (!json.isJsonArray()) { + JsonElement e = json; + json = new JsonArray(); + json.getAsJsonArray().add(e); + } + for (JsonElement e : json.getAsJsonArray()) { + LOGGER.debug(e); + if (e.isJsonObject()) { + executeSchemaRequest(index, e.getAsJsonObject()); } } - } catch (InterruptedException | IOException e) { - LOGGER.error(e); } } + /** + * Sends a schema request to SOLR server + * @param index to which the command will be send + * @param command the command + */ + private static void executeSchemaRequest(MCRSolrIndex index, JsonObject command) { + GenericSolrRequest request = new GenericSolrRequest(CollectionAdminRequest.METHOD.POST, + "/schema"); + request.setRequiresCollection(true); + request.withContent(command.toString().getBytes(StandardCharsets.UTF_8), "application/json"); + + MCRSolrAuthenticationManager.obtainInstance().applyAuthentication(request, + MCRSolrAuthenticationLevel.ADMIN); + String commandPrefix = command.keySet().stream().findFirst().orElse("unknown command"); + + try (InputStream is = MCRSolrSearchUtils.streamRequest(index.getClient(), request, "json")) { + String response = new String(is.readAllBytes(), StandardCharsets.UTF_8); + LOGGER.debug(() -> "SOLR config " + commandPrefix + " command was successful \n" + response); + } catch (SolrServerException e) { + LOGGER + .error(() -> "SOLR config " + commandPrefix + " error: " + e.getMessage() + "\n" + command, e); + } catch (IOException e) { + LOGGER.error(() -> "Could not execute the following Solr config command:\n" + command, e); + } + } } diff --git a/mycore-solr/src/main/java/org/mycore/solr/search/MCRSolrQueryAdapter.java b/mycore-solr/src/main/java/org/mycore/solr/search/MCRSolrQueryAdapter.java index fc7b8f891f..b96e845304 100644 --- a/mycore-solr/src/main/java/org/mycore/solr/search/MCRSolrQueryAdapter.java +++ b/mycore-solr/src/main/java/org/mycore/solr/search/MCRSolrQueryAdapter.java @@ -24,14 +24,15 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.request.SolrQuery; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.request.QueryRequest; -import org.apache.solr.client.solrj.request.SolrQuery; import org.apache.solr.client.solrj.response.QueryResponse; import org.mycore.frontend.servlets.MCRClassificationBrowser2.MCRQueryAdapter; import org.mycore.frontend.servlets.MCRServlet; import org.mycore.solr.MCRSolrConstants; -import org.mycore.solr.MCRSolrCoreManager; +import org.mycore.solr.MCRSolrIndexManager; import org.mycore.solr.auth.MCRSolrAuthenticationLevel; import org.mycore.solr.auth.MCRSolrAuthenticationManager; @@ -86,7 +87,9 @@ public long getResultCount() { QueryRequest queryRequest = new QueryRequest(solrQuery); MCRSolrAuthenticationManager.obtainInstance().applyAuthentication(queryRequest, MCRSolrAuthenticationLevel.SEARCH); - queryResponse = queryRequest.process(MCRSolrCoreManager.getMainSolrClient()); + SolrClient client = MCRSolrIndexManager.obtainInstance().requireMainIndex() + .getClient(); + queryResponse = queryRequest.process(client); } catch (SolrServerException | IOException e) { LOGGER.warn("Could not query SOLR.", e); return -1; diff --git a/mycore-solr/src/main/java/org/mycore/solr/search/MCRSolrSearchUtils.java b/mycore-solr/src/main/java/org/mycore/solr/search/MCRSolrSearchUtils.java index ed979a432a..708872b8a8 100644 --- a/mycore-solr/src/main/java/org/mycore/solr/search/MCRSolrSearchUtils.java +++ b/mycore-solr/src/main/java/org/mycore/solr/search/MCRSolrSearchUtils.java @@ -41,6 +41,8 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.solr.client.solrj.SolrClient; + +import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.request.QueryRequest; import org.apache.solr.client.solrj.request.SolrQuery; @@ -192,7 +194,21 @@ public static InputStream streamRawXML(SolrClient client, String path, SolrParam QueryRequest request = new QueryRequest(params); request.setPath(Objects.requireNonNull(path)); MCRSolrAuthenticationManager.obtainInstance().applyAuthentication(request, SEARCH); - InputStreamResponseParser responseParser = new InputStreamResponseParser("xml"); + return streamRequest(client, request, "xml"); + } + + /** + * Streams the solr response in the given format. + * @param client the client to query + * @param request the solr request to execute + * @param wt the format to stream the response in (e.g. "xml", "json", "csv") + * @return an InputStream of the solr response in the given format + * @throws SolrServerException Communication with the solr server failed in any way. + * @throws IOException If there is a low-level I/O error. + */ + public static InputStream streamRequest(SolrClient client, SolrRequest request, String wt) + throws SolrServerException, IOException { + InputStreamResponseParser responseParser = new InputStreamResponseParser(wt); request.setResponseParser(responseParser); NamedList nl = client.request(request); return (InputStream) nl.get("stream"); diff --git a/mycore-solr/src/main/java/org/mycore/solr/standalone/core/MCRConfigurableSolrCore.java b/mycore-solr/src/main/java/org/mycore/solr/standalone/core/MCRConfigurableSolrCore.java new file mode 100644 index 0000000000..83faa2633c --- /dev/null +++ b/mycore-solr/src/main/java/org/mycore/solr/standalone/core/MCRConfigurableSolrCore.java @@ -0,0 +1,186 @@ +/* + * This file is part of *** M y C o R e *** + * See https://www.mycore.de/ for details. + * + * MyCoRe is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * MyCoRe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with MyCoRe. If not, see . + */ + +package org.mycore.solr.standalone.core; + +import java.io.IOException; +import java.util.Optional; +import java.util.Set; + +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.impl.ConcurrentUpdateBaseSolrClient; +import org.apache.solr.client.solrj.impl.HttpSolrClientBase; +import org.apache.solr.client.solrj.impl.HttpSolrClientBuilderBase; +import org.apache.solr.client.solrj.jetty.ConcurrentUpdateJettySolrClient.Builder; +import org.apache.solr.client.solrj.jetty.HttpJettySolrClient; +import org.mycore.common.config.annotation.MCRConfigurationProxy; +import org.mycore.common.config.annotation.MCRProperty; +import org.mycore.solr.MCRAbstractHttpBasedIndexConfigAdapter; +import org.mycore.solr.MCRIndexType; +import org.mycore.solr.MCRSolrIndex; +import org.mycore.solr.MCRSolrUtils; +import org.mycore.solr.standalone.core.MCRConfigurableSolrCore.ConfigAdapter; + +@MCRConfigurationProxy(proxyClass = ConfigAdapter.class) +public class MCRConfigurableSolrCore implements MCRSolrIndex { + + private final String coreName; + private final HttpSolrClientBase client; + private final HttpSolrClientBase baseClient; + private final ConcurrentUpdateBaseSolrClient concurrentClient; + private final Set coreTypes; + + MCRConfigurableSolrCore(HttpSolrClientBase client, HttpSolrClientBase baseClient, + ConcurrentUpdateBaseSolrClient concurrentClient, String coreName, + Set coreTypes) { + this.client = client; + this.baseClient = baseClient; + this.concurrentClient = concurrentClient; + this.coreName = coreName; + this.coreTypes = coreTypes; + } + + @Override + public String getName() { + return coreName; + } + + @Override + public SolrClient getClient() { + return client; + } + + @Override + public SolrClient getBaseClient() { + return baseClient; + } + + @Override + public Optional getConcurrentClient() { + return Optional.ofNullable(this.concurrentClient); + } + + @Override + public Set getCoreTypes() { + return coreTypes; + } + + @Override + public void close() throws IOException { + MCRSolrUtils.shutdownSolrClient(this.client); + MCRSolrUtils.shutdownSolrClient(this.baseClient); + MCRSolrUtils.shutdownSolrClient(concurrentClient); + } + + public static class ConfigAdapter extends + MCRAbstractHttpBasedIndexConfigAdapter { + + private String solrUrl; + private String coreName; + private Integer concurrentQueueSize; + private Integer concurrentThreadCount; + private boolean concurrentEnabled; + + public String getCoreName() { + return coreName; + } + + @MCRProperty(name = "CoreName") + public void setCoreName(String coreName) { + this.coreName = coreName; + } + + public String getSolrUrl() { + return solrUrl; + } + + @MCRProperty(name = "SolrUrl") + public void setSolrUrl(String solrUrl) { + this.solrUrl = solrUrl; + } + + public Integer getConcurrentQueueSize() { + return concurrentQueueSize; + } + + @MCRProperty(name = "Concurrent.QueueSize", required = false, + defaultName = "MCR.Solr.Default.ConcurrentClient.QueueSize") + public void setConcurrentQueueSize(String concurrentQueueSize) { + this.concurrentQueueSize = Integer.parseInt(concurrentQueueSize); + } + + public Integer getConcurrentThreadCount() { + return concurrentThreadCount; + } + + @MCRProperty(name = "Concurrent.ThreadCount", required = false, + defaultName = "MCR.Solr.Default.ConcurrentClient.ThreadCount") + public void setConcurrentThreadCount(String concurrentThreadCount) { + this.concurrentThreadCount = Integer.parseInt(concurrentThreadCount); + } + + public boolean isConcurrentEnabled() { + return concurrentEnabled; + } + + @MCRProperty(name = "Concurrent.Enabled", required = false, + defaultName = "MCR.Solr.Default.ConcurrentClient.Enabled") + public void setConcurrentEnabled(String concurrentEnabled) { + this.concurrentEnabled = Boolean.parseBoolean(concurrentEnabled); + } + + @Override + public MCRConfigurableSolrCore get() { + + String normalizedSolrUrl = getSolrUrl(); + + if (!normalizedSolrUrl.endsWith("/")) { + normalizedSolrUrl += "/"; + } + + HttpSolrClientBuilderBase builder = getBuilder(); + + applySettings(builder); + + HttpSolrClientBase base = builder + .withBaseSolrUrl(normalizedSolrUrl + "solr/") + .build(); + + HttpSolrClientBase client = builder + .withBaseSolrUrl(normalizedSolrUrl + "solr/") + .withDefaultCollection(getCoreName()) + .build(); + + ConcurrentUpdateBaseSolrClient concurrentClient = null; + + if (this.isConcurrentEnabled() && useJettyHttpClient()) { + if (client instanceof HttpJettySolrClient jc) { + Builder concurrentBuilder = new Builder(normalizedSolrUrl, jc); + + concurrentBuilder.withThreadCount(getConcurrentThreadCount()); + concurrentBuilder.withQueueSize(getConcurrentQueueSize()); + + concurrentClient = concurrentBuilder.build(); + } + } + + return new MCRConfigurableSolrCore(client, base, concurrentClient, getCoreName(), buildCoreTypes()); + } + + } +} diff --git a/mycore-solr/src/main/resources/components/solr/config/deprecated.properties b/mycore-solr/src/main/resources/components/solr/config/deprecated.properties index cd285326d4..a89e125653 100644 --- a/mycore-solr/src/main/resources/components/solr/config/deprecated.properties +++ b/mycore-solr/src/main/resources/components/solr/config/deprecated.properties @@ -26,3 +26,18 @@ MCR.Module-solr.JoinQueryFields=MCR.Solr.JoinQueryFields MCR.Module-solr.NestedDocuments=MCR.Solr.NestedDocuments MCR.Module-solr.SolrClient.ShutdownSocketTimeout=MCR.Module-solr.SolrClient.SocketTimeout +MCR.Solr.SolrClient.JettyHttpClient.Enabled=MCR.Solr.Default.UseJettyHttpClient + +MCR.Solr.ConcurrentUpdateSolrClient.Enabled=MCR.Solr.Default.ConcurrentClient.Enabled +MCR.Solr.ConcurrentUpdateSolrClient.QueueSize=MCR.Solr.Default.ConcurrentClient.QueueSize +MCR.Solr.ConcurrentUpdateSolrClient.ThreadCount=MCR.Solr.Default.ConcurrentClient.ThreadCount + +MCR.Solr.SolrClient.ConnectionTimeout=MCR.Solr.Default.Client.ConnectionTimeout +MCR.Solr.SolrClient.SocketTimeout=MCR.Solr.Default.Client.RequestTimeout && MCR.Solr.Default.Client.IdleTimeout + +MCR.Solr.Core.main.ConfigSetTemplate=MCR.Solr.IndexManager.Index.main.ConfigSetTemplate +MCR.Solr.Core.main.Type=MCR.Solr.IndexManager.Index.main.CoreTypes + +MCR.Solr.Core.classification.ConfigSetTemplate=MCR.Solr.IndexManager.Index.classification.ConfigSetTemplate +MCR.Solr.Core.classification.Type=MCR.Solr.IndexManager.Index.classification.CoreTypes + diff --git a/mycore-solr/src/main/resources/components/solr/config/mycore.properties b/mycore-solr/src/main/resources/components/solr/config/mycore.properties index 43ce0059b2..b8d4d8656d 100644 --- a/mycore-solr/src/main/resources/components/solr/config/mycore.properties +++ b/mycore-solr/src/main/resources/components/solr/config/mycore.properties @@ -17,6 +17,37 @@ MCR.Solr.Server.Auth.Search.Class=org.mycore.solr.auth.MCRSolrNoOpAuthenticator # MCR.Solr.Server.Auth.Search.Username=searcher # MCR.Solr.Server.Auth.Search.Password=Alleswirdgut1! + +MCR.Solr.IndexManager.Class=org.mycore.solr.MCRConfigurableIndexManager +# MCR.Solr.IndexManager.Index.main.Class=org.mycore.solr.cloud.collection.MCRConfigurableSolrCloudCollection +# MCR.Solr.IndexManager.Index.main.CollectionName=mir +# MCR.Solr.IndexManager.Index.main.CoreTypes=main +# MCR.Solr.IndexManager.Index.main.SolrUrls=http\://localhost:8983/solr +# MCR.Solr.IndexManager.Index.main.ConfigSetTemplate=mycore_main + +# MCR.Solr.IndexManager.Index.classification.Class=org.mycore.solr.cloud.collection.MCRConfigurableSolrCloudCollection +# MCR.Solr.IndexManager.Index.classification.CollectionName=mir-classification +# MCR.Solr.IndexManager.Index.classification.CoreTypes=classification +# MCR.Solr.IndexManager.Index.classification.SolrUrls=http\://localhost:8983/solr +# MCR.Solr.IndexManager.Index.classification.ConfigSetTemplate=mycore_classification + +MCR.Solr.Default.ShardCount=1 +MCR.Solr.Default.ReplicaCount=1 +MCR.Solr.Default.UseJettyHttpClient=true + +MCR.Solr.Default.Client.IdleTimeout=50000 +MCR.Solr.Default.Client.IdleTimeout.Unit=MILLISECONDS + +MCR.Solr.Default.Client.ConnectionTimeout=0 +MCR.Solr.Default.Client.ConnectionTimeout.Unit=MILLISECONDS + +MCR.Solr.Default.Client.RequestTimeout=50000 +MCR.Solr.Default.Client.RequestTimeout.Unit=MILLISECONDS + +MCR.Solr.Default.ConcurrentClient.Enabled=true +MCR.Solr.Default.ConcurrentClient.QueueSize=100 +MCR.Solr.Default.ConcurrentClient.ThreadCount=%MCR.Solr.Indexer.ThreadCount% + MCR.Solr.Tika.Mapper.Default.Class=org.mycore.solr.index.file.tika.MCRTikaNOPMapper MCR.Solr.Tika.Mapper.x_tika_content.Class=org.mycore.solr.index.file.tika.MCRSimpleTikaMapper @@ -39,10 +70,10 @@ MCR.Solr.ConfigSet.mycore_classification.Class=org.mycore.solr.cloud.configsets. MCR.Solr.ConfigSet.mycore_classification.Files=managed-schema,solrconfig.xml,stopwords.txt,synonyms.txt MCR.Solr.ConfigSet.mycore_classification.Base=configset/mycore_main/conf/ -MCR.Solr.Core.main.ConfigSetTemplate=mycore_main -MCR.Solr.Core.main.Type=main -MCR.Solr.Core.classification.ConfigSetTemplate=mycore_classification -MCR.Solr.Core.classification.Type=classification +MCR.Solr.IndexManager.Index.main.ConfigSetTemplate=mycore_main +MCR.Solr.IndexManager.Index.main.CoreTypes=main +MCR.Solr.IndexManager.Index.classification.ConfigSetTemplate=mycore_classification +MCR.Solr.IndexManager.Index.classification.CoreTypes=classification MCR.EventHandler.MCRObject.100.Class=org.mycore.solr.index.MCRSolrIndexEventHandler MCR.EventHandler.MCRDerivate.100.Class=org.mycore.solr.index.MCRSolrIndexEventHandler @@ -69,12 +100,6 @@ MCR.Solr.ExtractPath=/update/extract #Proxy Servlet will only accept this comma separated list MCR.Solr.Proxy.WhiteList=/select -MCR.Solr.SolrClient.ConnectionTimeout=0 -MCR.Solr.SolrClient.SocketTimeout=50000 -MCR.Solr.SolrClient.JettyHttpClient.Enabled=true -MCR.Solr.ConcurrentUpdateSolrClient.Enabled=true -MCR.Solr.ConcurrentUpdateSolrClient.QueueSize=100 -MCR.Solr.ConcurrentUpdateSolrClient.ThreadCount=%MCR.Solr.Indexer.ThreadCount% MCR.Solr.SolrInputDocument.Factory=org.mycore.solr.index.document.MCRSolrTransformerInputDocumentFactory MCR.Solr.SolrInputDocument.Transformer=mycoreobject-solrdocument diff --git a/mycore-sword/src/main/java/org/mycore/sword/application/MCRSwordSolrObjectIDSupplier.java b/mycore-sword/src/main/java/org/mycore/sword/application/MCRSwordSolrObjectIDSupplier.java index 3d1b5335aa..a7ff5bbd62 100644 --- a/mycore-sword/src/main/java/org/mycore/sword/application/MCRSwordSolrObjectIDSupplier.java +++ b/mycore-sword/src/main/java/org/mycore/sword/application/MCRSwordSolrObjectIDSupplier.java @@ -27,7 +27,7 @@ import org.apache.solr.client.solrj.request.SolrQuery; import org.apache.solr.client.solrj.response.QueryResponse; import org.mycore.datamodel.metadata.MCRObjectID; -import org.mycore.solr.MCRSolrCoreManager; +import org.mycore.solr.MCRSolrIndexManager; import org.mycore.solr.auth.MCRSolrAuthenticationLevel; import org.mycore.solr.auth.MCRSolrAuthenticationManager; import org.swordapp.server.SwordServerException; @@ -59,7 +59,8 @@ public long getCount() throws SwordServerException { QueryRequest queryRequest = new QueryRequest(queryCopy); MCRSolrAuthenticationManager.obtainInstance().applyAuthentication(queryRequest, MCRSolrAuthenticationLevel.SEARCH); - final QueryResponse queryResponse = queryRequest.process(MCRSolrCoreManager.getMainSolrClient()); + final QueryResponse queryResponse = + queryRequest.process(MCRSolrIndexManager.obtainInstance().requireMainIndex().getClient()); return queryResponse.getResults().getNumFound(); } catch (SolrServerException | IOException e) { @@ -78,7 +79,8 @@ public List get(int from, int count) throws SwordServerException { QueryRequest queryRequest = new QueryRequest(queryCopy); MCRSolrAuthenticationManager.obtainInstance().applyAuthentication(queryRequest, MCRSolrAuthenticationLevel.SEARCH); - final QueryResponse queryResponse = queryRequest.process(MCRSolrCoreManager.getMainSolrClient()); + final QueryResponse queryResponse = + queryRequest.process(MCRSolrIndexManager.obtainInstance().requireMainIndex().getClient()); return queryResponse.getResults().stream() .map(r -> (String) r.getFieldValue("id")) .map(MCRObjectID::getInstance) diff --git a/pom.xml b/pom.xml index cf6fd0d6f0..49645d7f30 100644 --- a/pom.xml +++ b/pom.xml @@ -1389,6 +1389,11 @@ Applications based on MyCoRe use a common core, which provides the functionality solr-solrj-jetty ${solrj.version} + + org.apache.solr + solr-solrj-zookeeper + ${solrj.version} + org.apache.xmlgraphics fop-core