Skip to content

Commit 4cbff11

Browse files
dimorportheca47lawofcycles
authored andcommitted
add skippping deleteByQuery in RestRepository.delete() for OpenSearch Serverless
Add conditional logic to bypass deleteByQuery operations when targeting serverless clusters, ensuring compatibility with serverless architecture limitations.
1 parent 45bf9bd commit 4cbff11

1 file changed

Lines changed: 32 additions & 56 deletions

File tree

mr/src/main/java/org/opensearch/hadoop/rest/RestRepository.java

Lines changed: 32 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -277,56 +277,30 @@ public Map<ShardInfo, NodeInfo> getWriteTargetPrimaryShards(boolean clientNodesO
277277
}
278278

279279
protected Map<ShardInfo, NodeInfo> doGetWriteTargetPrimaryShards(boolean clientNodesOnly) {
280-
// For serverless mode, we skip shard information retrieval as it's not supported
281-
if (settings.getServerlessMode()) {
282-
Map<ShardInfo, NodeInfo> shards = new LinkedHashMap<ShardInfo, NodeInfo>();
283-
List<NodeInfo> nodes = client.getHttpNodes(clientNodesOnly);
284-
285-
if (nodes.isEmpty()) {
286-
log.warn("Cannot find any nodes (is HTTP enabled?)");
287-
return null;
288-
}
289-
290-
// In serverless mode, we create a single dummy shard assigned to the first available node
291-
NodeInfo firstNode = nodes.get(0);
292-
Map<String, Object> dummyShardData = new HashMap<String, Object>();
293-
dummyShardData.put("index", resources.getResourceWrite().index());
294-
dummyShardData.put("shard", 0);
295-
dummyShardData.put("primary", true);
296-
dummyShardData.put("node", firstNode.getId());
297-
dummyShardData.put("state", "STARTED");
298-
dummyShardData.put("relocating_node", null);
299-
300-
ShardInfo dummyShard = new ShardInfo(dummyShardData);
301-
shards.put(dummyShard, firstNode);
302-
303-
return shards;
304-
} else {
305-
List<List<Map<String, Object>>> info = client.targetShards(resources.getResourceWrite().index(), SettingsUtils.getFixedRouting(settings));
306-
Map<ShardInfo, NodeInfo> shards = new LinkedHashMap<ShardInfo, NodeInfo>();
307-
List<NodeInfo> nodes = client.getHttpNodes(clientNodesOnly);
308-
Map<String, NodeInfo> nodeMap = new HashMap<String, NodeInfo>(nodes.size());
309-
for (NodeInfo node : nodes) {
310-
nodeMap.put(node.getId(), node);
311-
}
280+
List<List<Map<String, Object>>> info = client.targetShards(resources.getResourceWrite().index(), SettingsUtils.getFixedRouting(settings));
281+
Map<ShardInfo, NodeInfo> shards = new LinkedHashMap<ShardInfo, NodeInfo>();
282+
List<NodeInfo> nodes = client.getHttpNodes(clientNodesOnly);
283+
Map<String, NodeInfo> nodeMap = new HashMap<String, NodeInfo>(nodes.size());
284+
for (NodeInfo node : nodes) {
285+
nodeMap.put(node.getId(), node);
286+
}
312287

313-
for (List<Map<String, Object>> shardGroup : info) {
314-
// consider only primary shards
315-
for (Map<String, Object> shardData : shardGroup) {
316-
ShardInfo shard = new ShardInfo(shardData);
317-
if (shard.isPrimary()) {
318-
NodeInfo node = nodeMap.get(shard.getNode());
319-
if (node == null) {
320-
log.warn(String.format("Cannot find node with id [%s] (is HTTP enabled?) from shard [%s] in nodes [%s]; layout [%s]", shard.getNode(), shard, nodes, info));
321-
return null;
322-
}
323-
shards.put(shard, node);
324-
break;
288+
for (List<Map<String, Object>> shardGroup : info) {
289+
// consider only primary shards
290+
for (Map<String, Object> shardData : shardGroup) {
291+
ShardInfo shard = new ShardInfo(shardData);
292+
if (shard.isPrimary()) {
293+
NodeInfo node = nodeMap.get(shard.getNode());
294+
if (node == null) {
295+
log.warn(String.format("Cannot find node with id [%s] (is HTTP enabled?) from shard [%s] in nodes [%s]; layout [%s]", shard.getNode(), shard, nodes, info));
296+
return null;
325297
}
298+
shards.put(shard, node);
299+
break;
326300
}
327301
}
328-
return shards;
329302
}
303+
return shards;
330304
}
331305

332306
public MappingSet getMappings() {
@@ -403,18 +377,20 @@ public boolean touch() {
403377
}
404378

405379
public void delete() {
406-
// try first a blind delete by query
407-
try {
408-
Resource res = resources.getResourceWrite();
409-
client.deleteByQuery(
410-
res.isTyped()
411-
? res.index() + "/" + res.type()
412-
: res.index(),
413-
MatchAllQueryBuilder.MATCH_ALL);
414-
} catch (OpenSearchHadoopInvalidRequest ehir) {
415-
log.error("Delete by query was not successful...", ehir);
380+
if (!this.settings.getServerlessMode()) {
381+
// try first a blind delete by query
382+
try {
383+
Resource res = resources.getResourceWrite();
384+
client.deleteByQuery(
385+
res.isTyped()
386+
? res.index() + "/" + res.type()
387+
: res.index(),
388+
MatchAllQueryBuilder.MATCH_ALL);
389+
} catch (OpenSearchHadoopInvalidRequest ehir) {
390+
log.error("Delete by query was not successful...", ehir);
391+
}
416392
}
417-
393+
418394
// in ES 2.0 and higher this means scrolling and deleting the docs by hand...
419395
// do a scroll-scan without source
420396

0 commit comments

Comments
 (0)