SERVER-83119 Qualify a ClusteredIndexscan as an allowed operation when notablescan is set. (#17342)

GitOrigin-RevId: 0556bfe

Author: HCSPete

etc/backports_required_for_multiversion_tests.yml

@@ -553,6 +553,12 @@ last-continuous:
     ticket: SERVER-84130
   - test_file: jstests/noPassthrough/disallow_new_transactions_at_shutdown_on_mongos.js
     ticket: SERVER-77667
+  - test_file: jstests/core/notablescan.js
+    ticket: SERVER-83119
+  - test_file: jstests/sharding/clustered_coll_scan.js
+    ticket: SERVER-83119
+  - test_file: jstests/sharding/eof_plan.js
+    ticket: SERVER-83119
   suites: null
 last-lts:
   all:
@@ -1164,4 +1170,10 @@ last-lts:
     ticket: SERVER-84130
   - test_file: jstests/noPassthrough/disallow_new_transactions_at_shutdown_on_mongos.js
     ticket: SERVER-77667
+  - test_file: jstests/core/notablescan.js
+    ticket: SERVER-83119
+  - test_file: jstests/sharding/clustered_coll_scan.js
+    ticket: SERVER-83119
+  - test_file: jstests/sharding/eof_plan.js
+    ticket: SERVER-83119
   suites: null

jstests/core/notablescan.js

@@ -16,44 +16,69 @@
 //   tenant_migration_incompatible,
 // ]
 
-let t = db.test_notablescan;
-t.drop();
+import {isClusteredIxscan} from "jstests/libs/analyze_plan.js";
+import {assertDropAndRecreateCollection} from "jstests/libs/collection_drop_recreate.js";
 
-try {
-    assert.commandWorked(db._adminCommand({setParameter: 1, notablescan: true}));
-    // commented lines are SERVER-2222
-    if (0) {  // SERVER-2222
-        assert.throws(function() {
-            t.find({a: 1}).toArray();
-        });
-    }
-    t.save({a: 1});
-    assert.throws(function() {
-        t.count({a: 1});
-    });
+function checkError(err) {
+    assert.includes(err.toString(), "'notablescan'");
+}
+
+const colName = jsTestName();
+let coll = db.getCollection(colName);
+coll.drop();
+
+assert.commandWorked(db.adminCommand({setParameter: 1, notablescan: true}));
+
+{
     if (0) {
+        // TODO: SERVER-2222 This should actually throw an error as it performs a collection
+        // scan.
         assert.throws(function() {
-            t.find({}).toArray();
+            coll.find({a: 1}).toArray();
         });
     }
-    assert.eq(1, t.find({}).itcount());  // SERVER-274
 
+    coll.insert({a: 1});
     let err = assert.throws(function() {
-        t.find({a: 1}).toArray();
+        coll.count({a: 1});
     });
-    assert.includes(err.toString(), "No indexed plans available, and running with 'notablescan'");
+    checkError(err);
+
+    // TODO: SERVER-2222 This should actually throw an error as it performs a collection scan.
+    assert.eq(1, coll.find({}).itcount());
+
+    err = assert.throws(function() {
+        coll.find({a: 1}).toArray();
+    });
+    checkError(err);
 
     err = assert.throws(function() {
-        t.find({a: 1}).hint({$natural: 1}).toArray();
+        coll.find({a: 1}).hint({$natural: 1}).toArray();
     });
     assert.includes(err.toString(), "$natural");
-    assert.includes(err.toString(), "notablescan");
-
-    t.createIndex({a: 1});
-    assert.eq(0, t.find({a: 1, b: 1}).itcount());
-    assert.eq(1, t.find({a: 1, b: null}).itcount());
-} finally {
-    // We assume notablescan was false before this test started and restore that
-    // expected value.
-    assert.commandWorked(db._adminCommand({setParameter: 1, notablescan: false}));
+    checkError(err);
+
+    coll.createIndex({a: 1});
+    assert.eq(0, coll.find({a: 1, b: 1}).itcount());
+    assert.eq(1, coll.find({a: 1, b: null}).itcount());
+}
+
+{  // Run the testcase with a clustered index.
+    assertDropAndRecreateCollection(db, colName, {clusteredIndex: {key: {_id: 1}, unique: true}})
+    coll = db.getCollection(colName);
+    assert.commandWorked(coll.insert({_id: 22}));
+    assert.eq(1, coll.find({_id: 22}).itcount());
+    let plan = coll.find({_id: 22}).explain();
+    // Make sure the plan has a clustered index scan.
+    assert(isClusteredIxscan(db, plan));
+
+    // Make sure the same works with an aggregate.
+    assert.eq(1, coll.aggregate([{$match: {_id: 22}}]).itcount());
+    plan = coll.explain().aggregate([{$match: {_id: 22}}]);
+    // Make sure the plan has a clustered index scan.
+    assert(isClusteredIxscan(db, plan));
+    assert.commandWorked(
+        db.runCommand({aggregate: colName, pipeline: [{$match: {_id: 22}}], cursor: {}}));
 }
+// Set it back to the original value.
+assert.commandWorked(db.adminCommand({setParameter: 1, notablescan: false}));

jstests/sharding/clustered_coll_scan.js

@@ -0,0 +1,45 @@
+/*
+ * Testing if mongos can deal with clustered collections and the related clustered IDX scan bounds
+ * (SERVER-83119)
+ */
+
+import {isClusteredIxscan} from "jstests/libs/analyze_plan.js";
+import {assertDropAndRecreateCollection} from "jstests/libs/collection_drop_recreate.js";
+
+const st = new ShardingTest({
+    shards: 2,
+    mongos: 1,
+});
+
+st.s.adminCommand({enableSharding: "test"});
+
+const db = st.getDB("test");
+// Create the collection as a clustered collection.
+const coll = assertDropAndRecreateCollection(
+    db, jsTestName(), {clusteredIndex: {key: {_id: 1}, unique: true}})
+st.shardColl(coll, {a: 1});
+// First of all check that we can execute the query.
+assert.commandWorked(coll.insertMany([...Array(10).keys()].map(i => {
+    return {_id: i, a: i};
+})));
+
+{
+    var explain = coll.find({_id: 2}).explain();
+    // Make sure that we have a clusteredIDXScan in the plan.
+
+    assert(isClusteredIxscan(db, explain));
+    assert.commandWorked(
+        st.getPrimaryShard("test").adminCommand({setParameter: 1, notablescan: 1}));
+    // Do the same thing only with notablescan enabled.
+    explain = coll.find({_id: 2}).explain();
+    assert(isClusteredIxscan(db, explain));
+    // Sanity count check.
+    assert.eq(1, coll.find({_id: 2}).itcount());
+}
+// Test the same with aggregate.
+{
+    var explain = coll.explain().aggregate([{$match: {_id: 22}}]);
+    assert(isClusteredIxscan(db, explain));
+}
+
+st.stop();

jstests/sharding/eof_plan.js

@@ -0,0 +1,33 @@
+/*
+ * In the case of an always false predicate the shards will return an EOF plan. Additionally this
+ * tests that mongos can deal with EOF plans coming from the shard.
+ */
+
+import {getWinningPlanFromExplain} from 'jstests/libs/analyze_plan.js'
+const st = new ShardingTest({
+    shards: 2,
+    mongos: 1,
+});
+
+st.s.adminCommand({enableSharding: "test"});
+const db = st.getDB("test");
+const coll = db[jsTestName()];
+st.shardColl(coll, {array: 1});
+
+assert.commandWorked(coll.insertMany([...Array(10).keys()].map(i => {
+    return {_id: i, a: 1};
+})));
+
+{
+    assert.eq(coll.aggregate([
+                      {"$match": {"array": {"$all": []}}},
+                      {"$sort": {"a": -1, "_id": 1}},
+                      {"$limit": 6}
+                  ])
+                  .itcount(),
+              0);
+    const explain = coll.explain().aggregate([{"$match": {"array": {"$all": []}}}]);
+    // check that we received an EOF plan
+    assert.eq(getWinningPlanFromExplain(explain).stage, "EOF");
+    st.stop();
+}

src/mongo/db/query/plan_explainer_sbe.cpp

@@ -107,8 +107,8 @@ void statsToBSON(const QuerySolutionNode* node,
 
     StageType nodeType = node->getType();
     if ((nodeType == STAGE_COLLSCAN) &&
-        static_cast<const CollectionScanNode*>(node)->doSbeClusteredCollectionScan()) {
-        bob->append("stage", sbeClusteredCollectionScanToString());
+        static_cast<const CollectionScanNode*>(node)->doClusteredCollectionScanSbe()) {
+        bob->append("stage", clusteredCollectionScanSbeToString());
     } else {
         bob->append("stage", stageTypeToString(node->getType()));
     }

src/mongo/db/query/query_planner.cpp

@@ -649,6 +649,9 @@ string optionString(size_t options) {
             case QueryPlannerParams::GENERATE_PER_COLUMN_FILTERS:
                 ss << "GENERATE_PER_COLUMN_FILTERS ";
                 break;
+            case QueryPlannerParams::STRICT_NO_TABLE_SCAN:
+                ss << "STRICT_NO_TABLE_SCAN ";
+                break;
             case QueryPlannerParams::DEFAULT:
                 MONGO_UNREACHABLE;
                 break;
@@ -1082,13 +1085,42 @@ StatusWith<std::vector<std::unique_ptr<QuerySolution>>> singleSolution(
     return {std::move(out)};
 }
 
-bool canTableScan(const QueryPlannerParams& params) {
-    return !(params.options & QueryPlannerParams::NO_TABLE_SCAN);
+// If no table scan option is set the planner may not return any plan containing a collection scan.
+// Yet clusteredIdxScans are still allowed as they are not a full collection scan but a bounded
+// collection scan.
+bool noTableScan(const QueryPlannerParams& params) {
+    return (params.options & QueryPlannerParams::NO_TABLE_SCAN);
+}
+
+// Used internally if the planner should also avoid retruning a plan containing a clusteredIDX scan.
+bool noTableAndClusteredIDXScan(const QueryPlannerParams& params) {
+    return (params.options & QueryPlannerParams::STRICT_NO_TABLE_SCAN);
+}
+
+bool isClusteredScan(QuerySolutionNode* node) {
+    if (node->getType() == STAGE_COLLSCAN) {
+        auto collectionScanSolnNode = dynamic_cast<CollectionScanNode*>(node);
+        return (collectionScanSolnNode->doClusteredCollectionScanClassic() ||
+                collectionScanSolnNode->doClusteredCollectionScanSbe());
+    }
+    return false;
+}
+
+// Check if this is a real coll scan or a hidden ClusteredIDX scan.
+bool isColusteredIDXScanSoln(QuerySolution* collscanSoln) {
+    if (collscanSoln->root()->getType() == STAGE_SHARDING_FILTER) {
+        auto child = collscanSoln->root()->children.begin();
+        return isClusteredScan(child->get());
+    }
+    if (collscanSoln->root()->getType() == STAGE_COLLSCAN) {
+        return isClusteredScan(collscanSoln->root());
+    }
+    return false;
 }
 
 StatusWith<std::vector<std::unique_ptr<QuerySolution>>> attemptCollectionScan(
     const CanonicalQuery& query, bool isTailable, const QueryPlannerParams& params) {
-    if (!canTableScan(params)) {
+    if (noTableScan(params)) {
         return Status(ErrorCodes::NoQueryExecutionPlans,
                       "not allowed to output a collection scan because 'notablescan' is enabled");
     }
@@ -1689,9 +1721,9 @@ StatusWith<std::vector<std::unique_ptr<QuerySolution>>> QueryPlanner::plan(
 
     // No indexed plans?  We must provide a collscan if possible or else we can't run the query.
     bool collScanRequired = 0 == out.size();
-    if (collScanRequired && !canTableScan(params)) {
+    if (collScanRequired && noTableAndClusteredIDXScan(params)) {
         return Status(ErrorCodes::NoQueryExecutionPlans,
-                      "No indexed plans available, and running with 'notablescan'");
+                      "No indexed plans available, and running with 'notablescan' 1");
     }
 
     bool clusteredCollection = params.clusteredInfo.has_value();
@@ -1706,6 +1738,7 @@ StatusWith<std::vector<std::unique_ptr<QuerySolution>>> QueryPlanner::plan(
         return Status(ErrorCodes::NoQueryExecutionPlans, "No query solutions");
     }
 
+    bool isClusteredIDXScan = false;
     if (possibleToCollscan && (collscanRequested || collScanRequired || clusteredCollection)) {
         boost::optional<int> clusteredScanDirection =
             QueryPlannerCommon::determineClusteredScanDirection(query, params);
@@ -1715,7 +1748,7 @@ StatusWith<std::vector<std::unique_ptr<QuerySolution>>> QueryPlanner::plan(
             return Status(ErrorCodes::NoQueryExecutionPlans,
                           "Failed to build collection scan soln");
         }
-
+        isClusteredIDXScan = isColusteredIDXScanSoln(collscanSoln.get());
         // We consider collection scan in the following cases:
         // 1. collScanRequested - specifically requested by caller.
         // 2. collScanRequired - there are no other possible plans, so we fallback to full scan.
@@ -1736,8 +1769,15 @@ StatusWith<std::vector<std::unique_ptr<QuerySolution>>> QueryPlanner::plan(
             out.push_back(std::move(collscanSoln));
         }
     }
+    // Make sure to respect the notablescan option. A clustered IDX scan is allowed even under a
+    // NOTABLE option. Only in the case of a strict NOTABLE scan option a clustered IDX scan is not
+    // allowed. This option is used in mongoS for shardPruning.
+    invariant(out.size() > 0);
+    if (collScanRequired && noTableScan(params) && !isClusteredIDXScan) {
+        return Status(ErrorCodes::NoQueryExecutionPlans,
+                      "No indexed plans available, and running with 'notablescan' 2");
+    }
 
-    invariant(!out.empty());
     return {std::move(out)};
 }  // QueryPlanner::plan
 

src/mongo/db/query/query_planner_params.h

@@ -160,6 +160,11 @@ struct QueryPlannerParams {
         // applied per-column. This is off by default, since the execution side doesn't support it
         // yet.
         GENERATE_PER_COLUMN_FILTERS = 1 << 11,
+
+        // This is an extension to the NO_TABLE_SCAN parameter. This more stricter option will also
+        // avoid a CLUSTEREDIDX_SCAN which comes built into a collection scan when the collection is
+        // clustered.
+        STRICT_NO_TABLE_SCAN = 1 << 12,
     };
 
     // See Options enum above.

src/mongo/db/query/query_solution.cpp

@@ -393,9 +393,33 @@ void CollectionScanNode::computeProperties() {
     }
 }
 
+/*
+ * IndexBounds exist when this collection scan is a ClusteredIndexScan. Since we need the index
+ * bounds very seldomly it is built adHoc instead of precomputed.
+ */
+IndexBounds CollectionScanNode::getIndexBounds() const {
+    tassert(
+        8311900,
+        "Requesting index bounds on a non ClusteredIndexScan (hidden in a collection scan node)",
+        !(doClusteredCollectionScanClassic() || doClusteredCollectionScanSbe()));
+
+    IndexBounds clusteredIdxScanBounds;
+    BSONObjBuilder maxRecordBson;
+    maxRecord->appendToBSONAs(&maxRecordBson, clusteredIndex->getName().value());
+    BSONObjBuilder minRecordBson;
+    minRecord->appendToBSONAs(&minRecordBson, clusteredIndex->getName().value());
+    clusteredIdxScanBounds.endKey = maxRecordBson.obj();
+    clusteredIdxScanBounds.startKey = minRecordBson.obj();
+    return clusteredIdxScanBounds;
+}
+
 void CollectionScanNode::appendToString(str::stream* ss, int indent) const {
     addIndent(ss, indent);
-    *ss << "COLLSCAN\n";
+    if (doClusteredCollectionScanClassic() || doClusteredCollectionScanSbe()) {
+        *ss << "CLUSTERED_IDXSCAN\n";
+    } else {
+        *ss << "COLLSCAN\n";
+    }
     addIndent(ss, indent + 1);
     *ss << "ns = " << toStringForLogging(nss) << '\n';
     if (nullptr != filter) {
@@ -413,6 +437,9 @@ std::unique_ptr<QuerySolutionNode> CollectionScanNode::clone() const {
     copy->tailable = this->tailable;
     copy->direction = this->direction;
     copy->isClustered = this->isClustered;
+    copy->minRecord = this->minRecord;
+    copy->maxRecord = this->maxRecord;
+    copy->clusteredIndex = this->clusteredIndex;
     copy->isOplog = this->isOplog;
     copy->shouldTrackLatestOplogTimestamp = this->shouldTrackLatestOplogTimestamp;
     copy->assertTsHasNotFallenOff = this->assertTsHasNotFallenOff;

src/mongo/db/query/query_solution.h

@@ -550,7 +550,7 @@ struct CollectionScanNode : public QuerySolutionNodeWithSortSet {
     }
 
     // Tells whether this scan will be performed as a clustered collection scan in SBE.
-    bool doSbeClusteredCollectionScan() const {
+    bool doClusteredCollectionScanSbe() const {
         return (isClustered && !isOplog && (minRecord || maxRecord || resumeAfterRecordId));
     }
 
@@ -563,6 +563,8 @@ struct CollectionScanNode : public QuerySolutionNodeWithSortSet {
         eligibleForPlanCache = false;
     }
 
+    IndexBounds getIndexBounds() const;
+
     std::unique_ptr<QuerySolutionNode> clone() const final;
 
     void hash(absl::HashState state) const override {