RUBY-3395 Vector Search GA (#2897)

Author: comandeo-mongo

.evergreen/config.yml

@@ -14,7 +14,7 @@ pre_error_fails_task: true
 command_type: system
 
 # Protect ourself against rogue test case, or curl gone wild, that runs forever.
-exec_timeout_secs: 3600
+exec_timeout_secs: 5400
 
 # What to do when evergreen hits the timeout (`post:` tasks are run automatically)
 timeout:
@@ -346,7 +346,7 @@ functions:
   "upload test results":
     - command: attach.xunit_results
       params:
-        file: ./src/tmp/rspec.xml
+        file: ./src/rspec.xml
 
   "delete private environment":
     - command: shell.exec
@@ -482,7 +482,7 @@ post:
   # Removed, causing timeouts
   # - func: "upload working dir"
   - func: "upload mo artifacts"
-  #- func: "upload test results"
+  # - func: "upload test results"
   - func: "upload test results to s3"
 
 task_groups:

.evergreen/config/common.yml.erb

@@ -11,7 +11,7 @@ pre_error_fails_task: true
 command_type: system
 
 # Protect ourself against rogue test case, or curl gone wild, that runs forever.
-exec_timeout_secs: 3600
+exec_timeout_secs: 5400
 
 # What to do when evergreen hits the timeout (`post:` tasks are run automatically)
 timeout:
@@ -343,7 +343,7 @@ functions:
   "upload test results":
     - command: attach.xunit_results
       params:
-        file: ./src/tmp/rspec.xml
+        file: ./src/rspec.xml
 
   "delete private environment":
     - command: shell.exec
@@ -479,7 +479,7 @@ post:
   # Removed, causing timeouts
   # - func: "upload working dir"
   - func: "upload mo artifacts"
-  #- func: "upload test results"
+  # - func: "upload test results"
   - func: "upload test results to s3"
 
 task_groups:

.evergreen/run-tests.sh

@@ -353,6 +353,7 @@ elif test "$SOLO" = 1; then
     fi
   done
 else
+  export JRUBY_OPTS=-J-Xmx2g
   bundle exec rake spec:ci
 fi
 
@@ -364,18 +365,18 @@ if test -f tmp/rspec-all.json; then
   mv tmp/rspec-all.json tmp/rspec.json
 fi
 
-kill_jruby
+kill_jruby || true
 
 if test -n "$OCSP_MOCK_PID"; then
   kill "$OCSP_MOCK_PID"
 fi
 
-python3 -m mtools.mlaunch.mlaunch stop --dir "$dbdir"
+python3 -m mtools.mlaunch.mlaunch stop --dir "$dbdir" || true
 
 if test -n "$FLE" && test "$DOCKER_PRELOAD" != 1; then
   # Terminate all kmip servers... and whatever else happens to be running
   # that is a python script.
-  pkill python3
+  pkill python3 || true
 fi
 
 exit ${test_status}

.mod/drivers-evergreen-tools

@@ -48,8 +48,8 @@ def initialize(collection, options = {})
       # @param [ nil | String ] name The name to give the new search index.
       #
       # @return [ String ] the name of the new search index.
-      def create_one(definition, name: nil)
-        create_many([ { name: name, definition: definition } ]).first
+      def create_one(definition, name: nil, type: 'search')
+        create_many([ { name: name, definition: definition, type: type } ]).first
       end
 
       # Create multiple search indexes with a single command.
@@ -99,7 +99,7 @@ def each(&block)
             s[:name] = requested_index_name if requested_index_name
           end
 
-          collection.aggregate(
+          collection.with(read_concern: {}).aggregate(
             [ { '$listSearchIndexes' => spec } ],
             aggregate_options
           )
@@ -200,7 +200,7 @@ def validate_search_index!(doc)
       #
       # @raise [ ArgumentError ] if the list contains any invalid keys
       def validate_search_index_keys!(keys)
-        extras = keys - [ 'name', 'definition', :name, :definition ]
+        extras = keys - [ 'name', 'definition', 'type', :name, :definition, :type ]
 
         raise ArgumentError, "invalid keys in search index creation: #{extras.inspect}" if extras.any?
       end

lib/mongo/search_index/view.rb

@@ -11,7 +11,8 @@ def create_search_index(op)
         model = args.use('model')
         name = model.use('name')
         definition = model.use('definition')
-        collection.search_indexes.create_one(definition, name: name)
+        type = model.use('type')
+        collection.search_indexes.create_one(definition, name: name, type: type)
       end
     end
 

spec/runners/unified/search_index_operations.rb

@@ -16,7 +16,13 @@ createEntities:
       collectionName: *collection0
 
 runOnRequirements:
-  - minServerVersion: "7.0.0"
+  # Skip server versions without fix of SERVER-83107 to avoid error message "BSON field 'createSearchIndexes.indexes.type' is an unknown field."
+  # SERVER-83107 was not backported to 7.1.
+  - minServerVersion: "7.0.5"
+    maxServerVersion: "7.0.99"
+    topologies: [ replicaset, load-balanced, sharded ]
+    serverless: forbid
+  - minServerVersion: "7.2.0"
     topologies: [ replicaset, load-balanced, sharded ]
     serverless: forbid
 
@@ -26,7 +32,7 @@ tests:
       - name: createSearchIndex
         object: *collection0
         arguments:
-          model: { definition: &definition { mappings: { dynamic: true } } }
+          model: { definition: &definition { mappings: { dynamic: true } } , type: 'search' }
         expectError:
           # This test always errors in a non-Atlas environment.  The test functions as a unit test  by asserting
           # that the driver constructs and sends the correct command.
@@ -39,15 +45,37 @@ tests:
           - commandStartedEvent:
               command:
                 createSearchIndexes: *collection0
-                indexes: [ { definition: *definition } ]
+                indexes: [ { definition: *definition, type: 'search'} ]
                 $db: *database0
 
   - description: "name provided for an index definition"
     operations:
       - name: createSearchIndex
         object: *collection0
         arguments: 
-          model: { definition: &definition { mappings: { dynamic: true } } , name: 'test index' }
+          model: { definition: &definition { mappings: { dynamic: true } } , name: 'test index', type: 'search' }
+        expectError:
+          # This test always errors in a non-Atlas environment.  The test functions as a unit test  by asserting
+          # that the driver constructs and sends the correct command.
+          # The expected error message was changed in SERVER-83003. Check for the substring "Atlas" shared by both error messages.
+          isError: true
+          errorContains: Atlas
+    expectEvents:
+      - client: *client0
+        events:
+          - commandStartedEvent:
+              command:
+                createSearchIndexes: *collection0
+                indexes: [ { definition: *definition, name: 'test index', type: 'search' } ]
+                $db: *database0
+
+  - description: "create a vector search index"
+    operations:
+      - name: createSearchIndex
+        object: *collection0
+        arguments:
+          model: { definition: &definition { fields: [ {"type": "vector", "path": "plot_embedding", "numDimensions": 1536, "similarity": "euclidean"} ] }
+            , name: 'test index', type: 'vectorSearch' }
         expectError:
           # This test always errors in a non-Atlas environment.  The test functions as a unit test  by asserting
           # that the driver constructs and sends the correct command.
@@ -60,5 +88,5 @@ tests:
           - commandStartedEvent:
               command:
                 createSearchIndexes: *collection0
-                indexes: [ { definition: *definition, name: 'test index' } ]
+                indexes: [ { definition: *definition, name: 'test index', type: 'vectorSearch' } ]
                 $db: *database0

spec/spec_tests/data/index_management/createSearchIndex.yml

@@ -16,7 +16,13 @@ createEntities:
       collectionName: *collection0
 
 runOnRequirements:
-  - minServerVersion: "7.0.0"
+  # Skip server versions without fix of SERVER-83107 to avoid error message "BSON field 'createSearchIndexes.indexes.type' is an unknown field."
+  # SERVER-83107 was not backported to 7.1.
+  - minServerVersion: "7.0.5"
+    maxServerVersion: "7.0.99"
+    topologies: [ replicaset, load-balanced, sharded ]
+    serverless: forbid
+  - minServerVersion: "7.2.0"
     topologies: [ replicaset, load-balanced, sharded ]
     serverless: forbid
 
@@ -48,7 +54,7 @@ tests:
       - name: createSearchIndexes
         object: *collection0
         arguments:
-          models: [ { definition: &definition { mappings: { dynamic: true } } } ]
+          models: [ { definition: &definition { mappings: { dynamic: true } } , type: 'search' } ]
         expectError:
           # This test always errors in a non-Atlas environment.  The test functions as a unit test  by asserting
           # that the driver constructs and sends the correct command.
@@ -61,15 +67,37 @@ tests:
           - commandStartedEvent:
               command:
                 createSearchIndexes: *collection0
-                indexes: [ { definition: *definition } ]
+                indexes: [ { definition: *definition, type: 'search'} ]
                 $db: *database0
 
   - description: "name provided for an index definition"
     operations:
       - name: createSearchIndexes
         object: *collection0
         arguments: 
-          models: [ { definition: &definition { mappings: { dynamic: true } } , name: 'test index' } ]
+          models: [ { definition: &definition { mappings: { dynamic: true } } , name: 'test index' , type: 'search' } ]
+        expectError:
+          # This test always errors in a non-Atlas environment.  The test functions as a unit test  by asserting
+          # that the driver constructs and sends the correct command.
+          # The expected error message was changed in SERVER-83003. Check for the substring "Atlas" shared by both error messages.
+          isError: true
+          errorContains: Atlas
+    expectEvents:
+      - client: *client0
+        events:
+          - commandStartedEvent:
+              command:
+                createSearchIndexes: *collection0
+                indexes: [ { definition: *definition, name: 'test index', type: 'search' } ]
+                $db: *database0
+
+  - description: "create a vector search index"
+    operations:
+      - name: createSearchIndexes
+        object: *collection0
+        arguments:
+          models: [ { definition: &definition { fields: [ {"type": "vector", "path": "plot_embedding", "numDimensions": 1536, "similarity": "euclidean"} ] },
+                                                name: 'test index' , type: 'vectorSearch' } ]
         expectError:
           # This test always errors in a non-Atlas environment.  The test functions as a unit test  by asserting
           # that the driver constructs and sends the correct command.
@@ -82,5 +110,5 @@ tests:
           - commandStartedEvent:
               command:
                 createSearchIndexes: *collection0
-                indexes: [ { definition: *definition, name: 'test index' } ]
+                indexes: [ { definition: *definition, name: 'test index', type: 'vectorSearch' } ]
                 $db: *database0

spec/spec_tests/data/index_management/createSearchIndexes.yml

@@ -15,14 +15,19 @@ createEntities:
       database: *database0
       collectionName: *collection0
 
+initialData:
+  - collectionName: *collection0
+    databaseName: *database0
+    documents:
+      - x: 1
+
 runOnRequirements:
   - minServerVersion: "7.0.0"
     topologies: [ replicaset, load-balanced, sharded ]
     serverless: forbid
 
 tests:
   - description: "when no name is provided, it does not populate the filter"
-    skipReason: https://jira.mongodb.org/browse/DRIVERS-2794
     operations:
       - name: listSearchIndexes
         object: *collection0
@@ -42,11 +47,10 @@ tests:
                   - $listSearchIndexes: {}
 
   - description: "when a name is provided, it is present in the filter"
-    skipReason: https://jira.mongodb.org/browse/DRIVERS-2794
     operations:
       - name: listSearchIndexes
         object: *collection0
-        arguments:
+        arguments: 
           name: &indexName "test index"
         expectError:
           # This test always errors in a non-Atlas environment.  The test functions as a unit test  by asserting
@@ -65,13 +69,12 @@ tests:
                 $db: *database0
 
   - description: aggregation cursor options are supported
-    skipReason: https://jira.mongodb.org/browse/DRIVERS-2794
     operations:
       - name: listSearchIndexes
         object: *collection0
-        arguments:
+        arguments: 
           name: &indexName "test index"
-          aggregationOptions:
+          aggregationOptions: 
             batchSize: 10
         expectError:
           # This test always errors in a non-Atlas environment.  The test functions as a unit test  by asserting