Merge branch 'main' into NODE-6245

Author: durran

Diff for: .evergreen/docker/Dockerfile.musl

@@ -12,10 +12,3 @@ git --version
 c++ --version
 g++ --version
 EOF
-
-# clone drivers evergreen tools into the same location that it lives on the host machine
-# this ensures any paths configured in variables by drivers-evergreen-tools work
-# in the container too.
-ARG DRIVERS_TOOLS=drivers-evergreen-tools
-WORKDIR ${DRIVERS_TOOLS}
-RUN git clone https://github.com/mongodb-labs/drivers-evergreen-tools.git .

Diff for: .evergreen/prepare-mongodb-aws-ecs-auth.sh

@@ -21,7 +21,7 @@ cat $ECS_SRC_DIR/.evergreen/run-mongodb-aws-ecs-test.sh
 
 # tar the file and drivers tools and do the same
 cd ..
-tar -czf src.tgz src drivers-tools
+tar -czf src.tgz src
 mv src.tgz $ECS_SRC_DIR/src.tgz
 
 export MONGODB_BINARIES="${MONGODB_BINARIES}"

Diff for: .evergreen/prepare-shell.sh

@@ -6,10 +6,9 @@
 
 # This script prepares a shell to run the remaining scripts in this folder
 # It MUST be kept idempotent! It will overwrite the orchestration config and expansion.yml file upon every run
-# and it will only clone drivers-tools if they do not exist one directory above our driver src
 
 PROJECT_DIRECTORY="$(pwd)"
-DRIVERS_TOOLS=$(cd .. && echo "$(pwd)/drivers-tools")
+DRIVERS_TOOLS="$(pwd)/drivers-evergreen-tools"
 MONGO_ORCHESTRATION_HOME="$DRIVERS_TOOLS/.evergreen/orchestration"
 MONGODB_BINARIES="$DRIVERS_TOOLS/mongodb/bin"
 UPLOAD_BUCKET="${project}"
@@ -30,12 +29,12 @@ export MONGODB_BINARIES
 export TMPDIR="$MONGO_ORCHESTRATION_HOME/db"
 export PATH="$MONGODB_BINARIES:$PATH"
 
-if [ ! -d "$DRIVERS_TOOLS" ]; then
-  # Only clone driver tools if it does not exist
-  git clone --depth=1 "https://github.com/mongodb-labs/drivers-evergreen-tools.git" "${DRIVERS_TOOLS}"
-fi
-
-echo "installed DRIVERS_TOOLS from commit $(git -C "${DRIVERS_TOOLS}" rev-parse HEAD)"
+# Note the evergreen option on git.get_project recurse_submodules does not work, so do it here.
+# We ignore errors in case we are running in a container where git doesn't trust the tmp directory.
+set +e
+git submodule init
+git submodule update
+set -e
 
 cat <<EOT > "$MONGO_ORCHESTRATION_HOME/orchestration.config"
 {

Diff for: .evergreen/run-alpine-fle-tests.sh

@@ -9,5 +9,10 @@ set -o xtrace
 export npm_config_cache=$(pwd)/.cache
 npm install
 
+# Fix to point at the drivers tools pems installed in src.
+export CSFLE_TLS_CA_FILE=$(pwd)/drivers-evergreen-tools/.evergreen/x509gen/ca.pem
+export CSFLE_TLS_CERT_FILE=$(pwd)/drivers-evergreen-tools/.evergreen/x509gen/server.pem
+export CSFLE_TLS_CLIENT_CERT_FILE=$(pwd)/drivers-evergreen-tools/.evergreen/x509gen/client.pem
+
 ALPINE=true \
     npm run check:csfle

Diff for: .evergreen/run-deployed-azure-kms-tests.sh

@@ -11,7 +11,7 @@ export AZUREKMS_PUBLICKEYPATH=/tmp/testazurekms_publickey
 export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms_privatekey
 
 echo "compressing node driver source and tools ... begin"
-tar -czf node-driver-source.tgz src drivers-tools
+tar -czf node-driver-source.tgz src
 echo "compressing node driver source and tools ... end"
 
 export AZUREKMS_SRC=node-driver-source.tgz

Diff for: .evergreen/run-mongodb-aws-ecs-test.sh

@@ -5,7 +5,7 @@ set -o errexit  # Exit the script with error if any of the commands fail
 export MONGODB_URI="$1"
 
 tar -xzf src/src.tgz
-# produces src/ and drivers-tools/
+# produces src/
 
 cd src
 

Diff for: .evergreen/run-oidc-tests-azure.sh

@@ -4,7 +4,7 @@ set -o errexit  # Exit the script with error if any of the commands fail
 
 export AZUREOIDC_DRIVERS_TAR_FILE=/tmp/node-mongodb-native.tgz
 cd ..
-tar -czf $AZUREOIDC_DRIVERS_TAR_FILE src drivers-tools
+tar -czf $AZUREOIDC_DRIVERS_TAR_FILE src
 cd -
 export AZUREOIDC_TEST_CMD="source ./env.sh && cd src && ENVIRONMENT=azure ./.evergreen/${SCRIPT}"
 export PROJECT_DIRECTORY=$PROJECT_DIRECTORY

Diff for: .evergreen/run-oidc-tests-gcp.sh

@@ -4,7 +4,7 @@ set -o errexit  # Exit the script with error if any of the commands fail
 
 export GCPOIDC_DRIVERS_TAR_FILE=/tmp/node-mongodb-native.tgz
 cd ..
-tar -czf $GCPOIDC_DRIVERS_TAR_FILE src drivers-tools
+tar -czf $GCPOIDC_DRIVERS_TAR_FILE src
 cd -
 export GCPOIDC_TEST_CMD="source ./secrets-export.sh drivers/gcpoidc && cd src && ENVIRONMENT=gcp ./.evergreen/${SCRIPT}"
 export PROJECT_DIRECTORY=$PROJECT_DIRECTORY

Diff for: .evergreen/run-oidc-tests-k8s.sh

@@ -4,7 +4,7 @@ set -o errexit  # Exit the script with error if any of the commands fail
 
 export K8S_DRIVERS_TAR_FILE=/tmp/node-mongodb-native.tgz
 cd ..
-tar -czf $K8S_DRIVERS_TAR_FILE src drivers-tools
+tar -czf $K8S_DRIVERS_TAR_FILE src
 cd -
 bash $DRIVERS_TOOLS/.evergreen/auth_oidc/k8s/setup-pod.sh
 bash $DRIVERS_TOOLS/.evergreen/auth_oidc/k8s/run-self-test.sh

Diff for: .github/dependabot.yml

@@ -5,6 +5,10 @@
 
 version: 2
 updates:
+  - package-ecosystem: "gitsubmodule" # See documentation for possible values
+    directory: "/"
+    schedule:
+      interval: "weekly"
   - package-ecosystem: "github-actions" # See documentation for possible values
     directory: "/" # Location of package manifests
     schedule:

Diff for: .gitmodules

@@ -0,0 +1,3 @@
+[submodule "drivers-evergreen-tools"]
+	path = drivers-evergreen-tools
+	url = https://github.com/mongodb-labs/drivers-evergreen-tools.git

Diff for: drivers-evergreen-tools

@@ -78,10 +78,17 @@ The actual implementations of the spec tests can be unit tests or integration te
 
 The easiest way to get started running the tests locally is to start a standalone server and run all of the tests.
 
-Start a `mongod` standalone with our [cluster_setup.sh](tools/cluster_setup.sh) script:
+Ensure the drivers tools submodule is cloned:
 
 ```sh
-./test/tools/cluster_setup.sh server
+git submodule init
+git submodule update
+```
+
+Start a `mongod` standalone with our [run-orchestration.sh](.evergreen/run-orchestration.sh) script with the environment set for the cluster:
+
+```sh
+VERSION='latest' TOPOLOGY='server' AUTH='noauth' ./.evergreen/run-orchestration.sh
 ```
 
 Then run the tests:
@@ -98,11 +105,10 @@ In the following subsections, we'll dig into the details of running the tests.
 
 ### Testing With Authorization-Enabled
 
-By default, the integration tests run with auth-enabled and the `cluster_setup.sh` script defaults to starting servers with auth-enabled. Tests can be run locally without auth by setting the environment
-variable `AUTH` to the value of `noauth`.  This must be a two-step process of starting a server without auth-enabled and then running the tests without auth-enabled.
+By default, the integration tests run with auth-enabled and the mongo orchestration script will run with auth enabled when the `AUTH` variable is set to `auth`. Tests can be run locally without auth by setting the environment variable `AUTH` to the value of `noauth`.  This must be a two-step process of starting a server without auth-enabled and then running the tests without auth-enabled.
 
 ```shell
-AUTH='noauth' ./test/tools/cluster_setup.sh <server>
+AUTH='noauth' TOPOLOGY='server' ./.evergreen/run-orchestration.sh
 AUTH='noauth' npm run check:test
 ```
 ### Testing Different MongoDB Topologies
@@ -112,21 +118,21 @@ As we mentioned earlier, the tests check the topology of the MongoDB server bein
 In the steps above, we started a standalone server:
 
 ```sh
-./test/tools/cluster_setup.sh server
+TOPOLOGY='server' ./.evergreen/run-orchestration.sh
 ```
 
-You can use the same [cluster_setup.sh](tools/cluster_setup.sh) script to start a replica set or sharded cluster by passing the appropriate option:
+You can use the same [run-orchestration.sh](.evergreen/run-orchestration.sh) script to start a replica set or sharded cluster by passing the appropriate option:
 ```sh
-./test/tools/cluster_setup.sh replica_set
+TOPOLOGY='replica_set' ./.evergreen/run-orchestration.sh
 ```
 or
 ```sh
-./test/tools/cluster_setup.sh sharded_cluster
+TOPOLOGY='sharded_cluster' ./.evergreen/run-orchestration.sh
 ```
 If you are running more than a standalone server, make sure your `ulimit` settings are in accordance with [MongoDB's recommendations][mongodb-ulimit]. Changing the settings on the latest versions of macOS can be tricky. See [this article][macos-ulimt] for tips. (You likely don't need to do the complicated `maxproc` steps.)
 
-The [cluster_setup.sh](tools/cluster_setup.sh) script automatically stores the files associated with the MongoDB server in the `data` directory, which is stored at the top-level of this repository.
-You can delete this directory if you want to ensure you're running a clean configuration. If you delete the directory, the associated database server will be stopped, and you will need to run [cluster_setup.sh](tools/cluster_setup.sh) again.
+The [run-orchestration.sh](.evergreen/run-orchestration.sh) script automatically stores the files associated with the MongoDB server in the `data` directory, which is stored at the top-level of this repository.
+You can delete this directory if you want to ensure you're running a clean configuration. If you delete the directory, the associated database server will be stopped, and you will need to run [run-orchestration.sh](.evergreen/run-orchestration.sh) again.
 
 You can prefix `npm test` with a `MONGODB_URI` environment variable to point the tests to a specific deployment. For example, for a standalone server, you might use:
 
@@ -360,12 +366,17 @@ In order to test some features, you will need to generate and set a specialized
 
 We recommend using a different terminal for each specialized environment to avoid the environment variables from one specialized environment impacting the test runs for another specialized environment.
 
-Before you begin any of the subsections below, clone the [drivers-evergreen-tools repo](https://github.com/mongodb-labs/drivers-evergreen-tools.git).
+Before you begin any of the subsections below, ensure the drivers-evergreen-tools submodule is updated:
+
+```sh
+git submodule init
+git submodule update
+```
 
-We recommend creating an environment variable named `DRIVERS_TOOLS` that stores the path to your local copy of the `driver-evergreen-tools` repo (code examples in this section will assume this has been done):
+We recommend creating an environment variable named `DRIVERS_TOOLS` that stores the path to the `driver-evergreen-tools` submodule (code examples in this section will assume this has been done):
 
 ```sh
-export DRIVERS_TOOLS="/path/to/your/copy/of/drivers-evergreen-tools"
+export DRIVERS_TOOLS="./drivers-evergreen-tools"
 ```
 
 ### Serverless