From da089153088b44cd61be4e37a7222a123277d3d1 Mon Sep 17 00:00:00 2001 From: mdb-ad <198671546+mdb-ad@users.noreply.github.com> Date: Sun, 4 May 2025 23:01:01 -0700 Subject: [PATCH 01/17] support unified tests --- .../src/mongoc/mongoc-util-private.h | 3 + src/libmongoc/src/mongoc/mongoc-util.c | 12 + .../unified/fle2v2-BypassQueryAnalysis.json | 322 +++++++++++++++++ ...EncryptedFields-vs-EncryptedFieldsMap.json | 256 +++++++++++++ .../unified/localSchema.json | 336 ++++++++++++++++++ .../unified/maxWireVersion.json | 101 ++++++ .../crud/unified/client-bulkWrite-qe.json | 272 ++++++++++++++ .../unified/poc-queryable-encryption.json | 179 ++++++++++ .../tests/json/unified/poc-transactions.json | 6 +- src/libmongoc/tests/unified/entity-map.c | 186 ++++++++-- src/libmongoc/tests/unified/operation.c | 5 +- src/libmongoc/tests/unified/result.c | 2 + src/libmongoc/tests/unified/runner.c | 182 ++++++++-- 13 files changed, 1796 insertions(+), 66 deletions(-) create mode 100644 src/libmongoc/tests/json/client_side_encryption/unified/fle2v2-BypassQueryAnalysis.json create mode 100644 src/libmongoc/tests/json/client_side_encryption/unified/fle2v2-EncryptedFields-vs-EncryptedFieldsMap.json create mode 100644 src/libmongoc/tests/json/client_side_encryption/unified/localSchema.json create mode 100644 src/libmongoc/tests/json/client_side_encryption/unified/maxWireVersion.json create mode 100644 src/libmongoc/tests/json/crud/unified/client-bulkWrite-qe.json create mode 100644 src/libmongoc/tests/json/unified/poc-queryable-encryption.json diff --git a/src/libmongoc/src/mongoc/mongoc-util-private.h b/src/libmongoc/src/mongoc/mongoc-util-private.h index 062fcb431bb..84e007396ca 100644 --- a/src/libmongoc/src/mongoc/mongoc-util-private.h +++ b/src/libmongoc/src/mongoc/mongoc-util-private.h @@ -100,6 +100,9 @@ mongoc_ends_with (const char *str, const char *suffix); void mongoc_lowercase (const char *src, char *buf /* OUT */); +void +mongoc_lowercase_inplace (char *src); + bool mongoc_parse_port (uint16_t *port, const char *str); diff --git a/src/libmongoc/src/mongoc/mongoc-util.c b/src/libmongoc/src/mongoc/mongoc-util.c index 54d8be56f92..5a790ae06e5 100644 --- a/src/libmongoc/src/mongoc/mongoc-util.c +++ b/src/libmongoc/src/mongoc/mongoc-util.c @@ -526,6 +526,18 @@ mongoc_lowercase (const char *src, char *buf /* OUT */) } } +void +mongoc_lowercase_inplace (char *src) +{ + for (; *src; ++src) { + /* UTF8 non-ascii characters have a 1 at the leftmost bit. If this is the + * case, just copy */ + if ((*src & (0x1 << 7)) == 0) { + *src = (char) tolower (*src); + } + } +} + bool mongoc_parse_port (uint16_t *port, const char *str) { diff --git a/src/libmongoc/tests/json/client_side_encryption/unified/fle2v2-BypassQueryAnalysis.json b/src/libmongoc/tests/json/client_side_encryption/unified/fle2v2-BypassQueryAnalysis.json new file mode 100644 index 00000000000..0817508f8f8 --- /dev/null +++ b/src/libmongoc/tests/json/client_side_encryption/unified/fle2v2-BypassQueryAnalysis.json @@ -0,0 +1,322 @@ +{ + "description": "fle2v2-BypassQueryAnalysis", + "schemaVersion": "1.23", + "runOnRequirements": [ + { + "minServerVersion": "7.0.0", + "serverless": "forbid", + "csfle": true, + "topologies": [ + "replicaset", + "sharded", + "load-balanced" + ] + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "autoEncryptOpts": { + "kmsProviders": { + "local": { + "key": "Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk" + } + }, + "keyVaultNamespace": "keyvault.datakeys", + "bypassQueryAnalysis": true + }, + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "encryptedDB", + "client": "client0", + "databaseName": "default" + } + }, + { + "collection": { + "id": "encryptedColl", + "database": "encryptedDB", + "collectionName": "default" + } + }, + { + "client": { + "id": "client1" + } + }, + { + "database": { + "id": "unencryptedDB", + "client": "client1", + "databaseName": "default" + } + }, + { + "collection": { + "id": "unencryptedColl", + "database": "unencryptedDB", + "collectionName": "default" + } + } + ], + "initialData": [ + { + "databaseName": "keyvault", + "collectionName": "datakeys", + "documents": [ + { + "_id": { + "$binary": { + "base64": "EjRWeBI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "keyMaterial": { + "$binary": { + "base64": "sHe0kz57YW7v8g9VP9sf/+K1ex4JqKc5rf/URX3n3p8XdZ6+15uXPaSayC6adWbNxkFskuMCOifDoTT+rkqMtFkDclOy884RuGGtUysq3X7zkAWYTKi8QAfKkajvVbZl2y23UqgVasdQu3OVBQCrH/xY00nNAs/52e958nVjBuzQkSb1T8pKJAyjZsHJ60+FtnfafDZSTAIBJYn7UWBCwQ==", + "subType": "00" + } + }, + "creationDate": { + "$date": { + "$numberLong": "1648914851981" + } + }, + "updateDate": { + "$date": { + "$numberLong": "1648914851981" + } + }, + "status": { + "$numberInt": "0" + }, + "masterKey": { + "provider": "local" + } + } + ] + }, + { + "databaseName": "default", + "collectionName": "default", + "documents": [], + "createOptions": { + "encryptedFields": { + "fields": [ + { + "keyId": { + "$binary": { + "base64": "EjRWeBI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "path": "encryptedIndexed", + "bsonType": "string", + "queries": { + "queryType": "equality", + "contention": { + "$numberLong": "0" + } + } + }, + { + "keyId": { + "$binary": { + "base64": "q83vqxI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "path": "encryptedUnindexed", + "bsonType": "string" + } + ] + } + } + } + ], + "tests": [ + { + "description": "BypassQueryAnalysis decrypts", + "operations": [ + { + "object": "encryptedColl", + "name": "insertOne", + "arguments": { + "document": { + "_id": 1, + "encryptedIndexed": { + "$binary": { + "base64": "C18BAAAFZAAgAAAAANnt+eLTkv4GdDPl8IAfJOvTzArOgFJQ2S/DcLza4W0DBXMAIAAAAAD2u+omZme3P2gBPehMQyQHQ153tPN1+z7bksYA9jKTpAVwADAAAAAAUnCOQqIvmR65YKyYnsiVfVrg9hwUVO3RhhKExo3RWOzgaS0QdsBL5xKFS0JhZSoWBXUAEAAAAAQSNFZ4EjSYdhI0EjRWeJASEHQAAgAAAAV2AFAAAAAAEjRWeBI0mHYSNBI0VniQEpQbp/ZJpWBKeDtKLiXb0P2E9wvc0g3f373jnYQYlJquOrlPOoEy3ngsHPJuSUijvWDsrQzqYa349K7G/66qaXEFZQAgAAAAAOuac/eRLYakKX6B0vZ1r3QodOQFfjqJD+xlGiPu4/PsBWwAIAAAAACkm0o9bj6j0HuADKc0svbqO2UHj6GrlNdF6yKNxh63xRJrAAAAAAAAAAAAAA==", + "subType": "06" + } + } + } + } + }, + { + "object": "encryptedColl", + "name": "find", + "arguments": { + "filter": { + "_id": 1 + } + }, + "expectResult": [ + { + "_id": 1, + "encryptedIndexed": "123" + } + ] + }, + { + "object": "unencryptedColl", + "name": "find", + "arguments": { + "filter": {} + }, + "expectResult": [ + { + "_id": 1, + "encryptedIndexed": { + "$$type": "binData" + }, + "__safeContent__": [ + { + "$binary": { + "base64": "31eCYlbQoVboc5zwC8IoyJVSkag9PxREka8dkmbXJeY=", + "subType": "00" + } + } + ] + } + ] + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "listCollections": 1, + "filter": { + "name": "default" + } + }, + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "command": { + "insert": "default", + "documents": [ + { + "_id": 1, + "encryptedIndexed": { + "$binary": { + "base64": "C18BAAAFZAAgAAAAANnt+eLTkv4GdDPl8IAfJOvTzArOgFJQ2S/DcLza4W0DBXMAIAAAAAD2u+omZme3P2gBPehMQyQHQ153tPN1+z7bksYA9jKTpAVwADAAAAAAUnCOQqIvmR65YKyYnsiVfVrg9hwUVO3RhhKExo3RWOzgaS0QdsBL5xKFS0JhZSoWBXUAEAAAAAQSNFZ4EjSYdhI0EjRWeJASEHQAAgAAAAV2AFAAAAAAEjRWeBI0mHYSNBI0VniQEpQbp/ZJpWBKeDtKLiXb0P2E9wvc0g3f373jnYQYlJquOrlPOoEy3ngsHPJuSUijvWDsrQzqYa349K7G/66qaXEFZQAgAAAAAOuac/eRLYakKX6B0vZ1r3QodOQFfjqJD+xlGiPu4/PsBWwAIAAAAACkm0o9bj6j0HuADKc0svbqO2UHj6GrlNdF6yKNxh63xRJrAAAAAAAAAAAAAA==", + "subType": "06" + } + } + } + ], + "ordered": true, + "encryptionInformation": { + "type": 1, + "schema": { + "default.default": { + "escCollection": "enxcol_.default.esc", + "ecocCollection": "enxcol_.default.ecoc", + "fields": [ + { + "keyId": { + "$binary": { + "base64": "EjRWeBI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "path": "encryptedIndexed", + "bsonType": "string", + "queries": { + "queryType": "equality", + "contention": { + "$numberLong": "0" + } + } + }, + { + "keyId": { + "$binary": { + "base64": "q83vqxI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "path": "encryptedUnindexed", + "bsonType": "string" + } + ] + } + } + } + }, + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "command": { + "find": "default", + "filter": { + "_id": 1 + } + }, + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "command": { + "find": "datakeys", + "filter": { + "$or": [ + { + "_id": { + "$in": [ + { + "$binary": { + "base64": "EjRWeBI0mHYSNBI0VniQEg==", + "subType": "04" + } + } + ] + } + }, + { + "keyAltNames": { + "$in": [] + } + } + ] + }, + "$db": "keyvault", + "readConcern": { + "level": "majority" + } + }, + "commandName": "find" + } + } + ] + } + ] + } + ] +} diff --git a/src/libmongoc/tests/json/client_side_encryption/unified/fle2v2-EncryptedFields-vs-EncryptedFieldsMap.json b/src/libmongoc/tests/json/client_side_encryption/unified/fle2v2-EncryptedFields-vs-EncryptedFieldsMap.json new file mode 100644 index 00000000000..bd1d0703ec1 --- /dev/null +++ b/src/libmongoc/tests/json/client_side_encryption/unified/fle2v2-EncryptedFields-vs-EncryptedFieldsMap.json @@ -0,0 +1,256 @@ +{ + "description": "fle2v2-EncryptedFields-vs-EncryptedFieldsMap", + "schemaVersion": "1.23", + "runOnRequirements": [ + { + "minServerVersion": "7.0.0", + "serverless": "forbid", + "csfle": true, + "topologies": [ + "replicaset", + "sharded", + "load-balanced" + ] + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "autoEncryptOpts": { + "keyVaultNamespace": "keyvault.datakeys", + "kmsProviders": { + "local": { + "key": "Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk" + } + }, + "encryptedFieldsMap": { + "default.default": { + "fields": [] + } + } + }, + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "encryptedDB", + "client": "client0", + "databaseName": "default" + } + }, + { + "collection": { + "id": "encryptedColl", + "database": "encryptedDB", + "collectionName": "default" + } + } + ], + "initialData": [ + { + "databaseName": "keyvault", + "collectionName": "datakeys", + "documents": [ + { + "_id": { + "$binary": { + "base64": "q83vqxI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "keyMaterial": { + "$binary": { + "base64": "HBk9BWihXExNDvTp1lUxOuxuZK2Pe2ZdVdlsxPEBkiO1bS4mG5NNDsQ7zVxJAH8BtdOYp72Ku4Y3nwc0BUpIKsvAKX4eYXtlhv5zUQxWdeNFhg9qK7qb8nqhnnLeT0f25jFSqzWJoT379hfwDeu0bebJHr35QrJ8myZdPMTEDYF08QYQ48ShRBli0S+QzBHHAQiM2iJNr4svg2WR8JSeWQ==", + "subType": "00" + } + }, + "creationDate": { + "$date": { + "$numberLong": "1648914851981" + } + }, + "updateDate": { + "$date": { + "$numberLong": "1648914851981" + } + }, + "status": { + "$numberInt": "0" + }, + "masterKey": { + "provider": "local" + } + } + ] + }, + { + "databaseName": "default", + "collectionName": "default", + "documents": [], + "createOptions": { + "encryptedFields": { + "fields": [ + { + "keyId": { + "$binary": { + "base64": "EjRWeBI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "path": "encryptedIndexed", + "bsonType": "string", + "queries": { + "queryType": "equality", + "contention": { + "$numberLong": "0" + } + } + }, + { + "keyId": { + "$binary": { + "base64": "q83vqxI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "path": "encryptedUnindexed", + "bsonType": "string" + } + ] + } + } + } + ], + "tests": [ + { + "description": "encryptedFieldsMap is preferred over remote encryptedFields", + "operations": [ + { + "object": "encryptedColl", + "name": "insertOne", + "arguments": { + "document": { + "_id": 1, + "encryptedUnindexed": { + "$binary": { + "base64": "BqvN76sSNJh2EjQSNFZ4kBICTQaVZPWgXp41I7mPV1rLFTtw1tXzjcdSEyxpKKqujlko5TeizkB9hHQ009dVY1+fgIiDcefh+eQrm3CkhQ==", + "subType": "06" + } + } + } + } + }, + { + "object": "encryptedColl", + "name": "find", + "arguments": { + "filter": { + "_id": 1 + } + }, + "expectResult": [ + { + "_id": 1, + "encryptedUnindexed": "value123" + } + ] + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "databaseName": "default", + "commandName": "insert", + "command": { + "insert": "default", + "documents": [ + { + "_id": 1, + "encryptedUnindexed": { + "$binary": { + "base64": "BqvN76sSNJh2EjQSNFZ4kBICTQaVZPWgXp41I7mPV1rLFTtw1tXzjcdSEyxpKKqujlko5TeizkB9hHQ009dVY1+fgIiDcefh+eQrm3CkhQ==", + "subType": "06" + } + } + } + ], + "ordered": true + } + } + }, + { + "commandStartedEvent": { + "databaseName": "default", + "commandName": "find", + "command": { + "find": "default", + "filter": { + "_id": 1 + } + } + } + }, + { + "commandStartedEvent": { + "databaseName": "keyvault", + "commandName": "find", + "command": { + "find": "datakeys", + "filter": { + "$or": [ + { + "_id": { + "$in": [ + { + "$binary": { + "base64": "q83vqxI0mHYSNBI0VniQEg==", + "subType": "04" + } + } + ] + } + }, + { + "keyAltNames": { + "$in": [] + } + } + ] + }, + "$db": "keyvault", + "readConcern": { + "level": "majority" + } + } + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "default", + "databaseName": "default", + "documents": [ + { + "_id": 1, + "encryptedUnindexed": { + "$binary": { + "base64": "BqvN76sSNJh2EjQSNFZ4kBICTQaVZPWgXp41I7mPV1rLFTtw1tXzjcdSEyxpKKqujlko5TeizkB9hHQ009dVY1+fgIiDcefh+eQrm3CkhQ==", + "subType": "06" + } + } + } + ] + } + ] + } + ] +} diff --git a/src/libmongoc/tests/json/client_side_encryption/unified/localSchema.json b/src/libmongoc/tests/json/client_side_encryption/unified/localSchema.json new file mode 100644 index 00000000000..fef4b4e1229 --- /dev/null +++ b/src/libmongoc/tests/json/client_side_encryption/unified/localSchema.json @@ -0,0 +1,336 @@ +{ + "description": "localSchema", + "schemaVersion": "1.23", + "runOnRequirements": [ + { + "minServerVersion": "4.1.10", + "csfle": true + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "autoEncryptOpts": { + "keyVaultNamespace": "keyvault.datakeys", + "schemaMap": { + "default.default": { + "properties": { + "encrypted_w_altname": { + "encrypt": { + "keyId": "/altname", + "bsonType": "string", + "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Random" + } + }, + "encrypted_string": { + "encrypt": { + "keyId": [ + { + "$binary": { + "base64": "AAAAAAAAAAAAAAAAAAAAAA==", + "subType": "04" + } + } + ], + "bsonType": "string", + "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic" + } + }, + "random": { + "encrypt": { + "keyId": [ + { + "$binary": { + "base64": "AAAAAAAAAAAAAAAAAAAAAA==", + "subType": "04" + } + } + ], + "bsonType": "string", + "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Random" + } + }, + "encrypted_string_equivalent": { + "encrypt": { + "keyId": [ + { + "$binary": { + "base64": "AAAAAAAAAAAAAAAAAAAAAA==", + "subType": "04" + } + } + ], + "bsonType": "string", + "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic" + } + } + }, + "bsonType": "object" + } + }, + "kmsProviders": { + "aws": { + "accessKeyId": { + "$$placeholder": 1 + }, + "secretAccessKey": { + "$$placeholder": 1 + } + } + } + }, + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "client": { + "id": "client1", + "autoEncryptOpts": { + "keyVaultNamespace": "keyvault.datakeys", + "schemaMap": { + "default.default": { + "properties": { + "test": { + "bsonType": "string" + } + }, + "bsonType": "object", + "required": [ + "test" + ] + } + }, + "kmsProviders": { + "aws": { + "accessKeyId": { + "$$placeholder": 1 + }, + "secretAccessKey": { + "$$placeholder": 1 + } + } + } + }, + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "encryptedDB", + "client": "client0", + "databaseName": "default" + } + }, + { + "collection": { + "id": "encryptedColl", + "database": "encryptedDB", + "collectionName": "default" + } + }, + { + "database": { + "id": "encryptedDB2", + "client": "client1", + "databaseName": "default" + } + }, + { + "collection": { + "id": "encryptedColl2", + "database": "encryptedDB2", + "collectionName": "default" + } + } + ], + "initialData": [ + { + "databaseName": "keyvault", + "collectionName": "datakeys", + "documents": [ + { + "status": 1, + "_id": { + "$binary": { + "base64": "AAAAAAAAAAAAAAAAAAAAAA==", + "subType": "04" + } + }, + "masterKey": { + "provider": "aws", + "key": "arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-9f25-e30687b580d0", + "region": "us-east-1" + }, + "updateDate": { + "$date": { + "$numberLong": "1552949630483" + } + }, + "keyMaterial": { + "$binary": { + "base64": "AQICAHhQNmWG2CzOm1dq3kWLM+iDUZhEqnhJwH9wZVpuZ94A8gEqnsxXlR51T5EbEVezUqqKAAAAwjCBvwYJKoZIhvcNAQcGoIGxMIGuAgEAMIGoBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDHa4jo6yp0Z18KgbUgIBEIB74sKxWtV8/YHje5lv5THTl0HIbhSwM6EqRlmBiFFatmEWaeMk4tO4xBX65eq670I5TWPSLMzpp8ncGHMmvHqRajNBnmFtbYxN3E3/WjxmdbOOe+OXpnGJPcGsftc7cB2shRfA4lICPnE26+oVNXT6p0Lo20nY5XC7jyCO", + "subType": "00" + } + }, + "creationDate": { + "$date": { + "$numberLong": "1552949630483" + } + }, + "keyAltNames": [ + "altname", + "another_altname" + ] + } + ] + }, + { + "databaseName": "default", + "collectionName": "default", + "documents": [] + } + ], + "tests": [ + { + "description": "A local schema should override", + "operations": [ + { + "object": "encryptedColl", + "name": "insertOne", + "arguments": { + "document": { + "_id": 1, + "encrypted_string": "string0" + } + } + }, + { + "object": "encryptedColl", + "name": "find", + "arguments": { + "filter": { + "_id": 1 + } + }, + "expectResult": [ + { + "_id": 1, + "encrypted_string": "string0" + } + ] + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "databaseName": "keyvault", + "commandName": "find", + "command": { + "find": "datakeys", + "filter": { + "$or": [ + { + "_id": { + "$in": [ + { + "$binary": { + "base64": "AAAAAAAAAAAAAAAAAAAAAA==", + "subType": "04" + } + } + ] + } + }, + { + "keyAltNames": { + "$in": [] + } + } + ] + }, + "readConcern": { + "level": "majority" + } + } + } + }, + { + "commandStartedEvent": { + "commandName": "insert", + "command": { + "insert": "default", + "documents": [ + { + "_id": 1, + "encrypted_string": { + "$binary": { + "base64": "AQAAAAAAAAAAAAAAAAAAAAACwj+3zkv2VM+aTfk60RqhXq6a/77WlLwu/BxXFkL7EppGsju/m8f0x5kBDD3EZTtGALGXlym5jnpZAoSIkswHoA==", + "subType": "06" + } + } + } + ], + "ordered": true + } + } + }, + { + "commandStartedEvent": { + "commandName": "find", + "command": { + "find": "default", + "filter": { + "_id": 1 + } + } + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "default", + "databaseName": "default", + "documents": [ + { + "_id": 1, + "encrypted_string": { + "$binary": { + "base64": "AQAAAAAAAAAAAAAAAAAAAAACwj+3zkv2VM+aTfk60RqhXq6a/77WlLwu/BxXFkL7EppGsju/m8f0x5kBDD3EZTtGALGXlym5jnpZAoSIkswHoA==", + "subType": "06" + } + } + } + ] + } + ] + }, + { + "description": "A local schema with no encryption is an error", + "operations": [ + { + "object": "encryptedColl2", + "name": "insertOne", + "arguments": { + "document": { + "_id": 1, + "encrypted_string": "string0" + } + }, + "expectError": { + "isClientError": true + } + } + ] + } + ] +} diff --git a/src/libmongoc/tests/json/client_side_encryption/unified/maxWireVersion.json b/src/libmongoc/tests/json/client_side_encryption/unified/maxWireVersion.json new file mode 100644 index 00000000000..07b565da18e --- /dev/null +++ b/src/libmongoc/tests/json/client_side_encryption/unified/maxWireVersion.json @@ -0,0 +1,101 @@ +{ + "description": "maxWireVersion", + "schemaVersion": "1.23", + "runOnRequirements": [ + { + "maxServerVersion": "4.0.99", + "csfle": true + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "autoEncryptOpts": { + "keyVaultNamespace": "keyvault.datakeys", + "kmsProviders": { + "aws": {} + }, + "extraOptions": { + "mongocryptdBypassSpawn": true + } + } + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "default" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "default" + } + } + ], + "initialData": [ + { + "databaseName": "keyvault", + "collectionName": "datakeys", + "documents": [ + { + "status": 1, + "_id": { + "$binary": { + "base64": "AAAAAAAAAAAAAAAAAAAAAA==", + "subType": "04" + } + }, + "masterKey": { + "provider": "aws", + "key": "arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-9f25-e30687b580d0", + "region": "us-east-1" + }, + "updateDate": { + "$date": { + "$numberLong": "1552949630483" + } + }, + "keyMaterial": { + "$binary": { + "base64": "AQICAHhQNmWG2CzOm1dq3kWLM+iDUZhEqnhJwH9wZVpuZ94A8gEqnsxXlR51T5EbEVezUqqKAAAAwjCBvwYJKoZIhvcNAQcGoIGxMIGuAgEAMIGoBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDHa4jo6yp0Z18KgbUgIBEIB74sKxWtV8/YHje5lv5THTl0HIbhSwM6EqRlmBiFFatmEWaeMk4tO4xBX65eq670I5TWPSLMzpp8ncGHMmvHqRajNBnmFtbYxN3E3/WjxmdbOOe+OXpnGJPcGsftc7cB2shRfA4lICPnE26+oVNXT6p0Lo20nY5XC7jyCO", + "subType": "00" + } + }, + "creationDate": { + "$date": { + "$numberLong": "1552949630483" + } + }, + "keyAltNames": [ + "altname", + "another_altname" + ] + } + ] + } + ], + "tests": [ + { + "description": "operation fails with maxWireVersion < 8", + "operations": [ + { + "name": "insertOne", + "object": "collection0", + "arguments": { + "document": { + "encrypted_string": "string0" + } + }, + "expectError": { + "errorContains": "Auto-encryption requires a minimum MongoDB version of 4.2" + } + } + ] + } + ] +} diff --git a/src/libmongoc/tests/json/crud/unified/client-bulkWrite-qe.json b/src/libmongoc/tests/json/crud/unified/client-bulkWrite-qe.json new file mode 100644 index 00000000000..4d73d01351f --- /dev/null +++ b/src/libmongoc/tests/json/crud/unified/client-bulkWrite-qe.json @@ -0,0 +1,272 @@ +{ + "description": "client bulkWrite with queryable encryption", + "schemaVersion": "1.23", + "runOnRequirements": [ + { + "minServerVersion": "8.0", + "serverless": "forbid" + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent", + "commandSucceededEvent" + ], + "autoEncryptOpts": { + "kmsProviders": { + "local": { + "key": "Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk" + } + } + } + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "crud-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "coll0" + } + }, + { + "database": { + "id": "database1", + "client": "client0", + "databaseName": "keyvault" + } + }, + { + "collection": { + "id": "collection1", + "database": "database0", + "collectionName": "datakeys" + } + } + ], + "initialData": [ + { + "databaseName": "keyvault", + "collectionName": "datakeys", + "documents": [ + { + "_id": { + "$binary": { + "base64": "EjRWeBI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "keyAltNames": [ + "local_key" + ], + "keyMaterial": { + "$binary": { + "base64": "sHe0kz57YW7v8g9VP9sf/+K1ex4JqKc5rf/URX3n3p8XdZ6+15uXPaSayC6adWbNxkFskuMCOifDoTT+rkqMtFkDclOy884RuGGtUysq3X7zkAWYTKi8QAfKkajvVbZl2y23UqgVasdQu3OVBQCrH/xY00nNAs/52e958nVjBuzQkSb1T8pKJAyjZsHJ60+FtnfafDZSTAIBJYn7UWBCwQ==", + "subType": "00" + } + }, + "creationDate": { + "$date": { + "$numberLong": "1641024000000" + } + }, + "updateDate": { + "$date": { + "$numberLong": "1641024000000" + } + }, + "status": 1, + "masterKey": { + "provider": "local" + } + } + ] + }, + { + "databaseName": "crud-tests", + "collectionName": "coll0", + "documents": [], + "createOptions": { + "encryptedFields": { + "fields": [ + { + "keyId": { + "$binary": { + "base64": "EjRWeBI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "path": "encryptedInt", + "bsonType": "int", + "queries": { + "queryType": "equality", + "contention": { + "$numberLong": "0" + } + } + } + ] + } + } + } + ], + "_yamlAnchors": { + "namespace": "crud-tests.coll0" + }, + "tests": [ + { + "description": "client bulkWrite QE replaceOne", + "operations": [ + { + "object": "collection0", + "name": "insertMany", + "arguments": { + "documents": [ + { + "_id": 1, + "encryptedInt": 11 + }, + { + "_id": 2, + "encryptedInt": 22 + }, + { + "_id": 3, + "encryptedInt": 33 + } + ] + } + }, + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "replaceOne": { + "namespace": "crud-tests.coll0", + "filter": { + "encryptedInt": { + "$eq": 11 + } + }, + "replacement": { + "encryptedInt": 44 + } + } + } + ] + }, + "expectResult": { + "insertedCount": 0, + "upsertedCount": 0, + "matchedCount": 1, + "modifiedCount": 1, + "deletedCount": 0 + } + }, + { + "object": "collection0", + "name": "find", + "arguments": { + "filter": { + "encryptedInt": 44 + } + }, + "expectResult": [ + { + "_id": 1, + "encryptedInt": 44 + } + ] + } + ], + "outcome": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "encryptedInt": { + "$$type": "binData" + }, + "__safeContent__": { + "$$type": "array" + } + }, + { + "_id": 2, + "encryptedInt": { + "$$type": "binData" + }, + "__safeContent__": { + "$$type": "array" + } + }, + { + "_id": 3, + "encryptedInt": { + "$$type": "binData" + }, + "__safeContent__": { + "$$type": "array" + } + } + ] + } + ] + }, + { + "description": "client bulkWrite QE with multiple replace fails", + "operations": [ + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "replaceOne": { + "namespace": "crud-tests.coll0", + "filter": { + "encryptedInt": { + "$eq": 11 + } + }, + "replacement": { + "encryptedInt": 44 + } + } + }, + { + "replaceOne": { + "namespace": "crud-tests.coll0", + "filter": { + "encryptedInt": { + "$eq": 22 + } + }, + "replacement": { + "encryptedInt": 44 + } + } + } + ] + }, + "expectError": { + "isError": true + } + } + ] + } + ] +} diff --git a/src/libmongoc/tests/json/unified/poc-queryable-encryption.json b/src/libmongoc/tests/json/unified/poc-queryable-encryption.json new file mode 100644 index 00000000000..4fa5a350e03 --- /dev/null +++ b/src/libmongoc/tests/json/unified/poc-queryable-encryption.json @@ -0,0 +1,179 @@ +{ + "description": "poc-queryable-encryption", + "schemaVersion": "1.23", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent", + "commandSucceededEvent" + ], + "autoEncryptOpts": { + "kmsProviders": { + "local": { + "key": "Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk" + } + } + } + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "poc-queryable-encryption" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "coll0" + } + }, + { + "database": { + "id": "database1", + "client": "client0", + "databaseName": "keyvault" + } + }, + { + "collection": { + "id": "collection1", + "database": "database0", + "collectionName": "datakeys" + } + } + ], + "initialData": [ + { + "databaseName": "keyvault", + "collectionName": "datakeys", + "documents": [ + { + "_id": { + "$binary": { + "base64": "EjRWeBI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "keyMaterial": { + "$binary": { + "base64": "sHe0kz57YW7v8g9VP9sf/+K1ex4JqKc5rf/URX3n3p8XdZ6+15uXPaSayC6adWbNxkFskuMCOifDoTT+rkqMtFkDclOy884RuGGtUysq3X7zkAWYTKi8QAfKkajvVbZl2y23UqgVasdQu3OVBQCrH/xY00nNAs/52e958nVjBuzQkSb1T8pKJAyjZsHJ60+FtnfafDZSTAIBJYn7UWBCwQ==", + "subType": "00" + } + }, + "creationDate": { + "$date": { + "$numberLong": "1641024000000" + } + }, + "updateDate": { + "$date": { + "$numberLong": "1641024000000" + } + }, + "status": 1, + "masterKey": { + "provider": "local" + } + } + ] + }, + { + "databaseName": "poc-queryable-encryption", + "collectionName": "coll0", + "documents": [], + "createOptions": { + "encryptedFields": { + "fields": [ + { + "keyId": { + "$binary": { + "base64": "EjRWeBI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "path": "encryptedInt", + "bsonType": "int", + "queries": { + "queryType": "equality", + "contention": { + "$numberLong": "0" + } + } + } + ] + } + } + } + ], + "tests": [ + { + "description": "insert, replace, and find with queryable encryption", + "operations": [ + { + "object": "collection0", + "name": "insertOne", + "arguments": { + "document": { + "_id": 1, + "encryptedInt": 11 + } + } + }, + { + "object": "collection0", + "name": "replaceOne", + "arguments": { + "filter": { + "encryptedInt": 11 + }, + "replacement": { + "encryptedInt": 22 + } + } + }, + { + "object": "collection0", + "name": "find", + "arguments": { + "filter": { + "encryptedInt": 22 + } + }, + "expectResult": [ + { + "_id": 1, + "encryptedInt": 22 + } + ] + } + ], + "outcome": [ + { + "collectionName": "coll0", + "databaseName": "poc-queryable-encryption", + "documents": [ + { + "_id": 1, + "encryptedInt": { + "$$type": "binData" + }, + "__safeContent__": { + "$$type": "array" + } + } + ] + } + ] + } + ] +} diff --git a/src/libmongoc/tests/json/unified/poc-transactions.json b/src/libmongoc/tests/json/unified/poc-transactions.json index 0355ca20605..2055a3b7057 100644 --- a/src/libmongoc/tests/json/unified/poc-transactions.json +++ b/src/libmongoc/tests/json/unified/poc-transactions.json @@ -11,7 +11,7 @@ { "minServerVersion": "4.1.8", "topologies": [ - "sharded-replicaset" + "sharded" ] } ], @@ -93,7 +93,7 @@ "minServerVersion": "4.3.4", "topologies": [ "replicaset", - "sharded-replicaset" + "sharded" ] } ], @@ -203,7 +203,7 @@ "minServerVersion": "4.3.4", "topologies": [ "replicaset", - "sharded-replicaset" + "sharded" ] } ], diff --git a/src/libmongoc/tests/unified/entity-map.c b/src/libmongoc/tests/unified/entity-map.c index e12d25c384d..fb3bd940a16 100644 --- a/src/libmongoc/tests/unified/entity-map.c +++ b/src/libmongoc/tests/unified/entity-map.c @@ -16,6 +16,7 @@ #include "entity-map.h" +#include "bson/bson.h" #include "bsonutil/bson-parser.h" #include "TestSuite.h" #include @@ -34,6 +35,7 @@ #include #include #include +#include "json-test.h" #define REDUCED_HEARTBEAT_FREQUENCY_MS 500 #define REDUCED_MIN_HEARTBEAT_FREQUENCY_MS 50 @@ -48,6 +50,9 @@ typedef void (*event_serialize_func_t) (bson_t *bson, const void *event); static void entity_destroy (entity_t *entity); +static bool +_parse_and_set_auto_encryption_opts (mongoc_client_t *client, bson_t *opts, bson_error_t *error); + entity_map_t * entity_map_new (void) { @@ -724,6 +729,7 @@ entity_client_new (entity_map_t *em, bson_t *bson, bson_error_t *error) bool ret = false; mongoc_apm_callbacks_t *callbacks = NULL; bson_t *uri_options = NULL; + bson_t *auto_encryption_opts = NULL; mongoc_structured_log_opts_t *log_opts = mongoc_structured_log_opts_new (); bool use_multiple_mongoses = false; bool use_multiple_mongoses_set = false; @@ -803,32 +809,37 @@ entity_client_new (entity_map_t *em, bson_t *bson, bson_error_t *error) *p = bsonAs (boolean); })), // Which events should be available as entities: - find ( - key ("storeEventsAsEntities"), - if (not(type (array)), then (error ("'storeEventsAsEntities' must be an array"))), - visitEach (parse ( - find (keyWithType ("id", utf8), storeStrRef (store_entity_id), do ({ - if (!entity_map_add_bson_array (em, store_entity_id, error)) { - test_error ("failed to create storeEventsAsEntities " - "entity '%s': %s", - store_entity_id, - error->message); - } - })), - find (keyWithType ("events", array), - visitEach (case (when (not(type (utf8)), - error ("Every 'storeEventsAsEntities.events' " - "element must be a string")), - when (eval (is_supported_event_type (bson_iter_utf8 (&bsonVisitIter, NULL))), do ({ - const char *const type = bson_iter_utf8 (&bsonVisitIter, NULL); - set_event_callback (callbacks, type); - add_store_event (entity, type, store_entity_id); - })), - when (eval (is_unsupported_event_type (bson_iter_utf8 (&bsonVisitIter, NULL))), - do (MONGOC_DEBUG ("Skipping unsupported event type '%s'", bsonAs (cstr)))), - else (do (test_error ("Unknown event type '%s'", bsonAs (cstr))))))), - visitOthers ( - errorf (err, "Unexpected field '%s' in storeEventsAsEntities", bson_iter_key (&bsonVisitIter)))))), + find (key ("storeEventsAsEntities"), + if (not(type (array)), then (error ("'storeEventsAsEntities' must be an array"))), + visitEach (parse ( + find (keyWithType ("id", utf8), storeStrRef (store_entity_id), do ({ + if (!entity_map_add_bson_array (em, store_entity_id, error)) { + test_error ("failed to create storeEventsAsEntities " + "entity '%s': %s", + store_entity_id, + error->message); + } + })), + find (keyWithType ("events", array), + visitEach (case (when (not(type (utf8)), + error ("Every 'storeEventsAsEntities.events' " + "element must be a string")), + when (anyOf (iStrEqual ("commandStartedEvent"), + iStrEqual ("commandFailedEvent"), + iStrEqual ("commandSucceededEvent")), + do ({ + const char *const type = bson_iter_utf8 (&bsonVisitIter, NULL); + set_event_callback (callbacks, type); + add_store_event (entity, type, store_entity_id); + })), + when (eval (is_unsupported_event_type (bson_iter_utf8 (&bsonVisitIter, NULL))), + do (MONGOC_DEBUG ("Skipping unsupported event type '%s'", bsonAs (cstr)))), + else (do (test_error ("Unknown event type '%s'", bsonAs (cstr))))))), + visitOthers ( + errorf (err, "Unexpected field '%s' in storeEventsAsEntities", bson_iter_key (&bsonVisitIter)))))), + find (key ("autoEncryptOpts"), + if (not(type (doc)), then (error ("'autoEncryptOpts' must be a document value"))), + storeDocDupPtr (auto_encryption_opts)), // Log messages to observe: find (key ("observeLogMessages"), if (not(type (doc)), then (error ("'observeLogMessages' must be a document"))), @@ -914,6 +925,10 @@ entity_client_new (entity_map_t *em, bson_t *bson, bson_error_t *error) client->topology->min_heartbeat_frequency_msec = REDUCED_MIN_HEARTBEAT_FREQUENCY_MS; } + if (auto_encryption_opts) { + _parse_and_set_auto_encryption_opts (client, auto_encryption_opts, error); + } + ret = true; done: mongoc_uri_destroy (uri); @@ -921,6 +936,7 @@ entity_client_new (entity_map_t *em, bson_t *bson, bson_error_t *error) mongoc_server_api_destroy (api); mongoc_structured_log_opts_destroy (log_opts); bson_destroy (uri_options); + bson_destroy (auto_encryption_opts); if (!ret) { entity_destroy (entity); return NULL; @@ -1232,7 +1248,7 @@ _parse_kms_provider_local ( } static bool -_parse_and_set_kms_providers (mongoc_client_encryption_opts_t *ce_opts, bson_t *kms_from_file, bson_error_t *error) +_get_kms_providers_docs (bson_t *kms_from_file, bson_t *kms_providers, bson_t *tls_opts, bson_error_t *error) { /* Map provider to corresponding KMS parser. */ typedef struct _prov_map_t { @@ -1255,10 +1271,6 @@ _parse_and_set_kms_providers (mongoc_client_encryption_opts_t *ce_opts, bson_t * {.provider = "local:name2", .parse = _parse_kms_provider_local}}; const size_t prov_map_size = sizeof (prov_map) / sizeof (prov_map[0]); - - bool ret = false; - bson_t kms_providers = BSON_INITIALIZER; - bson_t tls_opts = BSON_INITIALIZER; bson_iter_t iter; BSON_FOREACH (kms_from_file, iter) @@ -1270,12 +1282,12 @@ _parse_and_set_kms_providers (mongoc_client_encryption_opts_t *ce_opts, bson_t * if (!bson_init_from_value (&kms_doc, bson_iter_value (&iter))) { test_set_error (error, "kmsProviders field '%s' is not a valid document", provider); - goto done; + return false; } for (i = 0u; i < prov_map_size; ++i) { if (strcmp (provider, prov_map[i].provider) == 0) { - found = prov_map[i].parse (&kms_providers, &tls_opts, provider, &kms_doc, error); + found = prov_map[i].parse (kms_providers, tls_opts, provider, &kms_doc, error); goto parsed; } } @@ -1286,13 +1298,117 @@ _parse_and_set_kms_providers (mongoc_client_encryption_opts_t *ce_opts, bson_t * bson_destroy (&kms_doc); if (!found) { + return false; + } + } + return true; +} + +static bool +_parse_and_set_auto_encryption_opts (mongoc_client_t *client, bson_t *opts, bson_error_t *error) +{ + bool ret = false; + mongoc_auto_encryption_opts_t *auto_encryption_opts = mongoc_auto_encryption_opts_new (); + bson_t kms_providers = BSON_INITIALIZER; + bson_t tls_opts = BSON_INITIALIZER; + BSON_ASSERT (client); + + bson_parser_t *const parser = bson_parser_new (); + + bson_t *kms_providers_raw; + bson_parser_doc (parser, "kmsProviders", &kms_providers_raw); + + char *keyvault_ns; + bson_parser_utf8 (parser, "keyVaultNamespace", &keyvault_ns); + + bson_t *schema_map; + bson_parser_doc_optional (parser, "schemaMap", &schema_map); + + bool *bypass_auto_encryption; + bson_parser_bool_optional (parser, "bypassAutoEncryption", &bypass_auto_encryption); + + bool *bypass_query_analysis; + bson_parser_bool_optional (parser, "bypassQueryAnalysis", &bypass_query_analysis); + + bson_t *encrypted_fields_map; + bson_parser_doc_optional (parser, "encryptedFieldsMap", &encrypted_fields_map); + + int64_t *key_expiration_ms; + bson_parser_int_optional (parser, "keyExpirationMS", &key_expiration_ms); + + bson_t *extra_options; + bson_parser_doc_optional (parser, "extraOptions", &extra_options); + + if (!bson_parser_parse (parser, opts, error)) { + goto done; + } + + { + if (!_get_kms_providers_docs (kms_providers_raw, &kms_providers, &tls_opts, error)) { goto done; } + mongoc_auto_encryption_opts_set_kms_providers (auto_encryption_opts, &kms_providers); + mongoc_auto_encryption_opts_set_tls_opts (auto_encryption_opts, &tls_opts); + } + + { + // keyVaultNamespace + char *dot = strstr (keyvault_ns, "."); + BSON_ASSERT (dot); + char *db_name = bson_strndup (keyvault_ns, dot - keyvault_ns); + char *coll_name = bson_strdup (dot + 1); + mongoc_auto_encryption_opts_set_keyvault_namespace (auto_encryption_opts, db_name, coll_name); + + bson_free (db_name); + bson_free (coll_name); + } + + if (schema_map) { + mongoc_auto_encryption_opts_set_schema_map (auto_encryption_opts, schema_map); + } + + if (bypass_auto_encryption) { + mongoc_auto_encryption_opts_set_bypass_auto_encryption (auto_encryption_opts, *bypass_auto_encryption); + } + + if (bypass_query_analysis) { + mongoc_auto_encryption_opts_set_bypass_query_analysis (auto_encryption_opts, *bypass_query_analysis); + } + + if (encrypted_fields_map) { + mongoc_auto_encryption_opts_set_encrypted_fields_map (auto_encryption_opts, encrypted_fields_map); + } + + if (key_expiration_ms) { + mongoc_auto_encryption_opts_set_key_expiration (auto_encryption_opts, *key_expiration_ms); + } + + if (extra_options) { + mongoc_auto_encryption_opts_set_extra (auto_encryption_opts, extra_options); } + mongoc_client_enable_auto_encryption (client, auto_encryption_opts, error); + ret = true; + +done: + mongoc_auto_encryption_opts_destroy (auto_encryption_opts); + bson_destroy (&kms_providers); + bson_destroy (&tls_opts); + bson_parser_destroy_with_parsed_fields (parser); + return ret; +} + +static bool +_parse_and_set_kms_providers (mongoc_client_encryption_opts_t *ce_opts, bson_t *kms_from_file, bson_error_t *error) +{ + bool ret = false; + bson_t kms_providers = BSON_INITIALIZER; + bson_t tls_opts = BSON_INITIALIZER; + if (!_get_kms_providers_docs (kms_from_file, &kms_providers, &tls_opts, error)) { + goto done; + } mongoc_client_encryption_opts_set_kms_providers (ce_opts, &kms_providers); mongoc_client_encryption_opts_set_tls_opts (ce_opts, &tls_opts); - ret = true; done: @@ -1413,6 +1529,7 @@ typedef struct { mongoc_read_concern_t *rc; mongoc_write_concern_t *wc; mongoc_read_prefs_t *rp; + bson_t *encrypted_fields; } coll_or_db_opts_t; static coll_or_db_opts_t * @@ -1430,6 +1547,7 @@ coll_or_db_opts_destroy (coll_or_db_opts_t *opts) mongoc_read_concern_destroy (opts->rc); mongoc_read_prefs_destroy (opts->rp); mongoc_write_concern_destroy (opts->wc); + bson_destroy (opts->encrypted_fields); bson_free (opts); } diff --git a/src/libmongoc/tests/unified/operation.c b/src/libmongoc/tests/unified/operation.c index fda829dc18f..4e42ccf5965 100644 --- a/src/libmongoc/tests/unified/operation.c +++ b/src/libmongoc/tests/unified/operation.c @@ -1141,6 +1141,9 @@ operation_drop_collection (test_t *test, operation_t *op, result_t *result, bson goto done; } + /* Forward all arguments other than collection name as-is. */ + BSON_ASSERT (bson_concat (opts, bson_parser_get_extra (parser))); + coll = mongoc_database_get_collection (db, collection); mongoc_collection_drop_with_opts (coll, opts, &op_error); @@ -1850,7 +1853,7 @@ operation_distinct (test_t *test, operation_t *op, result_t *result, bson_error_ } distinct = BCON_NEW ( - "distinct", mongoc_collection_get_name (coll), "key", BCON_UTF8 (field_name), "query", "{", &filter, "}"); + "distinct", mongoc_collection_get_name (coll), "key", BCON_UTF8 (field_name), "query", BCON_DOCUMENT(filter)); bson_destroy (&op_reply); mongoc_collection_read_command_with_opts (coll, distinct, NULL /* read prefs */, opts, &op_reply, &op_error); diff --git a/src/libmongoc/tests/unified/result.c b/src/libmongoc/tests/unified/result.c index 6752b1f02dc..cd53e9999d3 100644 --- a/src/libmongoc/tests/unified/result.c +++ b/src/libmongoc/tests/unified/result.c @@ -413,6 +413,8 @@ result_check (result_t *result, entity_map_t *em, bson_val_t *expect_result, bso } if (error_contains) { + mongoc_lowercase_inplace(result->error.message); + mongoc_lowercase_inplace(error_contains); if (strstr (result->error.message, error_contains) == NULL) { test_set_error ( error, "expected error to contain \"%s\", but got: \"%s\"", error_contains, result->error.message); diff --git a/src/libmongoc/tests/unified/runner.c b/src/libmongoc/tests/unified/runner.c index e2d302ee423..96694b684ca 100644 --- a/src/libmongoc/tests/unified/runner.c +++ b/src/libmongoc/tests/unified/runner.c @@ -14,7 +14,10 @@ * limitations under the License. */ +#include "bson/bson.h" +#include "bsonutil/bson-match.h" #include "bsonutil/bson-parser.h" +#include "bsonutil/bson-val.h" #include "entity-map.h" #include "json-test.h" #include "operation.h" @@ -23,6 +26,7 @@ #include "test-libmongoc.h" #include "test-diagnostics.h" #include +#include #include "util.h" #include #include @@ -100,7 +104,14 @@ skipped_unified_test_t SKIPPED_TESTS[] = { // libmongoc does not support the optional findOne helper. {"retryable reads handshake failures", "collection.findOne succeeds after retryable handshake network error"}, {"retryable reads handshake failures", "collection.findOne succeeds after retryable handshake server error (ShutdownInProgress)"}, - + {"types", SKIP_ALL_TESTS}, + // libmongoc does not support the optional listIndexNames helper. + {"retryable reads handshake failures", "collection.listIndexNames succeeds after retryable handshake network error"}, + {"retryable reads handshake failures", "collection.listIndexNames succeeds after retryable handshake server error (ShutdownInProgress)"}, + // libmongoc does not support mapReduce. + {"unsupportedCommand", SKIP_ALL_TESTS}, + // libmongoc does not support the timeoutMS URI option + {"timeoutMS", SKIP_ALL_TESTS}, // libmongoc does not support the optional listIndexNames helper. {"retryable reads handshake failures", "collection.listIndexNames succeeds after retryable handshake network error"}, {"retryable reads handshake failures", "collection.listIndexNames succeeds after retryable handshake server error (ShutdownInProgress)"}, @@ -110,7 +121,6 @@ skipped_unified_test_t SKIPPED_TESTS[] = { // libmongoc does not include insertId in InsertOneResult {"cancel-server-check", SKIP_ALL_TESTS}, - {0}, }; // clang-format on @@ -636,7 +646,7 @@ check_schema_version (test_file_t *test_file) // 1.21 is partially supported (expectedError.writeErrors and expectedError.writeConcernErrors) // 1.22 is partially supported (keyExpirationMS in client encryption options) semver_t schema_version; - semver_parse ("1.22", &schema_version); + semver_parse ("1.23", &schema_version); if (schema_version.major != test_file->schema_version.major) { goto fail; @@ -883,7 +893,7 @@ test_setup_initial_data (test_t *test, bson_error_t *error) mongoc_write_concern_t *wc = NULL; bson_t *bulk_opts = NULL; bson_t *drop_opts = bson_new (); - bson_t *create_opts = bson_new (); + bson_t *create_opts = NULL; bool ret = false; bson_iter_bson (&initial_data_iter, &collection_data); @@ -891,10 +901,15 @@ test_setup_initial_data (test_t *test, bson_error_t *error) bson_parser_utf8 (parser, "databaseName", &database_name); bson_parser_utf8 (parser, "collectionName", &collection_name); bson_parser_array (parser, "documents", &documents); + bson_parser_doc_optional (parser, "createOptions", &create_opts); if (!bson_parser_parse (parser, &collection_data, error)) { goto loopexit; } + if (create_opts == NULL) { + create_opts = bson_new (); + } + wc = mongoc_write_concern_new (); mongoc_write_concern_set_w (wc, MONGOC_WRITE_CONCERN_W_MAJORITY); bulk_opts = bson_new (); @@ -915,6 +930,7 @@ test_setup_initial_data (test_t *test, bson_error_t *error) } coll = mongoc_client_get_collection (test_runner->internal_client, database_name, collection_name); + if (!mongoc_collection_drop_with_opts (coll, drop_opts, error)) { if (error->code != 26 && (NULL == strstr (error->message, "ns not found"))) { /* This is not a "ns not found" error. Fail the test. */ @@ -924,6 +940,47 @@ test_setup_initial_data (test_t *test, bson_error_t *error) memset (error, 0, sizeof (bson_error_t)); } + // Also drop `enxcol_..esc` and `enxcol_..ecoc` in case the collection will be used for QE. + { + char *collection_name_esc = bson_strdup_printf ("enxcol_.%s.esc", collection_name); + mongoc_collection_t *coll_esc = + mongoc_client_get_collection (test_runner->internal_client, database_name, collection_name_esc); + if (!mongoc_collection_drop_with_opts (coll_esc, drop_opts, error)) { + if (error->code != 26 && (NULL == strstr (error->message, "ns not found"))) { + /* This is not a "ns not found" error. Fail the test. */ + goto loopexit; + } + /* Clear an "ns not found" error. */ + memset (error, 0, sizeof (bson_error_t)); + } + mongoc_collection_destroy (coll_esc); + bson_free (collection_name_esc); + } + + { + char *collection_name_ecoc = bson_strdup_printf ("enxcol_.%s.ecoc", collection_name); + mongoc_collection_t *coll_ecoc = + mongoc_client_get_collection (test_runner->internal_client, database_name, collection_name_ecoc); + if (!mongoc_collection_drop_with_opts (coll_ecoc, drop_opts, error)) { + if (error->code != 26 && (NULL == strstr (error->message, "ns not found"))) { + /* This is not a "ns not found" error. Fail the test. */ + goto loopexit; + } + /* Clear an "ns not found" error. */ + memset (error, 0, sizeof (bson_error_t)); + } + mongoc_collection_destroy (coll_ecoc); + bson_free (collection_name_ecoc); + } + + mongoc_collection_t *new_coll = NULL; + db = mongoc_client_get_database (test_runner->internal_client, database_name); + new_coll = mongoc_database_create_collection (db, collection_name, create_opts, error); + if (!new_coll) { + goto loopexit; + } + mongoc_collection_destroy (new_coll); + /* Insert documents if specified. */ if (bson_count_keys (documents) > 0) { bson_iter_t documents_iter; @@ -946,15 +1003,6 @@ test_setup_initial_data (test_t *test, bson_error_t *error) if (!mongoc_bulk_operation_execute (bulk_insert, NULL, error)) { goto loopexit; } - } else { - mongoc_collection_t *new_coll = NULL; - /* Test does not need data inserted, just create the collection. */ - db = mongoc_client_get_database (test_runner->internal_client, database_name); - new_coll = mongoc_database_create_collection (db, collection_name, create_opts, error); - if (!new_coll) { - goto loopexit; - } - mongoc_collection_destroy (new_coll); } ret = true; @@ -1291,6 +1339,21 @@ test_count_matching_events_for_client ( return true; } +static bool +skip_cse_list_collections (const bson_t *event) +{ + if (!bson_has_field(event, "commandName") || !bson_has_field(event, "databaseName")) { + return false; + } + + const char *cmdname = bson_lookup_utf8 (event, "commandName"); + const char *dbname = bson_lookup_utf8 (event, "databaseName"); + if (cmdname && 0 == strcmp (cmdname, "listCollections") && dbname && 0 == strcmp (dbname, "keyvault")) { + return true; + } + return false; +} + static bool test_check_expected_events_for_client (test_t *test, bson_t *expected_events_for_client, bson_error_t *error) { @@ -1329,10 +1392,33 @@ test_check_expected_events_for_client (test_t *test, bson_t *expected_events_for goto done; } - uint32_t expected_num_events = bson_count_keys (expected_events); - uint32_t actual_num_events = 0; - event_t *eiter; + int actual_listCollections_count = 0; + LL_FOREACH (entity->events, eiter) + { + if (skip_cse_list_collections (eiter->serialized)) { + actual_listCollections_count++; + } + } + eiter = entity->events; + + bson_iter_t iter; + int expected_listCollections_count = 0; + BSON_FOREACH(expected_events, iter) { + bson_t expected_event; + bson_iter_bson (&iter, &expected_event); + BSON_FOREACH(&expected_event, iter) { + bson_t event_contents; + bson_iter_bson (&iter, &event_contents); + if (skip_cse_list_collections (&event_contents)) { + expected_listCollections_count++; + break; + } + } + } + + int expected_num_events = bson_count_keys (expected_events); + int actual_num_events = 0; LL_FOREACH (entity->events, eiter) { if (event_matches_eventtype (eiter, event_type)) { @@ -1340,39 +1426,71 @@ test_check_expected_events_for_client (test_t *test, bson_t *expected_events_for } } + { + const int difference = (actual_num_events - actual_listCollections_count) - (expected_num_events - expected_listCollections_count); + bool too_many_events = difference > 0; + bool too_few_events = difference < 0; + if (actual_listCollections_count < expected_listCollections_count) { + too_few_events = true; + } + if (expected_num_events != actual_num_events) { - bool too_many_events = actual_num_events > expected_num_events; if (ignore_extra_events && *ignore_extra_events) { // We can never have too many events too_many_events = false; } - bool too_few_events = actual_num_events < expected_num_events; if (too_few_events || too_many_events) { test_set_error ( - error, "expected: %" PRIu32 " events but got %" PRIu32, expected_num_events, actual_num_events); + error, "expected: %" PRIi32 " events but got %" PRIi32, expected_num_events, actual_num_events); goto done; } } eiter = entity->events; - bson_iter_t iter; BSON_FOREACH (expected_events, iter) { + bool matched = false; while (eiter && !event_matches_eventtype (eiter, event_type)) { eiter = eiter->next; } bson_t expected_event; bson_iter_bson (&iter, &expected_event); - if (!eiter) { - test_set_error (error, "could not find event: %s", tmp_json (&expected_event)); - goto done; - } - if (!test_check_event (test, &expected_event, eiter, error)) { - test_diagnostics_error_info ("checking for expected event: %s", tmp_json (&expected_event)); + + do { + if (!eiter) { + break; + } + matched = test_check_event (test, &expected_event, eiter, error); + if (matched) { + continue; + } + + if (ignore_extra_events) { + continue; + } + + if (skip_cse_list_collections (eiter->serialized)) { + continue; + } + + test_set_error (error, + "could not match event\n" + "\texpected: %s\n\n" + "\tactual : %s\n\n ", + bson_as_canonical_extended_json (&expected_event, NULL), + bson_as_canonical_extended_json (eiter->serialized, NULL)); + break; + } while ((eiter = eiter->next) && !matched); + + if (!matched) { goto done; + test_set_error (error, + "expectation unmatched\n" + "\texpected: %s\n\n", + tmp_json (&expected_event)); } - eiter = eiter->next; } +} ret = true; done: @@ -1762,16 +1880,24 @@ test_check_outcome_collection (test_t *test, bson_t *collection_data, bson_error bson_iter_bson (&eiter, &expected); expected_sorted = bson_copy_and_sort (&expected); + bson_val_t *actual_v = bson_val_from_bson (actual_sorted); + bson_val_t *expected_v = bson_val_from_bson (expected_sorted); - if (!bson_equal (actual_sorted, expected_sorted)) { + bson_matcher_context_t matcher_context = {.matcher = bson_matcher_new (), .path = "", .is_root = true}; + if (!bson_matcher_match (&matcher_context, expected_v, actual_v, error)) { test_set_error (error, "expected %s, but got %s", tmp_json (expected_sorted), tmp_json (actual_sorted)); + const char *got = tmp_json (actual_sorted); + printf ("this: %s", got); bson_destroy (actual_sorted); bson_destroy (expected_sorted); goto done; } + bson_matcher_destroy (matcher_context.matcher); bson_destroy (actual_sorted); bson_destroy (expected_sorted); + bson_val_destroy (actual_v); + bson_val_destroy (expected_v); bson_iter_next (&eiter); } From 6cad35f514aa988a5d344fa28f753842e8d0e95a Mon Sep 17 00:00:00 2001 From: mdb-ad <198671546+mdb-ad@users.noreply.github.com> Date: Sun, 4 May 2025 23:17:27 -0700 Subject: [PATCH 02/17] format --- src/libmongoc/tests/unified/operation.c | 2 +- src/libmongoc/tests/unified/result.c | 4 +- src/libmongoc/tests/unified/runner.c | 107 ++++++++++++------------ 3 files changed, 58 insertions(+), 55 deletions(-) diff --git a/src/libmongoc/tests/unified/operation.c b/src/libmongoc/tests/unified/operation.c index 4e42ccf5965..090f8b0d3c5 100644 --- a/src/libmongoc/tests/unified/operation.c +++ b/src/libmongoc/tests/unified/operation.c @@ -1853,7 +1853,7 @@ operation_distinct (test_t *test, operation_t *op, result_t *result, bson_error_ } distinct = BCON_NEW ( - "distinct", mongoc_collection_get_name (coll), "key", BCON_UTF8 (field_name), "query", BCON_DOCUMENT(filter)); + "distinct", mongoc_collection_get_name (coll), "key", BCON_UTF8 (field_name), "query", BCON_DOCUMENT (filter)); bson_destroy (&op_reply); mongoc_collection_read_command_with_opts (coll, distinct, NULL /* read prefs */, opts, &op_reply, &op_error); diff --git a/src/libmongoc/tests/unified/result.c b/src/libmongoc/tests/unified/result.c index cd53e9999d3..413c96609fb 100644 --- a/src/libmongoc/tests/unified/result.c +++ b/src/libmongoc/tests/unified/result.c @@ -413,8 +413,8 @@ result_check (result_t *result, entity_map_t *em, bson_val_t *expect_result, bso } if (error_contains) { - mongoc_lowercase_inplace(result->error.message); - mongoc_lowercase_inplace(error_contains); + mongoc_lowercase_inplace (result->error.message); + mongoc_lowercase_inplace (error_contains); if (strstr (result->error.message, error_contains) == NULL) { test_set_error ( error, "expected error to contain \"%s\", but got: \"%s\"", error_contains, result->error.message); diff --git a/src/libmongoc/tests/unified/runner.c b/src/libmongoc/tests/unified/runner.c index 96694b684ca..b7c2f00ceb9 100644 --- a/src/libmongoc/tests/unified/runner.c +++ b/src/libmongoc/tests/unified/runner.c @@ -1342,7 +1342,7 @@ test_count_matching_events_for_client ( static bool skip_cse_list_collections (const bson_t *event) { - if (!bson_has_field(event, "commandName") || !bson_has_field(event, "databaseName")) { + if (!bson_has_field (event, "commandName") || !bson_has_field (event, "databaseName")) { return false; } @@ -1404,10 +1404,12 @@ test_check_expected_events_for_client (test_t *test, bson_t *expected_events_for bson_iter_t iter; int expected_listCollections_count = 0; - BSON_FOREACH(expected_events, iter) { + BSON_FOREACH (expected_events, iter) + { bson_t expected_event; bson_iter_bson (&iter, &expected_event); - BSON_FOREACH(&expected_event, iter) { + BSON_FOREACH (&expected_event, iter) + { bson_t event_contents; bson_iter_bson (&iter, &event_contents); if (skip_cse_list_collections (&event_contents)) { @@ -1427,70 +1429,71 @@ test_check_expected_events_for_client (test_t *test, bson_t *expected_events_for } { - const int difference = (actual_num_events - actual_listCollections_count) - (expected_num_events - expected_listCollections_count); + const int difference = + (actual_num_events - actual_listCollections_count) - (expected_num_events - expected_listCollections_count); bool too_many_events = difference > 0; bool too_few_events = difference < 0; if (actual_listCollections_count < expected_listCollections_count) { too_few_events = true; } - if (expected_num_events != actual_num_events) { - if (ignore_extra_events && *ignore_extra_events) { - // We can never have too many events - too_many_events = false; - } - if (too_few_events || too_many_events) { - test_set_error ( - error, "expected: %" PRIi32 " events but got %" PRIi32, expected_num_events, actual_num_events); - goto done; - } - } - - eiter = entity->events; - BSON_FOREACH (expected_events, iter) - { - bool matched = false; - while (eiter && !event_matches_eventtype (eiter, event_type)) { - eiter = eiter->next; - } - bson_t expected_event; - bson_iter_bson (&iter, &expected_event); - - do { - if (!eiter) { - break; + if (expected_num_events != actual_num_events) { + if (ignore_extra_events && *ignore_extra_events) { + // We can never have too many events + too_many_events = false; } - matched = test_check_event (test, &expected_event, eiter, error); - if (matched) { - continue; + if (too_few_events || too_many_events) { + test_set_error ( + error, "expected: %" PRIi32 " events but got %" PRIi32, expected_num_events, actual_num_events); + goto done; } + } - if (ignore_extra_events) { - continue; + eiter = entity->events; + BSON_FOREACH (expected_events, iter) + { + bool matched = false; + while (eiter && !event_matches_eventtype (eiter, event_type)) { + eiter = eiter->next; } + bson_t expected_event; + bson_iter_bson (&iter, &expected_event); - if (skip_cse_list_collections (eiter->serialized)) { - continue; - } + do { + if (!eiter) { + break; + } + matched = test_check_event (test, &expected_event, eiter, error); + if (matched) { + continue; + } - test_set_error (error, - "could not match event\n" - "\texpected: %s\n\n" - "\tactual : %s\n\n ", - bson_as_canonical_extended_json (&expected_event, NULL), - bson_as_canonical_extended_json (eiter->serialized, NULL)); - break; - } while ((eiter = eiter->next) && !matched); + if (ignore_extra_events) { + continue; + } - if (!matched) { - goto done; - test_set_error (error, - "expectation unmatched\n" - "\texpected: %s\n\n", - tmp_json (&expected_event)); + if (skip_cse_list_collections (eiter->serialized)) { + continue; + } + + test_set_error (error, + "could not match event\n" + "\texpected: %s\n\n" + "\tactual : %s\n\n ", + bson_as_canonical_extended_json (&expected_event, NULL), + bson_as_canonical_extended_json (eiter->serialized, NULL)); + break; + } while ((eiter = eiter->next) && !matched); + + if (!matched) { + goto done; + test_set_error (error, + "expectation unmatched\n" + "\texpected: %s\n\n", + tmp_json (&expected_event)); + } } } -} ret = true; done: From 9f06c63d3dc6dbce2399089a35639458e227072f Mon Sep 17 00:00:00 2001 From: mdb-ad <198671546+mdb-ad@users.noreply.github.com> Date: Sun, 4 May 2025 23:35:36 -0700 Subject: [PATCH 03/17] sync poc-queryable-encryption --- .../unified/poc-queryable-encryption.json | 40 ++++++------------- 1 file changed, 12 insertions(+), 28 deletions(-) diff --git a/src/libmongoc/tests/json/unified/poc-queryable-encryption.json b/src/libmongoc/tests/json/unified/poc-queryable-encryption.json index 4fa5a350e03..b85bfffb939 100644 --- a/src/libmongoc/tests/json/unified/poc-queryable-encryption.json +++ b/src/libmongoc/tests/json/unified/poc-queryable-encryption.json @@ -3,18 +3,16 @@ "schemaVersion": "1.23", "runOnRequirements": [ { - "minServerVersion": "8.0" + "minServerVersion": "7.0", + "csfle": true } ], "createEntities": [ { "client": { "id": "client0", - "observeEvents": [ - "commandStartedEvent", - "commandSucceededEvent" - ], "autoEncryptOpts": { + "keyVaultNamespace": "keyvault.datakeys", "kmsProviders": { "local": { "key": "Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk" @@ -25,30 +23,16 @@ }, { "database": { - "id": "database0", + "id": "encryptedDB", "client": "client0", "databaseName": "poc-queryable-encryption" } }, { "collection": { - "id": "collection0", - "database": "database0", - "collectionName": "coll0" - } - }, - { - "database": { - "id": "database1", - "client": "client0", - "databaseName": "keyvault" - } - }, - { - "collection": { - "id": "collection1", - "database": "database0", - "collectionName": "datakeys" + "id": "encryptedColl", + "database": "encryptedDB", + "collectionName": "encrypted" } } ], @@ -89,7 +73,7 @@ }, { "databaseName": "poc-queryable-encryption", - "collectionName": "coll0", + "collectionName": "encrypted", "documents": [], "createOptions": { "encryptedFields": { @@ -120,7 +104,7 @@ "description": "insert, replace, and find with queryable encryption", "operations": [ { - "object": "collection0", + "object": "encryptedColl", "name": "insertOne", "arguments": { "document": { @@ -130,7 +114,7 @@ } }, { - "object": "collection0", + "object": "encryptedColl", "name": "replaceOne", "arguments": { "filter": { @@ -142,7 +126,7 @@ } }, { - "object": "collection0", + "object": "encryptedColl", "name": "find", "arguments": { "filter": { @@ -159,7 +143,7 @@ ], "outcome": [ { - "collectionName": "coll0", + "collectionName": "encrypted", "databaseName": "poc-queryable-encryption", "documents": [ { From 460c1439cce5712383388ed2dd6b5180388ca0a9 Mon Sep 17 00:00:00 2001 From: mdb-ad <198671546+mdb-ad@users.noreply.github.com> Date: Mon, 5 May 2025 00:03:25 -0700 Subject: [PATCH 04/17] backout bulkWrite --- .../crud/unified/client-bulkWrite-qe.json | 272 ------------------ 1 file changed, 272 deletions(-) delete mode 100644 src/libmongoc/tests/json/crud/unified/client-bulkWrite-qe.json diff --git a/src/libmongoc/tests/json/crud/unified/client-bulkWrite-qe.json b/src/libmongoc/tests/json/crud/unified/client-bulkWrite-qe.json deleted file mode 100644 index 4d73d01351f..00000000000 --- a/src/libmongoc/tests/json/crud/unified/client-bulkWrite-qe.json +++ /dev/null @@ -1,272 +0,0 @@ -{ - "description": "client bulkWrite with queryable encryption", - "schemaVersion": "1.23", - "runOnRequirements": [ - { - "minServerVersion": "8.0", - "serverless": "forbid" - } - ], - "createEntities": [ - { - "client": { - "id": "client0", - "observeEvents": [ - "commandStartedEvent", - "commandSucceededEvent" - ], - "autoEncryptOpts": { - "kmsProviders": { - "local": { - "key": "Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk" - } - } - } - } - }, - { - "database": { - "id": "database0", - "client": "client0", - "databaseName": "crud-tests" - } - }, - { - "collection": { - "id": "collection0", - "database": "database0", - "collectionName": "coll0" - } - }, - { - "database": { - "id": "database1", - "client": "client0", - "databaseName": "keyvault" - } - }, - { - "collection": { - "id": "collection1", - "database": "database0", - "collectionName": "datakeys" - } - } - ], - "initialData": [ - { - "databaseName": "keyvault", - "collectionName": "datakeys", - "documents": [ - { - "_id": { - "$binary": { - "base64": "EjRWeBI0mHYSNBI0VniQEg==", - "subType": "04" - } - }, - "keyAltNames": [ - "local_key" - ], - "keyMaterial": { - "$binary": { - "base64": "sHe0kz57YW7v8g9VP9sf/+K1ex4JqKc5rf/URX3n3p8XdZ6+15uXPaSayC6adWbNxkFskuMCOifDoTT+rkqMtFkDclOy884RuGGtUysq3X7zkAWYTKi8QAfKkajvVbZl2y23UqgVasdQu3OVBQCrH/xY00nNAs/52e958nVjBuzQkSb1T8pKJAyjZsHJ60+FtnfafDZSTAIBJYn7UWBCwQ==", - "subType": "00" - } - }, - "creationDate": { - "$date": { - "$numberLong": "1641024000000" - } - }, - "updateDate": { - "$date": { - "$numberLong": "1641024000000" - } - }, - "status": 1, - "masterKey": { - "provider": "local" - } - } - ] - }, - { - "databaseName": "crud-tests", - "collectionName": "coll0", - "documents": [], - "createOptions": { - "encryptedFields": { - "fields": [ - { - "keyId": { - "$binary": { - "base64": "EjRWeBI0mHYSNBI0VniQEg==", - "subType": "04" - } - }, - "path": "encryptedInt", - "bsonType": "int", - "queries": { - "queryType": "equality", - "contention": { - "$numberLong": "0" - } - } - } - ] - } - } - } - ], - "_yamlAnchors": { - "namespace": "crud-tests.coll0" - }, - "tests": [ - { - "description": "client bulkWrite QE replaceOne", - "operations": [ - { - "object": "collection0", - "name": "insertMany", - "arguments": { - "documents": [ - { - "_id": 1, - "encryptedInt": 11 - }, - { - "_id": 2, - "encryptedInt": 22 - }, - { - "_id": 3, - "encryptedInt": 33 - } - ] - } - }, - { - "object": "client0", - "name": "clientBulkWrite", - "arguments": { - "models": [ - { - "replaceOne": { - "namespace": "crud-tests.coll0", - "filter": { - "encryptedInt": { - "$eq": 11 - } - }, - "replacement": { - "encryptedInt": 44 - } - } - } - ] - }, - "expectResult": { - "insertedCount": 0, - "upsertedCount": 0, - "matchedCount": 1, - "modifiedCount": 1, - "deletedCount": 0 - } - }, - { - "object": "collection0", - "name": "find", - "arguments": { - "filter": { - "encryptedInt": 44 - } - }, - "expectResult": [ - { - "_id": 1, - "encryptedInt": 44 - } - ] - } - ], - "outcome": [ - { - "collectionName": "coll0", - "databaseName": "crud-tests", - "documents": [ - { - "_id": 1, - "encryptedInt": { - "$$type": "binData" - }, - "__safeContent__": { - "$$type": "array" - } - }, - { - "_id": 2, - "encryptedInt": { - "$$type": "binData" - }, - "__safeContent__": { - "$$type": "array" - } - }, - { - "_id": 3, - "encryptedInt": { - "$$type": "binData" - }, - "__safeContent__": { - "$$type": "array" - } - } - ] - } - ] - }, - { - "description": "client bulkWrite QE with multiple replace fails", - "operations": [ - { - "object": "client0", - "name": "clientBulkWrite", - "arguments": { - "models": [ - { - "replaceOne": { - "namespace": "crud-tests.coll0", - "filter": { - "encryptedInt": { - "$eq": 11 - } - }, - "replacement": { - "encryptedInt": 44 - } - } - }, - { - "replaceOne": { - "namespace": "crud-tests.coll0", - "filter": { - "encryptedInt": { - "$eq": 22 - } - }, - "replacement": { - "encryptedInt": 44 - } - } - } - ] - }, - "expectError": { - "isError": true - } - } - ] - } - ] -} From 0ecf3761b07636be8b2689d6cf291aa2eb67b098 Mon Sep 17 00:00:00 2001 From: mdb-ad <198671546+mdb-ad@users.noreply.github.com> Date: Wed, 7 May 2025 16:28:30 -0700 Subject: [PATCH 05/17] Apply suggestions from code review Co-authored-by: Kevin Albertson --- src/libmongoc/tests/unified/entity-map.c | 9 ++++++--- src/libmongoc/tests/unified/runner.c | 9 ++++++++- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/libmongoc/tests/unified/entity-map.c b/src/libmongoc/tests/unified/entity-map.c index fb3bd940a16..f2ac1253fa2 100644 --- a/src/libmongoc/tests/unified/entity-map.c +++ b/src/libmongoc/tests/unified/entity-map.c @@ -35,7 +35,6 @@ #include #include #include -#include "json-test.h" #define REDUCED_HEARTBEAT_FREQUENCY_MS 500 #define REDUCED_MIN_HEARTBEAT_FREQUENCY_MS 50 @@ -926,7 +925,9 @@ entity_client_new (entity_map_t *em, bson_t *bson, bson_error_t *error) } if (auto_encryption_opts) { - _parse_and_set_auto_encryption_opts (client, auto_encryption_opts, error); + if (!_parse_and_set_auto_encryption_opts (client, auto_encryption_opts, error)) { + goto done; + } } ret = true; @@ -1387,7 +1388,9 @@ _parse_and_set_auto_encryption_opts (mongoc_client_t *client, bson_t *opts, bson mongoc_auto_encryption_opts_set_extra (auto_encryption_opts, extra_options); } - mongoc_client_enable_auto_encryption (client, auto_encryption_opts, error); + if (!mongoc_client_enable_auto_encryption (client, auto_encryption_opts, error)) { + goto done; + } ret = true; done: diff --git a/src/libmongoc/tests/unified/runner.c b/src/libmongoc/tests/unified/runner.c index b7c2f00ceb9..75f0db745b5 100644 --- a/src/libmongoc/tests/unified/runner.c +++ b/src/libmongoc/tests/unified/runner.c @@ -948,6 +948,8 @@ test_setup_initial_data (test_t *test, bson_error_t *error) if (!mongoc_collection_drop_with_opts (coll_esc, drop_opts, error)) { if (error->code != 26 && (NULL == strstr (error->message, "ns not found"))) { /* This is not a "ns not found" error. Fail the test. */ + mongoc_collection_destroy (coll_esc); + bson_free (collection_name_esc); goto loopexit; } /* Clear an "ns not found" error. */ @@ -964,6 +966,8 @@ test_setup_initial_data (test_t *test, bson_error_t *error) if (!mongoc_collection_drop_with_opts (coll_ecoc, drop_opts, error)) { if (error->code != 26 && (NULL == strstr (error->message, "ns not found"))) { /* This is not a "ns not found" error. Fail the test. */ + mongoc_collection_destroy (coll_ecoc); + bson_free (collection_name_ecoc); goto loopexit; } /* Clear an "ns not found" error. */ @@ -1339,8 +1343,11 @@ test_count_matching_events_for_client ( return true; } +// `is_keyvault_listcollections` returns true if a `listCollections` event produced by libmongoc should be ignored. +// The extra events are caused by operations on the key vault collection. Unlike other drivers, libmongoc does not +// create a separate client for key vault operations. static bool -skip_cse_list_collections (const bson_t *event) +is_keyvault_listcollections (const bson_t *event) { if (!bson_has_field (event, "commandName") || !bson_has_field (event, "databaseName")) { return false; From 2b6fe86cac56a0ec8f5d2f405ce3388c3d3850db Mon Sep 17 00:00:00 2001 From: mdb-ad <198671546+mdb-ad@users.noreply.github.com> Date: Wed, 7 May 2025 16:33:19 -0700 Subject: [PATCH 06/17] remove skips --- src/libmongoc/tests/unified/runner.c | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/libmongoc/tests/unified/runner.c b/src/libmongoc/tests/unified/runner.c index 75f0db745b5..a5c3384d5a9 100644 --- a/src/libmongoc/tests/unified/runner.c +++ b/src/libmongoc/tests/unified/runner.c @@ -104,14 +104,7 @@ skipped_unified_test_t SKIPPED_TESTS[] = { // libmongoc does not support the optional findOne helper. {"retryable reads handshake failures", "collection.findOne succeeds after retryable handshake network error"}, {"retryable reads handshake failures", "collection.findOne succeeds after retryable handshake server error (ShutdownInProgress)"}, - {"types", SKIP_ALL_TESTS}, - // libmongoc does not support the optional listIndexNames helper. - {"retryable reads handshake failures", "collection.listIndexNames succeeds after retryable handshake network error"}, - {"retryable reads handshake failures", "collection.listIndexNames succeeds after retryable handshake server error (ShutdownInProgress)"}, - // libmongoc does not support mapReduce. - {"unsupportedCommand", SKIP_ALL_TESTS}, - // libmongoc does not support the timeoutMS URI option - {"timeoutMS", SKIP_ALL_TESTS}, + // libmongoc does not support the optional listIndexNames helper. {"retryable reads handshake failures", "collection.listIndexNames succeeds after retryable handshake network error"}, {"retryable reads handshake failures", "collection.listIndexNames succeeds after retryable handshake server error (ShutdownInProgress)"}, @@ -121,6 +114,7 @@ skipped_unified_test_t SKIPPED_TESTS[] = { // libmongoc does not include insertId in InsertOneResult {"cancel-server-check", SKIP_ALL_TESTS}, + {0}, }; // clang-format on From f4c466719c021288e1937ce37581e6585431f64a Mon Sep 17 00:00:00 2001 From: mdb-ad <198671546+mdb-ad@users.noreply.github.com> Date: Wed, 7 May 2025 16:33:28 -0700 Subject: [PATCH 07/17] exact bson matching --- src/libmongoc/tests/unified/runner.c | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/src/libmongoc/tests/unified/runner.c b/src/libmongoc/tests/unified/runner.c index a5c3384d5a9..b55bb9ffa0c 100644 --- a/src/libmongoc/tests/unified/runner.c +++ b/src/libmongoc/tests/unified/runner.c @@ -1397,7 +1397,7 @@ test_check_expected_events_for_client (test_t *test, bson_t *expected_events_for int actual_listCollections_count = 0; LL_FOREACH (entity->events, eiter) { - if (skip_cse_list_collections (eiter->serialized)) { + if (is_keyvault_listcollections (eiter->serialized)) { actual_listCollections_count++; } } @@ -1413,7 +1413,7 @@ test_check_expected_events_for_client (test_t *test, bson_t *expected_events_for { bson_t event_contents; bson_iter_bson (&iter, &event_contents); - if (skip_cse_list_collections (&event_contents)) { + if (is_keyvault_listcollections (&event_contents)) { expected_listCollections_count++; break; } @@ -1473,7 +1473,7 @@ test_check_expected_events_for_client (test_t *test, bson_t *expected_events_for continue; } - if (skip_cse_list_collections (eiter->serialized)) { + if (is_keyvault_listcollections (eiter->serialized)) { continue; } @@ -1884,24 +1884,16 @@ test_check_outcome_collection (test_t *test, bson_t *collection_data, bson_error bson_iter_bson (&eiter, &expected); expected_sorted = bson_copy_and_sort (&expected); - bson_val_t *actual_v = bson_val_from_bson (actual_sorted); - bson_val_t *expected_v = bson_val_from_bson (expected_sorted); - bson_matcher_context_t matcher_context = {.matcher = bson_matcher_new (), .path = "", .is_root = true}; - if (!bson_matcher_match (&matcher_context, expected_v, actual_v, error)) { + if (!bson_equal (actual_sorted, expected_sorted)) { test_set_error (error, "expected %s, but got %s", tmp_json (expected_sorted), tmp_json (actual_sorted)); - const char *got = tmp_json (actual_sorted); - printf ("this: %s", got); bson_destroy (actual_sorted); bson_destroy (expected_sorted); goto done; } - bson_matcher_destroy (matcher_context.matcher); bson_destroy (actual_sorted); bson_destroy (expected_sorted); - bson_val_destroy (actual_v); - bson_val_destroy (expected_v); bson_iter_next (&eiter); } From 7f91903f021b35998d54e4e0c78c3afb13ab3757 Mon Sep 17 00:00:00 2001 From: mdb-ad <198671546+mdb-ad@users.noreply.github.com> Date: Wed, 7 May 2025 16:41:32 -0700 Subject: [PATCH 08/17] add and use case insensitive matching --- src/libmongoc/tests/unified/result.c | 4 +--- src/libmongoc/tests/unified/util.c | 27 +++++++++++++++++++++++++++ src/libmongoc/tests/unified/util.h | 2 ++ 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/src/libmongoc/tests/unified/result.c b/src/libmongoc/tests/unified/result.c index 413c96609fb..3f6074b5b1e 100644 --- a/src/libmongoc/tests/unified/result.c +++ b/src/libmongoc/tests/unified/result.c @@ -413,9 +413,7 @@ result_check (result_t *result, entity_map_t *em, bson_val_t *expect_result, bso } if (error_contains) { - mongoc_lowercase_inplace (result->error.message); - mongoc_lowercase_inplace (error_contains); - if (strstr (result->error.message, error_contains) == NULL) { + if (mongoc_strcasestr (result->error.message, error_contains) == NULL) { test_set_error ( error, "expected error to contain \"%s\", but got: \"%s\"", error_contains, result->error.message); goto done; diff --git a/src/libmongoc/tests/unified/util.c b/src/libmongoc/tests/unified/util.c index ff35613a7a7..478622a646e 100644 --- a/src/libmongoc/tests/unified/util.c +++ b/src/libmongoc/tests/unified/util.c @@ -161,3 +161,30 @@ usecs_since_epoch (void) return secs * factor + usecs; } + +const char * +mongoc_strcasestr (const char *haystack, const char *needle) +{ + if (!*needle) { + return haystack; + } + + const char *h = haystack; + while (*h) { + const char *h_start = h; + const char *n = needle; + + while (*n && *h && tolower (*h) == tolower (*n)) { + h++; + n++; + } + + if (!*n) { + return h_start; // Match found + } + + h = h_start + 1; // Move to the next starting position in haystack + } + + return NULL; // No match found +} \ No newline at end of file diff --git a/src/libmongoc/tests/unified/util.h b/src/libmongoc/tests/unified/util.h index 4ea8c98a027..9ca4b8dd568 100644 --- a/src/libmongoc/tests/unified/util.h +++ b/src/libmongoc/tests/unified/util.h @@ -36,4 +36,6 @@ is_unsupported_event_type (const char *event_type); int64_t usecs_since_epoch (void); +const char * +mongoc_strcasestr (const char *haystack, const char *needle); #endif /* UNIFIED_UTIL_H */ From ecb725d13da7513dc508773de5f85ac6d9d925ea Mon Sep 17 00:00:00 2001 From: mdb-ad <198671546+mdb-ad@users.noreply.github.com> Date: Wed, 7 May 2025 16:51:29 -0700 Subject: [PATCH 09/17] remove extra event checking - already covered by events count --- src/libmongoc/tests/unified/runner.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/libmongoc/tests/unified/runner.c b/src/libmongoc/tests/unified/runner.c index b55bb9ffa0c..e2a91a52b07 100644 --- a/src/libmongoc/tests/unified/runner.c +++ b/src/libmongoc/tests/unified/runner.c @@ -1469,10 +1469,6 @@ test_check_expected_events_for_client (test_t *test, bson_t *expected_events_for continue; } - if (ignore_extra_events) { - continue; - } - if (is_keyvault_listcollections (eiter->serialized)) { continue; } @@ -1487,11 +1483,11 @@ test_check_expected_events_for_client (test_t *test, bson_t *expected_events_for } while ((eiter = eiter->next) && !matched); if (!matched) { - goto done; test_set_error (error, "expectation unmatched\n" "\texpected: %s\n\n", tmp_json (&expected_event)); + goto done; } } } From 449c8858cbb21f2627fbac4c2650b647e1f1fcb5 Mon Sep 17 00:00:00 2001 From: mdb-ad <198671546+mdb-ad@users.noreply.github.com> Date: Wed, 7 May 2025 16:54:05 -0700 Subject: [PATCH 10/17] whitespace --- src/libmongoc/tests/unified/util.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libmongoc/tests/unified/util.c b/src/libmongoc/tests/unified/util.c index 478622a646e..f514e3c067c 100644 --- a/src/libmongoc/tests/unified/util.c +++ b/src/libmongoc/tests/unified/util.c @@ -187,4 +187,4 @@ mongoc_strcasestr (const char *haystack, const char *needle) } return NULL; // No match found -} \ No newline at end of file +} From 4093dbee2130e5c88e8ef205f6a8694ae61d98e3 Mon Sep 17 00:00:00 2001 From: mdb-ad <198671546+mdb-ad@users.noreply.github.com> Date: Wed, 7 May 2025 17:21:42 -0700 Subject: [PATCH 11/17] revise poc-queryable-encryption --- .../unified/poc-queryable-encryption.json | 32 +++++++++++++++---- 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/src/libmongoc/tests/json/unified/poc-queryable-encryption.json b/src/libmongoc/tests/json/unified/poc-queryable-encryption.json index b85bfffb939..6f6e781ff87 100644 --- a/src/libmongoc/tests/json/unified/poc-queryable-encryption.json +++ b/src/libmongoc/tests/json/unified/poc-queryable-encryption.json @@ -34,6 +34,25 @@ "database": "encryptedDB", "collectionName": "encrypted" } + }, + { + "client": { + "id": "client1" + } + }, + { + "database": { + "id": "unencryptedDB", + "client": "client1", + "databaseName": "poc-queryable-encryption" + } + }, + { + "collection": { + "id": "unencryptedColl", + "database": "unencryptedDB", + "collectionName": "encrypted" + } } ], "initialData": [ @@ -139,13 +158,14 @@ "encryptedInt": 22 } ] - } - ], - "outcome": [ + }, { - "collectionName": "encrypted", - "databaseName": "poc-queryable-encryption", - "documents": [ + "object": "unencryptedColl", + "name": "find", + "arguments": { + "filter": {} + }, + "expectResult": [ { "_id": 1, "encryptedInt": { From 838941eee9e476e99b852ebe4ae4e168ce197da4 Mon Sep 17 00:00:00 2001 From: mdb-ad <198671546+mdb-ad@users.noreply.github.com> Date: Fri, 16 May 2025 15:00:38 -0700 Subject: [PATCH 12/17] sync json --- .../tests/json/unified/poc-queryable-encryption.json | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/libmongoc/tests/json/unified/poc-queryable-encryption.json b/src/libmongoc/tests/json/unified/poc-queryable-encryption.json index 6f6e781ff87..9788977cb60 100644 --- a/src/libmongoc/tests/json/unified/poc-queryable-encryption.json +++ b/src/libmongoc/tests/json/unified/poc-queryable-encryption.json @@ -171,9 +171,14 @@ "encryptedInt": { "$$type": "binData" }, - "__safeContent__": { - "$$type": "array" - } + "__safeContent__": [ + { + "$binary": { + "base64": "rhS16TJojgDDBtbluxBokvcotP1mQTGeYpNt8xd3MJQ=", + "subType": "00" + } + } + ] } ] } From 3ad2156974011b77df4657a29c03f29e61cc1065 Mon Sep 17 00:00:00 2001 From: mdb-ad <198671546+mdb-ad@users.noreply.github.com> Date: Fri, 16 May 2025 15:05:23 -0700 Subject: [PATCH 13/17] undo most runner changes --- src/libmongoc/tests/unified/runner.c | 123 +++++++++------------------ 1 file changed, 40 insertions(+), 83 deletions(-) diff --git a/src/libmongoc/tests/unified/runner.c b/src/libmongoc/tests/unified/runner.c index e2a91a52b07..b049d464ef9 100644 --- a/src/libmongoc/tests/unified/runner.c +++ b/src/libmongoc/tests/unified/runner.c @@ -1393,103 +1393,60 @@ test_check_expected_events_for_client (test_t *test, bson_t *expected_events_for goto done; } + uint32_t expected_num_events = bson_count_keys (expected_events); + uint32_t actual_num_events = 0; + event_t *eiter; - int actual_listCollections_count = 0; LL_FOREACH (entity->events, eiter) { if (is_keyvault_listcollections (eiter->serialized)) { - actual_listCollections_count++; + // Ignore. + continue; } - } - eiter = entity->events; - - bson_iter_t iter; - int expected_listCollections_count = 0; - BSON_FOREACH (expected_events, iter) - { - bson_t expected_event; - bson_iter_bson (&iter, &expected_event); - BSON_FOREACH (&expected_event, iter) - { - bson_t event_contents; - bson_iter_bson (&iter, &event_contents); - if (is_keyvault_listcollections (&event_contents)) { - expected_listCollections_count++; - break; - } + if (event_matches_eventtype (eiter, event_type)) { + actual_num_events++; } } - int expected_num_events = bson_count_keys (expected_events); - int actual_num_events = 0; - LL_FOREACH (entity->events, eiter) - { - if (event_matches_eventtype (eiter, event_type)) { - actual_num_events++; + if (expected_num_events != actual_num_events) { + bool too_many_events = actual_num_events > expected_num_events; + if (ignore_extra_events && *ignore_extra_events) { + // We can never have too many events + too_many_events = false; + } + bool too_few_events = actual_num_events < expected_num_events; + if (too_few_events || too_many_events) { + test_set_error ( + error, "expected: %" PRIu32 " events but got %" PRIu32, expected_num_events, actual_num_events); + goto done; } } + eiter = entity->events; + bson_iter_t iter; + BSON_FOREACH (expected_events, iter) { - const int difference = - (actual_num_events - actual_listCollections_count) - (expected_num_events - expected_listCollections_count); - bool too_many_events = difference > 0; - bool too_few_events = difference < 0; - if (actual_listCollections_count < expected_listCollections_count) { - too_few_events = true; - } - - if (expected_num_events != actual_num_events) { - if (ignore_extra_events && *ignore_extra_events) { - // We can never have too many events - too_many_events = false; - } - if (too_few_events || too_many_events) { - test_set_error ( - error, "expected: %" PRIi32 " events but got %" PRIi32, expected_num_events, actual_num_events); - goto done; - } + while (eiter && + (is_keyvault_listcollections (eiter->serialized) || !event_matches_eventtype (eiter, event_type))) { + // Skip. + eiter = eiter->next; + continue; } - - eiter = entity->events; - BSON_FOREACH (expected_events, iter) - { - bool matched = false; - while (eiter && !event_matches_eventtype (eiter, event_type)) { - eiter = eiter->next; - } - bson_t expected_event; - bson_iter_bson (&iter, &expected_event); - - do { - if (!eiter) { - break; - } - matched = test_check_event (test, &expected_event, eiter, error); - if (matched) { - continue; - } - - if (is_keyvault_listcollections (eiter->serialized)) { - continue; - } - - test_set_error (error, - "could not match event\n" - "\texpected: %s\n\n" - "\tactual : %s\n\n ", - bson_as_canonical_extended_json (&expected_event, NULL), - bson_as_canonical_extended_json (eiter->serialized, NULL)); - break; - } while ((eiter = eiter->next) && !matched); - - if (!matched) { - test_set_error (error, - "expectation unmatched\n" - "\texpected: %s\n\n", - tmp_json (&expected_event)); - goto done; - } + bson_t expected_event; + bson_iter_bson (&iter, &expected_event); + if (!eiter) { + test_set_error (error, "could not find event: %s", tmp_json (&expected_event)); + goto done; + } + if (!test_check_event (test, &expected_event, eiter, error)) { + test_diagnostics_error_info ("could not match event\n" + "\texpected: %s\n" + "\tactual: %s", + tmp_json (&expected_event), + tmp_json (eiter->serialized)); + goto done; } + eiter = eiter->next; } ret = true; From 676a8bd65b780998db63633c182fea4452cf3950 Mon Sep 17 00:00:00 2001 From: mdb-ad <198671546+mdb-ad@users.noreply.github.com> Date: Fri, 16 May 2025 15:10:07 -0700 Subject: [PATCH 14/17] remove migrated tests --- .../legacy/fle2v2-BypassQueryAnalysis.json | 262 ------------------ ...EncryptedFields-vs-EncryptedFieldsMap.json | 213 -------------- .../legacy/localSchema.json | 258 ----------------- .../legacy/maxWireVersion.json | 74 ----- 4 files changed, 807 deletions(-) delete mode 100644 src/libmongoc/tests/json/client_side_encryption/legacy/fle2v2-BypassQueryAnalysis.json delete mode 100644 src/libmongoc/tests/json/client_side_encryption/legacy/fle2v2-EncryptedFields-vs-EncryptedFieldsMap.json delete mode 100644 src/libmongoc/tests/json/client_side_encryption/legacy/localSchema.json delete mode 100644 src/libmongoc/tests/json/client_side_encryption/legacy/maxWireVersion.json diff --git a/src/libmongoc/tests/json/client_side_encryption/legacy/fle2v2-BypassQueryAnalysis.json b/src/libmongoc/tests/json/client_side_encryption/legacy/fle2v2-BypassQueryAnalysis.json deleted file mode 100644 index dcc3983ae0c..00000000000 --- a/src/libmongoc/tests/json/client_side_encryption/legacy/fle2v2-BypassQueryAnalysis.json +++ /dev/null @@ -1,262 +0,0 @@ -{ - "runOn": [ - { - "minServerVersion": "7.0.0", - "serverless": "forbid", - "topology": [ - "replicaset", - "sharded", - "load-balanced" - ] - } - ], - "database_name": "default", - "collection_name": "default", - "data": [], - "encrypted_fields": { - "fields": [ - { - "keyId": { - "$binary": { - "base64": "EjRWeBI0mHYSNBI0VniQEg==", - "subType": "04" - } - }, - "path": "encryptedIndexed", - "bsonType": "string", - "queries": { - "queryType": "equality", - "contention": { - "$numberLong": "0" - } - } - }, - { - "keyId": { - "$binary": { - "base64": "q83vqxI0mHYSNBI0VniQEg==", - "subType": "04" - } - }, - "path": "encryptedUnindexed", - "bsonType": "string" - } - ] - }, - "key_vault_data": [ - { - "_id": { - "$binary": { - "base64": "EjRWeBI0mHYSNBI0VniQEg==", - "subType": "04" - } - }, - "keyMaterial": { - "$binary": { - "base64": "sHe0kz57YW7v8g9VP9sf/+K1ex4JqKc5rf/URX3n3p8XdZ6+15uXPaSayC6adWbNxkFskuMCOifDoTT+rkqMtFkDclOy884RuGGtUysq3X7zkAWYTKi8QAfKkajvVbZl2y23UqgVasdQu3OVBQCrH/xY00nNAs/52e958nVjBuzQkSb1T8pKJAyjZsHJ60+FtnfafDZSTAIBJYn7UWBCwQ==", - "subType": "00" - } - }, - "creationDate": { - "$date": { - "$numberLong": "1648914851981" - } - }, - "updateDate": { - "$date": { - "$numberLong": "1648914851981" - } - }, - "status": { - "$numberInt": "0" - }, - "masterKey": { - "provider": "local" - } - } - ], - "tests": [ - { - "description": "BypassQueryAnalysis decrypts", - "clientOptions": { - "autoEncryptOpts": { - "kmsProviders": { - "local": { - "key": { - "$binary": { - "base64": "Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk", - "subType": "00" - } - } - } - }, - "bypassQueryAnalysis": true - } - }, - "operations": [ - { - "name": "insertOne", - "arguments": { - "document": { - "_id": 1, - "encryptedIndexed": { - "$binary": { - "base64": "C18BAAAFZAAgAAAAANnt+eLTkv4GdDPl8IAfJOvTzArOgFJQ2S/DcLza4W0DBXMAIAAAAAD2u+omZme3P2gBPehMQyQHQ153tPN1+z7bksYA9jKTpAVwADAAAAAAUnCOQqIvmR65YKyYnsiVfVrg9hwUVO3RhhKExo3RWOzgaS0QdsBL5xKFS0JhZSoWBXUAEAAAAAQSNFZ4EjSYdhI0EjRWeJASEHQAAgAAAAV2AFAAAAAAEjRWeBI0mHYSNBI0VniQEpQbp/ZJpWBKeDtKLiXb0P2E9wvc0g3f373jnYQYlJquOrlPOoEy3ngsHPJuSUijvWDsrQzqYa349K7G/66qaXEFZQAgAAAAAOuac/eRLYakKX6B0vZ1r3QodOQFfjqJD+xlGiPu4/PsBWwAIAAAAACkm0o9bj6j0HuADKc0svbqO2UHj6GrlNdF6yKNxh63xRJrAAAAAAAAAAAAAA==", - "subType": "06" - } - } - } - } - }, - { - "name": "find", - "arguments": { - "filter": { - "_id": 1 - } - }, - "result": [ - { - "_id": 1, - "encryptedIndexed": "123" - } - ] - } - ], - "expectations": [ - { - "command_started_event": { - "command": { - "listCollections": 1, - "filter": { - "name": "default" - } - }, - "command_name": "listCollections" - } - }, - { - "command_started_event": { - "command": { - "insert": "default", - "documents": [ - { - "_id": 1, - "encryptedIndexed": { - "$binary": { - "base64": "C18BAAAFZAAgAAAAANnt+eLTkv4GdDPl8IAfJOvTzArOgFJQ2S/DcLza4W0DBXMAIAAAAAD2u+omZme3P2gBPehMQyQHQ153tPN1+z7bksYA9jKTpAVwADAAAAAAUnCOQqIvmR65YKyYnsiVfVrg9hwUVO3RhhKExo3RWOzgaS0QdsBL5xKFS0JhZSoWBXUAEAAAAAQSNFZ4EjSYdhI0EjRWeJASEHQAAgAAAAV2AFAAAAAAEjRWeBI0mHYSNBI0VniQEpQbp/ZJpWBKeDtKLiXb0P2E9wvc0g3f373jnYQYlJquOrlPOoEy3ngsHPJuSUijvWDsrQzqYa349K7G/66qaXEFZQAgAAAAAOuac/eRLYakKX6B0vZ1r3QodOQFfjqJD+xlGiPu4/PsBWwAIAAAAACkm0o9bj6j0HuADKc0svbqO2UHj6GrlNdF6yKNxh63xRJrAAAAAAAAAAAAAA==", - "subType": "06" - } - } - } - ], - "ordered": true, - "encryptionInformation": { - "type": 1, - "schema": { - "default.default": { - "escCollection": "enxcol_.default.esc", - "ecocCollection": "enxcol_.default.ecoc", - "fields": [ - { - "keyId": { - "$binary": { - "base64": "EjRWeBI0mHYSNBI0VniQEg==", - "subType": "04" - } - }, - "path": "encryptedIndexed", - "bsonType": "string", - "queries": { - "queryType": "equality", - "contention": { - "$numberLong": "0" - } - } - }, - { - "keyId": { - "$binary": { - "base64": "q83vqxI0mHYSNBI0VniQEg==", - "subType": "04" - } - }, - "path": "encryptedUnindexed", - "bsonType": "string" - } - ] - } - } - } - }, - "command_name": "insert" - } - }, - { - "command_started_event": { - "command": { - "find": "default", - "filter": { - "_id": 1 - } - }, - "command_name": "find" - } - }, - { - "command_started_event": { - "command": { - "find": "datakeys", - "filter": { - "$or": [ - { - "_id": { - "$in": [ - { - "$binary": { - "base64": "EjRWeBI0mHYSNBI0VniQEg==", - "subType": "04" - } - } - ] - } - }, - { - "keyAltNames": { - "$in": [] - } - } - ] - }, - "$db": "keyvault", - "readConcern": { - "level": "majority" - } - }, - "command_name": "find" - } - } - ], - "outcome": { - "collection": { - "data": [ - { - "_id": 1, - "encryptedIndexed": { - "$$type": "binData" - }, - "__safeContent__": [ - { - "$binary": { - "base64": "31eCYlbQoVboc5zwC8IoyJVSkag9PxREka8dkmbXJeY=", - "subType": "00" - } - } - ] - } - ] - } - } - } - ] -} diff --git a/src/libmongoc/tests/json/client_side_encryption/legacy/fle2v2-EncryptedFields-vs-EncryptedFieldsMap.json b/src/libmongoc/tests/json/client_side_encryption/legacy/fle2v2-EncryptedFields-vs-EncryptedFieldsMap.json deleted file mode 100644 index b579979e945..00000000000 --- a/src/libmongoc/tests/json/client_side_encryption/legacy/fle2v2-EncryptedFields-vs-EncryptedFieldsMap.json +++ /dev/null @@ -1,213 +0,0 @@ -{ - "runOn": [ - { - "minServerVersion": "7.0.0", - "serverless": "forbid", - "topology": [ - "replicaset", - "sharded", - "load-balanced" - ] - } - ], - "database_name": "default", - "collection_name": "default", - "data": [], - "encrypted_fields": { - "fields": [ - { - "keyId": { - "$binary": { - "base64": "EjRWeBI0mHYSNBI0VniQEg==", - "subType": "04" - } - }, - "path": "encryptedIndexed", - "bsonType": "string", - "queries": { - "queryType": "equality", - "contention": { - "$numberLong": "0" - } - } - }, - { - "keyId": { - "$binary": { - "base64": "q83vqxI0mHYSNBI0VniQEg==", - "subType": "04" - } - }, - "path": "encryptedUnindexed", - "bsonType": "string" - } - ] - }, - "key_vault_data": [ - { - "_id": { - "$binary": { - "base64": "q83vqxI0mHYSNBI0VniQEg==", - "subType": "04" - } - }, - "keyMaterial": { - "$binary": { - "base64": "HBk9BWihXExNDvTp1lUxOuxuZK2Pe2ZdVdlsxPEBkiO1bS4mG5NNDsQ7zVxJAH8BtdOYp72Ku4Y3nwc0BUpIKsvAKX4eYXtlhv5zUQxWdeNFhg9qK7qb8nqhnnLeT0f25jFSqzWJoT379hfwDeu0bebJHr35QrJ8myZdPMTEDYF08QYQ48ShRBli0S+QzBHHAQiM2iJNr4svg2WR8JSeWQ==", - "subType": "00" - } - }, - "creationDate": { - "$date": { - "$numberLong": "1648914851981" - } - }, - "updateDate": { - "$date": { - "$numberLong": "1648914851981" - } - }, - "status": { - "$numberInt": "0" - }, - "masterKey": { - "provider": "local" - } - } - ], - "tests": [ - { - "description": "encryptedFieldsMap is preferred over remote encryptedFields", - "clientOptions": { - "autoEncryptOpts": { - "kmsProviders": { - "local": { - "key": { - "$binary": { - "base64": "Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk", - "subType": "00" - } - } - } - }, - "encryptedFieldsMap": { - "default.default": { - "fields": [] - } - } - } - }, - "operations": [ - { - "name": "insertOne", - "arguments": { - "document": { - "_id": 1, - "encryptedUnindexed": { - "$binary": { - "base64": "BqvN76sSNJh2EjQSNFZ4kBICTQaVZPWgXp41I7mPV1rLFTtw1tXzjcdSEyxpKKqujlko5TeizkB9hHQ009dVY1+fgIiDcefh+eQrm3CkhQ==", - "subType": "06" - } - } - } - } - }, - { - "name": "find", - "arguments": { - "filter": { - "_id": 1 - } - }, - "result": [ - { - "_id": 1, - "encryptedUnindexed": "value123" - } - ] - } - ], - "expectations": [ - { - "command_started_event": { - "command": { - "insert": "default", - "documents": [ - { - "_id": 1, - "encryptedUnindexed": { - "$binary": { - "base64": "BqvN76sSNJh2EjQSNFZ4kBICTQaVZPWgXp41I7mPV1rLFTtw1tXzjcdSEyxpKKqujlko5TeizkB9hHQ009dVY1+fgIiDcefh+eQrm3CkhQ==", - "subType": "06" - } - } - } - ], - "ordered": true - }, - "command_name": "insert" - } - }, - { - "command_started_event": { - "command": { - "find": "default", - "filter": { - "_id": 1 - } - }, - "command_name": "find" - } - }, - { - "command_started_event": { - "command": { - "find": "datakeys", - "filter": { - "$or": [ - { - "_id": { - "$in": [ - { - "$binary": { - "base64": "q83vqxI0mHYSNBI0VniQEg==", - "subType": "04" - } - } - ] - } - }, - { - "keyAltNames": { - "$in": [] - } - } - ] - }, - "$db": "keyvault", - "readConcern": { - "level": "majority" - } - }, - "command_name": "find" - } - } - ], - "outcome": { - "collection": { - "data": [ - { - "_id": 1, - "encryptedUnindexed": { - "$binary": { - "base64": "BqvN76sSNJh2EjQSNFZ4kBICTQaVZPWgXp41I7mPV1rLFTtw1tXzjcdSEyxpKKqujlko5TeizkB9hHQ009dVY1+fgIiDcefh+eQrm3CkhQ==", - "subType": "06" - } - } - } - ] - } - } - } - ] -} diff --git a/src/libmongoc/tests/json/client_side_encryption/legacy/localSchema.json b/src/libmongoc/tests/json/client_side_encryption/legacy/localSchema.json deleted file mode 100644 index 4698520f6fa..00000000000 --- a/src/libmongoc/tests/json/client_side_encryption/legacy/localSchema.json +++ /dev/null @@ -1,258 +0,0 @@ -{ - "runOn": [ - { - "minServerVersion": "4.1.10" - } - ], - "database_name": "default", - "collection_name": "default", - "data": [], - "json_schema": {}, - "key_vault_data": [ - { - "status": 1, - "_id": { - "$binary": { - "base64": "AAAAAAAAAAAAAAAAAAAAAA==", - "subType": "04" - } - }, - "masterKey": { - "provider": "aws", - "key": "arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-9f25-e30687b580d0", - "region": "us-east-1" - }, - "updateDate": { - "$date": { - "$numberLong": "1552949630483" - } - }, - "keyMaterial": { - "$binary": { - "base64": "AQICAHhQNmWG2CzOm1dq3kWLM+iDUZhEqnhJwH9wZVpuZ94A8gEqnsxXlR51T5EbEVezUqqKAAAAwjCBvwYJKoZIhvcNAQcGoIGxMIGuAgEAMIGoBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDHa4jo6yp0Z18KgbUgIBEIB74sKxWtV8/YHje5lv5THTl0HIbhSwM6EqRlmBiFFatmEWaeMk4tO4xBX65eq670I5TWPSLMzpp8ncGHMmvHqRajNBnmFtbYxN3E3/WjxmdbOOe+OXpnGJPcGsftc7cB2shRfA4lICPnE26+oVNXT6p0Lo20nY5XC7jyCO", - "subType": "00" - } - }, - "creationDate": { - "$date": { - "$numberLong": "1552949630483" - } - }, - "keyAltNames": [ - "altname", - "another_altname" - ] - } - ], - "tests": [ - { - "description": "A local schema should override", - "clientOptions": { - "autoEncryptOpts": { - "schemaMap": { - "default.default": { - "properties": { - "encrypted_w_altname": { - "encrypt": { - "keyId": "/altname", - "bsonType": "string", - "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Random" - } - }, - "encrypted_string": { - "encrypt": { - "keyId": [ - { - "$binary": { - "base64": "AAAAAAAAAAAAAAAAAAAAAA==", - "subType": "04" - } - } - ], - "bsonType": "string", - "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic" - } - }, - "random": { - "encrypt": { - "keyId": [ - { - "$binary": { - "base64": "AAAAAAAAAAAAAAAAAAAAAA==", - "subType": "04" - } - } - ], - "bsonType": "string", - "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Random" - } - }, - "encrypted_string_equivalent": { - "encrypt": { - "keyId": [ - { - "$binary": { - "base64": "AAAAAAAAAAAAAAAAAAAAAA==", - "subType": "04" - } - } - ], - "bsonType": "string", - "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic" - } - } - }, - "bsonType": "object" - } - }, - "kmsProviders": { - "aws": {} - } - } - }, - "operations": [ - { - "name": "insertOne", - "arguments": { - "document": { - "_id": 1, - "encrypted_string": "string0" - } - } - }, - { - "name": "find", - "arguments": { - "filter": { - "_id": 1 - } - }, - "result": [ - { - "_id": 1, - "encrypted_string": "string0" - } - ] - } - ], - "expectations": [ - { - "command_started_event": { - "command": { - "find": "datakeys", - "filter": { - "$or": [ - { - "_id": { - "$in": [ - { - "$binary": { - "base64": "AAAAAAAAAAAAAAAAAAAAAA==", - "subType": "04" - } - } - ] - } - }, - { - "keyAltNames": { - "$in": [] - } - } - ] - }, - "$db": "keyvault", - "readConcern": { - "level": "majority" - } - }, - "command_name": "find" - } - }, - { - "command_started_event": { - "command": { - "insert": "default", - "documents": [ - { - "_id": 1, - "encrypted_string": { - "$binary": { - "base64": "AQAAAAAAAAAAAAAAAAAAAAACwj+3zkv2VM+aTfk60RqhXq6a/77WlLwu/BxXFkL7EppGsju/m8f0x5kBDD3EZTtGALGXlym5jnpZAoSIkswHoA==", - "subType": "06" - } - } - } - ], - "ordered": true - }, - "command_name": "insert" - } - }, - { - "command_started_event": { - "command": { - "find": "default", - "filter": { - "_id": 1 - } - }, - "command_name": "find" - } - } - ], - "outcome": { - "collection": { - "data": [ - { - "_id": 1, - "encrypted_string": { - "$binary": { - "base64": "AQAAAAAAAAAAAAAAAAAAAAACwj+3zkv2VM+aTfk60RqhXq6a/77WlLwu/BxXFkL7EppGsju/m8f0x5kBDD3EZTtGALGXlym5jnpZAoSIkswHoA==", - "subType": "06" - } - } - } - ] - } - } - }, - { - "description": "A local schema with no encryption is an error", - "clientOptions": { - "autoEncryptOpts": { - "schemaMap": { - "default.default": { - "properties": { - "test": { - "bsonType": "string" - } - }, - "bsonType": "object", - "required": [ - "test" - ] - } - }, - "kmsProviders": { - "aws": {} - } - } - }, - "operations": [ - { - "name": "insertOne", - "arguments": { - "document": { - "_id": 1, - "encrypted_string": "string0" - } - }, - "result": { - "errorContains": "JSON schema keyword 'required' is only allowed with a remote schema" - } - } - ] - } - ] -} diff --git a/src/libmongoc/tests/json/client_side_encryption/legacy/maxWireVersion.json b/src/libmongoc/tests/json/client_side_encryption/legacy/maxWireVersion.json deleted file mode 100644 index f04f58dffde..00000000000 --- a/src/libmongoc/tests/json/client_side_encryption/legacy/maxWireVersion.json +++ /dev/null @@ -1,74 +0,0 @@ -{ - "runOn": [ - { - "maxServerVersion": "4.0.99" - } - ], - "database_name": "default", - "collection_name": "default", - "data": [], - "key_vault_data": [ - { - "status": 1, - "_id": { - "$binary": { - "base64": "AAAAAAAAAAAAAAAAAAAAAA==", - "subType": "04" - } - }, - "masterKey": { - "provider": "aws", - "key": "arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-9f25-e30687b580d0", - "region": "us-east-1" - }, - "updateDate": { - "$date": { - "$numberLong": "1552949630483" - } - }, - "keyMaterial": { - "$binary": { - "base64": "AQICAHhQNmWG2CzOm1dq3kWLM+iDUZhEqnhJwH9wZVpuZ94A8gEqnsxXlR51T5EbEVezUqqKAAAAwjCBvwYJKoZIhvcNAQcGoIGxMIGuAgEAMIGoBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDHa4jo6yp0Z18KgbUgIBEIB74sKxWtV8/YHje5lv5THTl0HIbhSwM6EqRlmBiFFatmEWaeMk4tO4xBX65eq670I5TWPSLMzpp8ncGHMmvHqRajNBnmFtbYxN3E3/WjxmdbOOe+OXpnGJPcGsftc7cB2shRfA4lICPnE26+oVNXT6p0Lo20nY5XC7jyCO", - "subType": "00" - } - }, - "creationDate": { - "$date": { - "$numberLong": "1552949630483" - } - }, - "keyAltNames": [ - "altname", - "another_altname" - ] - } - ], - "tests": [ - { - "description": "operation fails with maxWireVersion < 8", - "clientOptions": { - "autoEncryptOpts": { - "kmsProviders": { - "aws": {} - }, - "extraOptions": { - "mongocryptdBypassSpawn": true - } - } - }, - "operations": [ - { - "name": "insertOne", - "arguments": { - "document": { - "encrypted_string": "string0" - } - }, - "result": { - "errorContains": "Auto-encryption requires a minimum MongoDB version of 4.2" - } - } - ] - } - ] -} From 3788a54c6dc2f3051a90541dda3d64d8c00f8bd3 Mon Sep 17 00:00:00 2001 From: mdb-ad <198671546+mdb-ad@users.noreply.github.com> Date: Mon, 19 May 2025 14:59:23 -0700 Subject: [PATCH 15/17] Apply suggestions from code review Co-authored-by: Ezra Chung <88335979+eramongodb@users.noreply.github.com> --- src/libmongoc/src/mongoc/mongoc-util.c | 2 +- src/libmongoc/tests/unified/entity-map.c | 7 ++++++- src/libmongoc/tests/unified/util.c | 3 +++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/libmongoc/src/mongoc/mongoc-util.c b/src/libmongoc/src/mongoc/mongoc-util.c index 60c8b35d4ba..30654e329ff 100644 --- a/src/libmongoc/src/mongoc/mongoc-util.c +++ b/src/libmongoc/src/mongoc/mongoc-util.c @@ -531,7 +531,7 @@ mongoc_lowercase_inplace (char *src) { for (; *src; ++src) { /* UTF8 non-ascii characters have a 1 at the leftmost bit. If this is the - * case, just copy */ + * case, just leave as-is */ if ((*src & (0x1 << 7)) == 0) { *src = (char) tolower (*src); } diff --git a/src/libmongoc/tests/unified/entity-map.c b/src/libmongoc/tests/unified/entity-map.c index 738b22ebd5e..3c661a7374b 100644 --- a/src/libmongoc/tests/unified/entity-map.c +++ b/src/libmongoc/tests/unified/entity-map.c @@ -1250,7 +1250,12 @@ _parse_kms_provider_local ( static bool _get_kms_providers_docs (bson_t *kms_from_file, bson_t *kms_providers, bson_t *tls_opts, bson_error_t *error) -{ +{ + BSON_ASSERT_PARAM (kms_from_file); + BSON_ASSERT_PARAM (kms_providers); + BSON_ASSERT_PARAM (tls_opts); + BSON_OPTIONAL_PARAM (error); + /* Map provider to corresponding KMS parser. */ typedef struct _prov_map_t { const char *provider; diff --git a/src/libmongoc/tests/unified/util.c b/src/libmongoc/tests/unified/util.c index f514e3c067c..664c28636c0 100644 --- a/src/libmongoc/tests/unified/util.c +++ b/src/libmongoc/tests/unified/util.c @@ -165,6 +165,9 @@ usecs_since_epoch (void) const char * mongoc_strcasestr (const char *haystack, const char *needle) { + BSON_ASSERT_PARAM (haystack); + BSON_ASSERT_PARAM (needle); + if (!*needle) { return haystack; } From 3f7a8b0fa715837dc343f40e401b93f4c86d850f Mon Sep 17 00:00:00 2001 From: mdb-ad <198671546+mdb-ad@users.noreply.github.com> Date: Wed, 28 May 2025 18:43:06 -0700 Subject: [PATCH 16/17] sync tests --- .../unified/fle2v2-BypassQueryAnalysis.json | 11 +- .../unified/keyCache.json | 239 +++++++++++++----- 2 files changed, 189 insertions(+), 61 deletions(-) diff --git a/src/libmongoc/tests/json/client_side_encryption/unified/fle2v2-BypassQueryAnalysis.json b/src/libmongoc/tests/json/client_side_encryption/unified/fle2v2-BypassQueryAnalysis.json index 0817508f8f8..eef78b0f4c3 100644 --- a/src/libmongoc/tests/json/client_side_encryption/unified/fle2v2-BypassQueryAnalysis.json +++ b/src/libmongoc/tests/json/client_side_encryption/unified/fle2v2-BypassQueryAnalysis.json @@ -18,12 +18,12 @@ "client": { "id": "client0", "autoEncryptOpts": { + "keyVaultNamespace": "keyvault.datakeys", "kmsProviders": { "local": { "key": "Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk" } }, - "keyVaultNamespace": "keyvault.datakeys", "bypassQueryAnalysis": true }, "observeEvents": [ @@ -47,13 +47,16 @@ }, { "client": { - "id": "client1" + "id": "unencryptedClient", + "observeEvents": [ + "commandStartedEvent" + ] } }, { "database": { "id": "unencryptedDB", - "client": "client1", + "client": "unencryptedClient", "databaseName": "default" } }, @@ -175,8 +178,8 @@ ] }, { - "object": "unencryptedColl", "name": "find", + "object": "unencryptedColl", "arguments": { "filter": {} }, diff --git a/src/libmongoc/tests/json/client_side_encryption/unified/keyCache.json b/src/libmongoc/tests/json/client_side_encryption/unified/keyCache.json index a39701e2861..938b4fcb928 100644 --- a/src/libmongoc/tests/json/client_side_encryption/unified/keyCache.json +++ b/src/libmongoc/tests/json/client_side_encryption/unified/keyCache.json @@ -1,66 +1,138 @@ { - "description": "keyCache-explicit", - "schemaVersion": "1.22", + "description": "keyCache", + "schemaVersion": "1.23", "runOnRequirements": [ { - "csfle": true + "minServerVersion": "4.1.10" } ], "createEntities": [ { "client": { "id": "client0", - "observeEvents": [ - "commandStartedEvent" - ] - } - }, - { - "clientEncryption": { - "id": "clientEncryption0", - "clientEncryptionOpts": { - "keyVaultClient": "client0", + "autoEncryptOpts": { "keyVaultNamespace": "keyvault.datakeys", "kmsProviders": { - "local": { - "key": "OCTP9uKPPmvuqpHlqq83gPk4U6rUPxKVRRyVtrjFmVjdoa4Xzm1SzUbr7aIhNI42czkUBmrCtZKF31eaaJnxEBkqf0RFukA9Mo3NEHQWgAQ2cn9duOcRbaFUQo2z0/rB" + "aws": { + "accessKeyId": { + "$$placeholder": 1 + }, + "secretAccessKey": { + "$$placeholder": 1 + } } }, "keyExpirationMS": 1 - } + }, + "observeEvents": [ + "commandStartedEvent" + ] } }, { "database": { - "id": "database0", + "id": "db", "client": "client0", - "databaseName": "keyvault" + "databaseName": "default" } }, { "collection": { - "id": "collection0", - "database": "database0", - "collectionName": "datakeys" + "id": "coll", + "database": "db", + "collectionName": "default" } } ], "initialData": [ + { + "databaseName": "default", + "collectionName": "default", + "documents": [], + "createOptions": { + "validator": { + "$jsonSchema": { + "properties": { + "encrypted_w_altname": { + "encrypt": { + "keyId": "/altname", + "bsonType": "string", + "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Random" + } + }, + "encrypted_string": { + "encrypt": { + "keyId": [ + { + "$binary": { + "base64": "AAAAAAAAAAAAAAAAAAAAAA==", + "subType": "04" + } + } + ], + "bsonType": "string", + "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic" + } + }, + "random": { + "encrypt": { + "keyId": [ + { + "$binary": { + "base64": "AAAAAAAAAAAAAAAAAAAAAA==", + "subType": "04" + } + } + ], + "bsonType": "string", + "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Random" + } + }, + "encrypted_string_equivalent": { + "encrypt": { + "keyId": [ + { + "$binary": { + "base64": "AAAAAAAAAAAAAAAAAAAAAA==", + "subType": "04" + } + } + ], + "bsonType": "string", + "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic" + } + } + }, + "bsonType": "object" + } + } + } + }, { "databaseName": "keyvault", "collectionName": "datakeys", "documents": [ { + "status": 1, "_id": { "$binary": { - "base64": "a+YWzdygTAG62/cNUkqZiQ==", + "base64": "AAAAAAAAAAAAAAAAAAAAAA==", "subType": "04" } }, - "keyAltNames": [], + "masterKey": { + "provider": "aws", + "key": "arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-9f25-e30687b580d0", + "region": "us-east-1" + }, + "updateDate": { + "$date": { + "$numberLong": "1552949630483" + } + }, "keyMaterial": { "$binary": { - "base64": "iocBkhO3YBokiJ+FtxDTS71/qKXQ7tSWhWbcnFTXBcMjarsepvALeJ5li+SdUd9ePuatjidxAdMo7vh1V2ZESLMkQWdpPJ9PaJjA67gKQKbbbB4Ik5F2uKjULvrMBnFNVRMup4JNUwWFQJpqbfMveXnUVcD06+pUpAkml/f+DSXrV3e5rxciiNVtz03dAG8wJrsKsFXWj6vTjFhsfknyBA==", + "base64": "AQICAHhQNmWG2CzOm1dq3kWLM+iDUZhEqnhJwH9wZVpuZ94A8gEqnsxXlR51T5EbEVezUqqKAAAAwjCBvwYJKoZIhvcNAQcGoIGxMIGuAgEAMIGoBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDHa4jo6yp0Z18KgbUgIBEIB74sKxWtV8/YHje5lv5THTl0HIbhSwM6EqRlmBiFFatmEWaeMk4tO4xBX65eq670I5TWPSLMzpp8ncGHMmvHqRajNBnmFtbYxN3E3/WjxmdbOOe+OXpnGJPcGsftc7cB2shRfA4lICPnE26+oVNXT6p0Lo20nY5XC7jyCO", "subType": "00" } }, @@ -69,63 +141,83 @@ "$numberLong": "1552949630483" } }, - "updateDate": { - "$date": { - "$numberLong": "1552949630483" - } - }, - "status": { - "$numberInt": "0" - }, - "masterKey": { - "provider": "local" - } + "keyAltNames": [ + "altname", + "another_altname" + ] } ] } ], "tests": [ { - "description": "decrypt, wait, and decrypt again", + "description": "Insert with deterministic encryption, then find it", "operations": [ { - "name": "decrypt", - "object": "clientEncryption0", + "name": "insertOne", "arguments": { - "value": { - "$binary": { - "base64": "AWvmFs3coEwButv3DVJKmYkCJ6lUzRX9R28WNlw5uyndb+8gurA+p8q14s7GZ04K2ZvghieRlAr5UwZbow3PMq27u5EIhDDczwBFcbdP1amllw==", - "subType": "06" - } + "document": { + "_id": 1, + "encrypted_string": "string0" } }, - "expectResult": "foobar" + "object": "coll" }, { "name": "wait", - "object": "testRunner", + "object": "coll", "arguments": { "ms": 50 } }, { - "name": "decrypt", - "object": "clientEncryption0", + "name": "find", "arguments": { - "value": { - "$binary": { - "base64": "AWvmFs3coEwButv3DVJKmYkCJ6lUzRX9R28WNlw5uyndb+8gurA+p8q14s7GZ04K2ZvghieRlAr5UwZbow3PMq27u5EIhDDczwBFcbdP1amllw==", - "subType": "06" - } + "filter": { + "_id": 1 } }, - "expectResult": "foobar" + "object": "coll", + "expectResult": [ + { + "_id": 1, + "encrypted_string": "string0" + } + ] + } + ], + "outcome": [ + { + "documents": [ + { + "_id": 1, + "encrypted_string": { + "$binary": { + "base64": "AQAAAAAAAAAAAAAAAAAAAAACwj+3zkv2VM+aTfk60RqhXq6a/77WlLwu/BxXFkL7EppGsju/m8f0x5kBDD3EZTtGALGXlym5jnpZAoSIkswHoA==", + "subType": "06" + } + } + } + ], + "collectionName": "default", + "databaseName": "default" } ], "expectEvents": [ { "client": "client0", "events": [ + { + "commandStartedEvent": { + "command": { + "listCollections": 1, + "filter": { + "name": "default" + } + }, + "commandName": "listCollections" + } + }, { "commandStartedEvent": { "command": { @@ -137,7 +229,7 @@ "$in": [ { "$binary": { - "base64": "a+YWzdygTAG62/cNUkqZiQ==", + "base64": "AAAAAAAAAAAAAAAAAAAAAA==", "subType": "04" } } @@ -155,7 +247,39 @@ "readConcern": { "level": "majority" } - } + }, + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "command": { + "insert": "default", + "documents": [ + { + "_id": 1, + "encrypted_string": { + "$binary": { + "base64": "AQAAAAAAAAAAAAAAAAAAAAACwj+3zkv2VM+aTfk60RqhXq6a/77WlLwu/BxXFkL7EppGsju/m8f0x5kBDD3EZTtGALGXlym5jnpZAoSIkswHoA==", + "subType": "06" + } + } + } + ], + "ordered": true + }, + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "command": { + "find": "default", + "filter": { + "_id": 1 + } + }, + "commandName": "find" } }, { @@ -169,7 +293,7 @@ "$in": [ { "$binary": { - "base64": "a+YWzdygTAG62/cNUkqZiQ==", + "base64": "AAAAAAAAAAAAAAAAAAAAAA==", "subType": "04" } } @@ -187,7 +311,8 @@ "readConcern": { "level": "majority" } - } + }, + "commandName": "find" } } ] From b249b2cca6a4584f7c23d7d52c00800efc5b00af Mon Sep 17 00:00:00 2001 From: mdb-ad <198671546+mdb-ad@users.noreply.github.com> Date: Wed, 28 May 2025 19:13:45 -0700 Subject: [PATCH 17/17] document drop helpers --- src/libmongoc/tests/unified/runner.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libmongoc/tests/unified/runner.c b/src/libmongoc/tests/unified/runner.c index 6ef52c4dfaf..a2bb97062e9 100644 --- a/src/libmongoc/tests/unified/runner.c +++ b/src/libmongoc/tests/unified/runner.c @@ -640,6 +640,7 @@ check_schema_version (test_file_t *test_file) // 1.18 is partially supported (additional properties in kmsProviders) // 1.21 is partially supported (expectedError.writeErrors and expectedError.writeConcernErrors) // 1.22 is partially supported (keyExpirationMS in client encryption options) + // 1.23 is partially supported (automatic encryption) semver_t schema_version; semver_parse ("1.23", &schema_version); @@ -963,7 +964,8 @@ test_setup_initial_data (test_t *test, bson_error_t *error) memset (error, 0, sizeof (bson_error_t)); } - // Also drop `enxcol_..esc` and `enxcol_..ecoc` in case the collection will be used for QE. + // Drop `enxcol_..esc` and `enxcol_..ecoc` in case the collection will be used for QE. + // https://github.com/mongodb/specifications/blob/f4c0bbdbf8a8560580c947ca2c331794431a0c78/source/unified-test-format/unified-test-format.md#executing-a-test { char *collection_name_esc = bson_strdup_printf ("enxcol_.%s.esc", collection_name); mongoc_collection_t *coll_esc =