Migrate remaining chaos/MTLS/CRUD tests for edge server

client/src/cbltest/api/edgeserver.py

@@ -152,9 +152,15 @@ def _create_session(
         self, scheme: str, url: str, port: int, auth: BasicAuth | None
     ) -> ClientSession:
         if self.__secure:
-            ssl_context = ssl.create_default_context()
+            CERT_DIR = Path.home() / ".cbl_certs"
+            ssl_context = ssl.create_default_context(cafile=CERT_DIR / "ca_cert.pem")
             ssl_context.check_hostname = False
-            ssl_context.verify_mode = ssl.CERT_NONE
+            if self.__mtls:
+                ssl_context.load_cert_chain(
+                    certfile=str(CERT_DIR / "client_cert.pem"),
+                    keyfile=str(CERT_DIR / "client_key.pem"),
+                )
+
             return ClientSession(
                 f"{scheme}{url}:{port}",
                 auth=auth,

environment/aws/common/x509_certificate.py

@@ -9,6 +9,7 @@
     pkcs12,
 )
 from cryptography.x509 import (
+    BasicConstraints,
     Certificate,
     CertificateBuilder,
     ExtendedKeyUsage,
@@ -79,3 +80,69 @@ def create_self_signed_certificate(CN: str) -> CertKeyPair:
     )
 
     return CertKeyPair(leaf_certificate, private_key)
+
+
+def create_ca():
+    key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
+
+    subject = issuer = Name([NameAttribute(NameOID.COMMON_NAME, "EdgeTestCA")])
+
+    cert = (
+        CertificateBuilder()
+        .subject_name(subject)
+        .issuer_name(issuer)
+        .public_key(key.public_key())
+        .serial_number(random_serial_number())
+        .not_valid_before(datetime.now(timezone.utc))
+        .not_valid_after(datetime.now(timezone.utc) + timedelta(days=365))
+        .add_extension(BasicConstraints(ca=True, path_length=None), critical=True)
+        .sign(key, hashes.SHA256())
+    )
+
+    return CertKeyPair(cert, key)
+
+
+def create_signed_cert(cn: str, ca: CertKeyPair):
+    key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
+
+    subject = Name([NameAttribute(NameOID.COMMON_NAME, cn)])
+
+    cert = (
+        CertificateBuilder()
+        .subject_name(subject)
+        .issuer_name(ca.certificate.subject)
+        .public_key(key.public_key())
+        .serial_number(random_serial_number())
+        .not_valid_before(datetime.now(timezone.utc))
+        .not_valid_after(datetime.now(timezone.utc) + timedelta(days=365))
+        .add_extension(
+            ExtendedKeyUsage([ExtendedKeyUsageOID.SERVER_AUTH]),
+            critical=False,
+        )
+        .sign(ca.private_key, hashes.SHA256())
+    )
+
+    return CertKeyPair(cert, key)
+
+
+def create_client_cert(cn: str, ca: CertKeyPair):
+    key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
+
+    subject = Name([NameAttribute(NameOID.COMMON_NAME, cn)])
+
+    cert = (
+        CertificateBuilder()
+        .subject_name(subject)
+        .issuer_name(ca.certificate.subject)
+        .public_key(key.public_key())
+        .serial_number(random_serial_number())
+        .not_valid_before(datetime.now(timezone.utc))
+        .not_valid_after(datetime.now(timezone.utc) + timedelta(days=365))
+        .add_extension(
+            ExtendedKeyUsage([ExtendedKeyUsageOID.CLIENT_AUTH]),
+            critical=False,
+        )
+        .sign(ca.private_key, hashes.SHA256())
+    )
+
+    return CertKeyPair(cert, key)

environment/aws/es_setup/setup_edge_servers.py

@@ -35,7 +35,11 @@
 
 from environment.aws.common.io import LIGHT_GRAY, sftp_progress_bar
 from environment.aws.common.output import header
-from environment.aws.common.x509_certificate import create_self_signed_certificate
+from environment.aws.common.x509_certificate import (
+    create_ca,
+    create_client_cert,
+    create_signed_cert,
+)
 from environment.aws.topology_setup.setup_topology import TopologyConfig
 
 SCRIPT_DIR = Path(__file__).resolve().parent
@@ -257,17 +261,43 @@ def setup_server(
         )
 
     sftp_progress_bar(sftp, SCRIPT_DIR / "Caddyfile", "/home/ec2-user/Caddyfile")
-    cert = create_self_signed_certificate(hostname)
+    ca = create_ca()
+    ca_cert = ca.pem_bytes()
+    cert = create_signed_cert(hostname, ca)
     cert_pem = cert.pem_bytes()
     key_pem = cert.private_pem_bytes()
+    client = create_client_cert("test-client", ca)
+    client_cert = client.pem_bytes()
+    client_key = client.private_pem_bytes()
+
     with open("/tmp/es_key.pem", "wb") as f:
         f.write(key_pem)
 
     with open("/tmp/es_cert.pem", "wb") as f:
         f.write(cert_pem)
 
+    with open("/tmp/ca_cert.pem", "wb") as f:
+        f.write(ca_cert)
+
+    CERT_DIR = Path.home() / ".cbl_certs"
+    CERT_DIR.mkdir(exist_ok=True)
+
+    client_cert_path = CERT_DIR / "client_cert.pem"
+    client_key_path = CERT_DIR / "client_key.pem"
+    ca_cert_path = CERT_DIR / "ca_cert.pem"
+
+    with open(client_cert_path, "wb") as f:
+        f.write(client_cert)
+
+    with open(client_key_path, "wb") as f:
+        f.write(client_key)
+
+    with open(ca_cert_path, "wb") as f:
+        f.write(ca_cert)
+
     sftp_progress_bar(sftp, Path("/tmp/es_cert.pem"), "/home/ec2-user/cert/es_cert.pem")
     sftp_progress_bar(sftp, Path("/tmp/es_key.pem"), "/home/ec2-user/cert/es_key.pem")
+    sftp_progress_bar(sftp, Path("/tmp/ca_cert.pem"), "/home/ec2-user/cert/ca_cert.pem")
     sftp_progress_bar(
         sftp,
         SCRIPT_DIR / "config" / "config.json",

spec/tests/QE/edge_server/test_authentication.md

@@ -12,9 +12,11 @@ Test basic authentication with valid, invalid, and anonymous credentials.
 4. Set invalid credentials and verify fetching active tasks fails.
 5. Disable auth (anonymous) and verify fetching active tasks fails.
 
-## test_valid_tls
+## test_valid_tls_mtls
 
-Test TLS configuration for Edge Server.
+Test TLS and MTLS configurations for Edge Server.
 
 1. Configure Edge Server with the `names` dataset using TLS config.
 2. Fetch server version information and verify the call succeeds.
+3. Re-configure Edge Server with the `names` dataset using MTLS config.
+4. Fetch server version information and verify the call succeeds.

spec/tests/QE/edge_server/test_chaos_scenarios.md

@@ -33,4 +33,29 @@ Test multi-edge synchronization and recovery with chained replications.
 9. Kill Edge Server 1, update documents via Edge Servers 2/3, and verify replication.
 10. Kill Edge Servers 1 and 3, update documents via Edge Server 2, and verify replication.
 
+## test_edge_server_offline_sync_and_recovery
 
+Test the offline synchronization and recovery capabilities of a chained Edge Server topology.
+
+1. Configure Edge Server 1 to replicate from Sync Gateway for the `travel` dataset.
+2. Configure Edge Server 2 to replicate from Edge Server 1.
+3. Configure Edge Server 3 to replicate from Edge Server 2.
+4. Wait for replication to become idle across all Edge Servers.
+5. Delete all documents in the `travel.hotels` collection on Edge Server 3 using a bulk delete.
+6. Wait for replication to become idle, then kill Edge Server 3.
+7. Verify the documents are deleted in Edge Server 1, Edge Server 2, and Sync Gateway.
+8. Restart Edge Server 3 and wait for it to come online.
+9. Create 10,000 documents in `travel.hotels` via bulk create on Edge Server 3.
+10. Verify documents are created on Edge Server 3 and wait for replication to become idle.
+11. Kill Edge Server 3 again.
+12. Verify the created documents propagated successfully to Edge Server 2, Edge Server 1, and Sync Gateway.
+
+## test_edge_server_with_concurrent_rest_requests
+
+Test the stability and consistency of Edge Server under a randomized concurrent REST CRUD workload.
+
+1. Configure Edge Server with a database named `db`.
+2. Seed the database with 10,000 documents using bulk create operations.
+3. Verify the initial document load is successful and document counts match.
+4. Run a randomized CRUD workload for 1,000 iterations containing a mix of create, update, delete, and read operations.
+5. Perform final data consistency validation by verifying the documents on Edge Server match the expected state tracked locally.

spec/tests/QE/edge_server/test_crud.md

@@ -49,3 +49,15 @@ Test CRUD operations with document expiry (TTL).
 2. Create a document with TTL=20 seconds and verify retrieval fails after expiry.
 3. Create a document with TTL=50 seconds, update it with TTL=20 seconds, and verify retrieval fails after expiry.
 4. Create a document with TTL=60 seconds and delete it, then verify retrieval fails.
+
+## test_multiple_doc_crud
+
+Test bulk CRUD operations combining document creations, updates, and deletions in a single request.
+
+1. Configure Edge Server with the `names` database.
+2. Fetch existing documents to prepare a list of bulk changes.
+3. Prepare a bulk operation containing 10 document creations, 10 document updates, and 10 document deletions.
+4. Execute the bulk document operations on the Edge Server.
+5. Fetch the specifically created documents by key and verify all 10 exist.
+6. Fetch the updated documents by key and verify they exist with updated revisions.
+7. Fetch the deleted documents by key and verify they are no longer returned.

tests/QE/edge_server/config/test_mtls_config.json

@@ -0,0 +1,30 @@
+{
+    "$schema": "https://packages.couchbase.com/couchbase-edge-server/config_schema.json",
+    "interface": "0.0.0.0:59840",
+    "enable_anonymous_users": true,
+    "databases": {
+        "names":
+        {
+            "path": "/home/ec2-user/database/names.cblite2",
+            "create": true,
+            "enable_adhoc_queries": true,
+            "enable_client_writes": true,
+            "enable_client_sync": true
+        }
+    },
+  "https": {
+    "tls_cert_path": "/home/ec2-user/cert/es_cert.pem",
+    "tls_key_path": "/home/ec2-user/cert/es_key.pem",
+    "client_cert_path": "/home/ec2-user/cert/ca_cert.pem"
+  },
+  "logging": {
+        "console": false,
+        "file": {
+            "dir": "/home/ec2-user/log",
+            "format": "text"
+        },
+        "domains": {
+            "Listener": "info"
+        }
+    }
+}

tests/QE/edge_server/test_authentication.py

@@ -44,11 +44,19 @@ async def test_basic_auth(self, cblpytest: CBLPyTest) -> None:
         assert failed, "No auth did not fail as expected"
 
     @pytest.mark.asyncio(loop_scope="session")
-    async def test_valid_tls(self, cblpytest: CBLPyTest, dataset_path: Path) -> None:
+    async def test_valid_tls_mtls(
+        self, cblpytest: CBLPyTest, dataset_path: Path
+    ) -> None:
         self.mark_test_step("test_valid_tls")
         edge_server = await cblpytest.edge_servers[0].configure_dataset(
             db_name="names", config_file=f"{SCRIPT_DIR}/config/test_tls_config.json"
         )
-        self.mark_test_step("get server information")
+        self.mark_test_step("get server information with TLS")
+        version = await edge_server.get_version()
+        self.mark_test_step(f"VERSION:{version}")
+        edge_server = await cblpytest.edge_servers[0].configure_dataset(
+            db_name="names", config_file=f"{SCRIPT_DIR}/config/test_mtls_config.json"
+        )
+        self.mark_test_step("get server information with TLS")
         version = await edge_server.get_version()
         self.mark_test_step(f"VERSION:{version}")