From 5ba4f62ac951b30a4c4e005b08d62364aa307f00 Mon Sep 17 00:00:00 2001 From: barkha06 Date: Thu, 23 Apr 2026 20:18:39 +0530 Subject: [PATCH 1/5] Migrate pending edge server tests --- client/src/cbltest/api/edgeserver.py | 16 +- environment/aws/common/x509_certificate.py | 67 +++++ .../aws/es_setup/setup_edge_servers.py | 34 ++- .../edge_server/config/test_mtls_config.json | 30 +++ tests/QE/edge_server/test_authentication.py | 12 +- tests/QE/edge_server/test_chaos_scenarios.py | 251 ++++++++++++++++++ tests/QE/edge_server/test_crud.py | 61 +++++ 7 files changed, 465 insertions(+), 6 deletions(-) create mode 100644 tests/QE/edge_server/config/test_mtls_config.json diff --git a/client/src/cbltest/api/edgeserver.py b/client/src/cbltest/api/edgeserver.py index a9637c404..591ceddce 100644 --- a/client/src/cbltest/api/edgeserver.py +++ b/client/src/cbltest/api/edgeserver.py @@ -152,9 +152,15 @@ def _create_session( self, scheme: str, url: str, port: int, auth: BasicAuth | None ) -> ClientSession: if self.__secure: - ssl_context = ssl.create_default_context() + CERT_DIR = Path.home() / ".cbl_certs" + ssl_context = ssl.create_default_context(cafile=CERT_DIR / "ca_cert.pem") ssl_context.check_hostname = False - ssl_context.verify_mode = ssl.CERT_NONE + if self.__mtls: + ssl_context.load_cert_chain( + certfile=str(CERT_DIR / "client_cert.pem"), + keyfile=str(CERT_DIR / "client_key.pem"), + ) + return ClientSession( f"{scheme}{url}:{port}", auth=auth, @@ -1026,6 +1032,12 @@ async def reset_firewall(self): with self.__tracer.start_as_current_span("reset firewall"): await self._send_request("post", "firewall", session=self.__shell_session) + async def get_ca(self): + with self.__tracer.start_as_current_span("get_ca"): + return await self._send_request( + "get", "get-ca", session=self.__shell_session + ) + async def add_user(self, name, password, role="admin"): with self.__tracer.start_as_current_span("Add user"): await self.kill_server() diff --git a/environment/aws/common/x509_certificate.py b/environment/aws/common/x509_certificate.py index 9faa20de1..cc9973518 100644 --- a/environment/aws/common/x509_certificate.py +++ b/environment/aws/common/x509_certificate.py @@ -9,6 +9,7 @@ pkcs12, ) from cryptography.x509 import ( + BasicConstraints, Certificate, CertificateBuilder, ExtendedKeyUsage, @@ -79,3 +80,69 @@ def create_self_signed_certificate(CN: str) -> CertKeyPair: ) return CertKeyPair(leaf_certificate, private_key) + + +def create_ca(): + key = rsa.generate_private_key(public_exponent=65537, key_size=2048) + + subject = issuer = Name([NameAttribute(NameOID.COMMON_NAME, "EdgeTestCA")]) + + cert = ( + CertificateBuilder() + .subject_name(subject) + .issuer_name(issuer) + .public_key(key.public_key()) + .serial_number(random_serial_number()) + .not_valid_before(datetime.now(timezone.utc)) + .not_valid_after(datetime.now(timezone.utc) + timedelta(days=365)) + .add_extension(BasicConstraints(ca=True, path_length=None), critical=True) + .sign(key, hashes.SHA256()) + ) + + return CertKeyPair(cert, key) + + +def create_signed_cert(cn: str, ca: CertKeyPair): + key = rsa.generate_private_key(public_exponent=65537, key_size=2048) + + subject = Name([NameAttribute(NameOID.COMMON_NAME, cn)]) + + cert = ( + CertificateBuilder() + .subject_name(subject) + .issuer_name(ca.certificate.subject) + .public_key(key.public_key()) + .serial_number(random_serial_number()) + .not_valid_before(datetime.now(timezone.utc)) + .not_valid_after(datetime.now(timezone.utc) + timedelta(days=1)) + .add_extension( + ExtendedKeyUsage([ExtendedKeyUsageOID.SERVER_AUTH]), + critical=False, + ) + .sign(ca.private_key, hashes.SHA256()) + ) + + return CertKeyPair(cert, key) + + +def create_client_cert(cn: str, ca: CertKeyPair): + key = rsa.generate_private_key(public_exponent=65537, key_size=2048) + + subject = Name([NameAttribute(NameOID.COMMON_NAME, cn)]) + + cert = ( + CertificateBuilder() + .subject_name(subject) + .issuer_name(ca.certificate.subject) + .public_key(key.public_key()) + .serial_number(random_serial_number()) + .not_valid_before(datetime.now(timezone.utc)) + .not_valid_after(datetime.now(timezone.utc) + timedelta(days=1)) + .add_extension( + ExtendedKeyUsage([ExtendedKeyUsageOID.CLIENT_AUTH]), + critical=False, + ) + .sign(ca.private_key, hashes.SHA256()) + ) + + return CertKeyPair(cert, key) diff --git a/environment/aws/es_setup/setup_edge_servers.py b/environment/aws/es_setup/setup_edge_servers.py index 9fcfba65e..683d41250 100644 --- a/environment/aws/es_setup/setup_edge_servers.py +++ b/environment/aws/es_setup/setup_edge_servers.py @@ -35,7 +35,11 @@ from environment.aws.common.io import LIGHT_GRAY, sftp_progress_bar from environment.aws.common.output import header -from environment.aws.common.x509_certificate import create_self_signed_certificate +from environment.aws.common.x509_certificate import ( + create_ca, + create_client_cert, + create_signed_cert, +) from environment.aws.topology_setup.setup_topology import TopologyConfig SCRIPT_DIR = Path(__file__).resolve().parent @@ -257,17 +261,43 @@ def setup_server( ) sftp_progress_bar(sftp, SCRIPT_DIR / "Caddyfile", "/home/ec2-user/Caddyfile") - cert = create_self_signed_certificate(hostname) + ca = create_ca() + ca_cert = ca.pem_bytes() + cert = create_signed_cert(hostname, ca) cert_pem = cert.pem_bytes() key_pem = cert.private_pem_bytes() + client = create_client_cert("test-client", ca) + client_cert = client.pem_bytes() + client_key = client.private_pem_bytes() + with open("/tmp/es_key.pem", "wb") as f: f.write(key_pem) with open("/tmp/es_cert.pem", "wb") as f: f.write(cert_pem) + with open("/tmp/ca_cert.pem", "wb") as f: + f.write(ca_cert) + + CERT_DIR = Path.home() / ".cbl_certs" + CERT_DIR.mkdir(exist_ok=True) + + client_cert_path = CERT_DIR / "client_cert.pem" + client_key_path = CERT_DIR / "client_key.pem" + ca_cert_path = CERT_DIR / "ca_cert.pem" + + with open(client_cert_path, "wb") as f: + f.write(client_cert) + + with open(client_key_path, "wb") as f: + f.write(client_key) + + with open(ca_cert_path, "wb") as f: + f.write(ca_cert) + sftp_progress_bar(sftp, Path("/tmp/es_cert.pem"), "/home/ec2-user/cert/es_cert.pem") sftp_progress_bar(sftp, Path("/tmp/es_key.pem"), "/home/ec2-user/cert/es_key.pem") + sftp_progress_bar(sftp, Path("/tmp/ca_cert.pem"), "/home/ec2-user/cert/ca_cert.pem") sftp_progress_bar( sftp, SCRIPT_DIR / "config" / "config.json", diff --git a/tests/QE/edge_server/config/test_mtls_config.json b/tests/QE/edge_server/config/test_mtls_config.json new file mode 100644 index 000000000..3138cef49 --- /dev/null +++ b/tests/QE/edge_server/config/test_mtls_config.json @@ -0,0 +1,30 @@ +{ + "$schema": "https://packages.couchbase.com/couchbase-edge-server/config_schema.json", + "interface": "0.0.0.0:59840", + "enable_anonymous_users": true, + "databases": { + "names": + { + "path": "/home/ec2-user/database/names.cblite2", + "create": true, + "enable_adhoc_queries": true, + "enable_client_writes": true, + "enable_client_sync": true + } + }, + "https": { + "tls_cert_path": "/home/ec2-user/cert/es_cert.pem", + "tls_key_path": "/home/ec2-user/cert/es_key.pem", + "client_cert_path": "/home/ec2-user/cert/ca_cert.pem" + }, + "logging": { + "console": false, + "file": { + "dir": "/home/ec2-user/log", + "format": "text" + }, + "domains": { + "Listener": "info" + } + } +} \ No newline at end of file diff --git a/tests/QE/edge_server/test_authentication.py b/tests/QE/edge_server/test_authentication.py index b45d66e73..67b865f59 100644 --- a/tests/QE/edge_server/test_authentication.py +++ b/tests/QE/edge_server/test_authentication.py @@ -44,11 +44,19 @@ async def test_basic_auth(self, cblpytest: CBLPyTest) -> None: assert failed, "No auth did not fail as expected" @pytest.mark.asyncio(loop_scope="session") - async def test_valid_tls(self, cblpytest: CBLPyTest, dataset_path: Path) -> None: + async def test_valid_tls_mtls( + self, cblpytest: CBLPyTest, dataset_path: Path + ) -> None: self.mark_test_step("test_valid_tls") edge_server = await cblpytest.edge_servers[0].configure_dataset( db_name="names", config_file=f"{SCRIPT_DIR}/config/test_tls_config.json" ) - self.mark_test_step("get server information") + self.mark_test_step("get server information with TLS") + version = await edge_server.get_version() + self.mark_test_step(f"VERSION:{version}") + edge_server = await cblpytest.edge_servers[0].configure_dataset( + db_name="names", config_file=f"{SCRIPT_DIR}/config/test_mtls_config.json" + ) + self.mark_test_step("get server information with TLS") version = await edge_server.get_version() self.mark_test_step(f"VERSION:{version}") diff --git a/tests/QE/edge_server/test_chaos_scenarios.py b/tests/QE/edge_server/test_chaos_scenarios.py index a1bcec55c..2f9183940 100644 --- a/tests/QE/edge_server/test_chaos_scenarios.py +++ b/tests/QE/edge_server/test_chaos_scenarios.py @@ -1,6 +1,7 @@ import asyncio import json import random +import uuid from pathlib import Path import pytest @@ -267,3 +268,253 @@ async def test_3_edge_with_sync(self, cblpytest, dataset_path) -> None: edge1_docs.revmap, docs_list, ) + + @pytest.mark.asyncio(loop_scope="session") + async def test_edge_server_offline_sync_and_recovery( + self, cblpytest, dataset_path + ) -> None: + self.mark_test_step("Edge Server Offline Sync and Recovery") + cloud = cblpytest.simple_cloud() + await cloud.configure_dataset(dataset_path, "travel") + sgw = cblpytest.sync_gateways[0] + source_db = sgw.replication_url("travel") + + self.mark_test_step("Configure Edge Server with travel dataset") + config_path = f"{SCRIPT_DIR}/config/test_sgw_edge_server.json" + with open(config_path, "r") as file: + config = json.load(file) + config["replications"][0]["source"] = source_db + with open(config_path, "w") as file: + json.dump(config, file, indent=4) + edge_server1 = await cblpytest.edge_servers[0].configure_dataset( + db_name="travel", config_file=config_path + ) + + config_path2 = f"{SCRIPT_DIR}/config/test_edge_to_edge_server.json" + source_db = edge_server1.replication_url("travel") + with open(config_path2, "r") as file: + config = json.load(file) + config["replications"][0]["source"] = source_db + with open(config_path2, "w") as file: + json.dump(config, file, indent=4) + edge_server2 = await cblpytest.edge_servers[1].configure_dataset( + db_name="travel", config_file=config_path2 + ) + + source_db = edge_server2.replication_url("travel") + config["replications"][0]["source"] = source_db + with open(config_path2, "w") as file: + json.dump(config, file, indent=4) + edge_server3 = await cblpytest.edge_servers[2].configure_dataset( + db_name="travel", config_file=config_path2 + ) + + self.mark_test_step("Monitor replication progress") + await edge_server1.wait_for_idle() + await edge_server2.wait_for_idle() + await edge_server3.wait_for_idle() + + self.mark_test_step("Delete all docs in collection travel.hotels in es3") + all_docs = await edge_server3.get_all_documents( + db_name="travel", collection="travel.hotels" + ) + revmap = all_docs.revmap + bulk_ops = [ + BulkDocOperation({"_deleted": True}, _id=doc_id, rev=rev, optype="delete") + for doc_id, rev in revmap.items() + ] + await edge_server3.bulk_doc_op(bulk_ops, "travel", "travel", "hotels") + await edge_server1.wait_for_idle() + await edge_server2.wait_for_idle() + await edge_server3.wait_for_idle() + await edge_server3.kill_server() + + self.mark_test_step("Verify document deleted") + edge2_docs = await edge_server2.get_all_documents( + "travel", collection="travel.hotels" + ) + edge1_docs = await edge_server1.get_all_documents( + "travel", collection="travel.hotels" + ) + sgw_docs = await sgw.get_all_documents( + "travel", scope="travel", collection="hotels" + ) + + assert ( + len(edge1_docs.rows) == len(sgw_docs.rows) == len(edge2_docs.rows) == 0 + ), ( + f"Collection hotels count mismatch in len(edge1_docs.rows): {len(edge1_docs.rows)} , len(sgw_docs.rows): {len(sgw_docs.rows)}, len(edge2_docs.rows): {len(edge2_docs.rows)}" + ) + + self.mark_test_step("Start ES3") + await edge_server3.start_server() + await asyncio.sleep(100) + + self.mark_test_step("Test document inserts") + docgen = JSONGenerator(seed=10, size=10000) + create_docs = docgen.generate_all_documents() + bulk_ops = [ + BulkDocOperation(body=doc, _id=id, optype="create") + for id, doc in create_docs.items() + ] + await edge_server3.bulk_doc_op(bulk_ops, "travel", "travel", "hotels") + + self.mark_test_step("Verify document inserted") + all_docs = await edge_server3.get_all_documents( + db_name="travel", collection="travel.hotels" + ) + assert len(all_docs.rows) == docgen.size, "Inserted document count mismatch" + await edge_server1.wait_for_idle() + await edge_server2.wait_for_idle() + await edge_server3.wait_for_idle() + + await edge_server3.kill_server() + + self.mark_test_step("Verify create propagated to ES2, Es1, SGW") + edge2_docs = await edge_server2.get_all_documents( + "travel", collection="travel.hotels" + ) + edge1_docs = await edge_server1.get_all_documents( + "travel", collection="travel.hotels" + ) + sgw_docs = await sgw.get_all_documents( + "travel", scope="travel", collection="hotels" + ) + + assert ( + len(edge1_docs.rows) + == len(sgw_docs.rows) + == len(edge2_docs.rows) + == docgen.size + ), ( + f"Collection hotels count mismatch in len(edge1_docs.rows): {len(edge1_docs.rows)} , len(sgw_docs.rows): {len(sgw_docs.rows)}, len(edge2_docs.rows): {len(edge2_docs.rows)} and {docgen.size}" + ) + + async def perform_operation( + self, client, optype, docs_dict, docgen, db_name, scope, collection, revmap + ): + """Perform async CRUD operation based on random optype""" + doc_id = None + + # Nothing to operate on (except create) + if optype != "create" and not docs_dict: + return True + + try: + if optype == "create": + doc_id = str(uuid.uuid4()) + new_doc = docgen.generate_document(doc_id) + + response = await client.put_document_with_id( + document=new_doc, + db_name=db_name, + scope=scope, + collection=collection, + doc_id=doc_id, + ) + + if response.get("ok"): + docs_dict[doc_id] = new_doc + revmap[doc_id] = response.get("rev") + return response.get("ok", False) + + # Pick existing doc + doc_id = random.choice(list(docs_dict.keys())) + + if optype == "update" and doc_id in revmap: + updated_doc = docgen.update_document(docs_dict[doc_id], doc_id) + + response = await client.put_document_with_id( + doc_id=doc_id, + document=updated_doc, + db_name=db_name, + scope=scope, + collection=collection, + rev=revmap[doc_id], + ) + + if response.get("ok"): + docs_dict[doc_id] = updated_doc + revmap[doc_id] = response.get("rev") + return response.get("ok", False) + + if optype == "delete" and doc_id in revmap: + response = await client.delete_document( + doc_id, + revid=revmap[doc_id], + db_name=db_name, + scope=scope, + collection=collection, + ) + + if response.get("ok"): + docs_dict.pop(doc_id, None) + revmap.pop(doc_id, None) + return response.get("ok", False) + + if optype == "read": + remote_doc = await client.get_document( + db_name=db_name, scope=scope, collection=collection, doc_id=doc_id + ) + local_doc = docs_dict.get(doc_id) + if not local_doc: + return True # deleted concurrently + + for key, value in remote_doc.body.items(): + assert local_doc.get(key) == value + return True + + except Exception as e: + return {"error": str(e), "optype": optype, "doc_id": doc_id} + + @pytest.mark.asyncio(loop_scope="session") + async def test_edge_server_with_concurrent_rest_requests( + self, cblpytest, dataset_path + ) -> None: + self.mark_test_step("Edge Server with concurrent REST CRUD operations") + + edge_server = await cblpytest.edge_servers[0].configure_dataset( + db_name="db", + config_file=f"{SCRIPT_DIR}/config/test_edge_server_with_multiple_rest_clients.json", + ) + + docgen = JSONGenerator(seed=10, size=10000) + docs = docgen.generate_all_documents() + + # Seed database + bulk_ops = [ + BulkDocOperation(body=doc, _id=doc_id, optype="create") + for doc_id, doc in docs.items() + ] + await edge_server.bulk_doc_op(bulk_ops, db_name="db") + + self.mark_test_step("Verify initial document load") + all_docs = await edge_server.get_all_documents(db_name="db") + assert len(all_docs.rows) == docgen.size + + revmap = all_docs.revmap + operations = ["create", "update", "delete", "read"] + + self.mark_test_step("Run randomized CRUD workload") + for i in range(1000): + op = random.choice(operations) + self.mark_test_step(f"Iteration {i}: {op}") + assert await self.perform_operation( + edge_server, + op, + docs, + docgen, + db_name="db", + scope="", + collection="", + revmap=revmap, + ) + + self.mark_test_step("Final consistency validation") + final_docs = await edge_server.get_all_documents(db_name="db") + assert len(final_docs.rows) == len(docs), "Final document count mismatch" + + for row in final_docs.rows: + assert docs.get(row.id), ( + f"ID {row.id} exists on Edge server but not in local dict" + ) diff --git a/tests/QE/edge_server/test_crud.py b/tests/QE/edge_server/test_crud.py index c4c7d1254..fbced92c9 100644 --- a/tests/QE/edge_server/test_crud.py +++ b/tests/QE/edge_server/test_crud.py @@ -4,6 +4,7 @@ import pytest from cbltest import CBLPyTest from cbltest.api.cbltestclass import CBLTestClass +from cbltest.api.edgeserver import BulkDocOperation SCRIPT_DIR = str(Path(__file__).parent) @@ -176,3 +177,63 @@ async def test_single_doc_crud_ttl( self.mark_test_step( f"Deleted doc successfully threw exception on retrieval: {e}" ) + + @pytest.mark.asyncio(loop_scope="session") + async def test_multiple_doc_crud( + self, cblpytest: CBLPyTest, dataset_path: Path + ) -> None: + self.mark_test_step("test_multiple_doc_crud") + + edge_server = await cblpytest.edge_servers[0].configure_dataset( + db_name="names", + config_file=f"{SCRIPT_DIR}/config/adhoc_disabled_config.json", + ) + + self.mark_test_step("Prepare multiple CRUD operations") + db_name = "names" + all_docs = await edge_server.get_all_documents(db_name=db_name) + + bulk_changes = [] + created, deleted, updated = [], [], [] + + for i in range(1, 11): + new_id = f"doc_{i + 200}" + bulk_changes.append(BulkDocOperation(_id=new_id, body={"rev": 1, "idx": i})) + created.append(new_id) + + bulk_changes.append( + BulkDocOperation( + _id=all_docs.rows[i].id, + body={}, + rev=all_docs.rows[i].revid, + optype="delete", + ) + ) + deleted.append(all_docs.rows[i].id) + + bulk_changes.append( + BulkDocOperation( + _id=all_docs.rows[i + 50].id, + body={"rev": 2, "idx": i + 50}, + rev=all_docs.rows[i + 50].revid, + optype="update", + ) + ) + updated.append(all_docs.rows[i + 50].id) + + self.mark_test_step("Execute bulk document operations") + await edge_server.bulk_doc_op(docs=bulk_changes, db_name=db_name) + + self.mark_test_step("Verify created documents") + create_task = await edge_server.get_all_documents(db_name=db_name, keys=created) + assert len(create_task) == 10, "Created documents missing" + + self.mark_test_step("Verify updated documents") + update_task = await edge_server.get_all_documents(db_name=db_name, keys=updated) + assert len(update_task) == 10, "Updated documents missing" + for row in update_task.rows: + assert row.revid.startswith("2") + + self.mark_test_step("Verify deleted documents") + delete_task = await edge_server.get_all_documents(db_name=db_name, keys=deleted) + assert len(delete_task) == 0, "Deleted documents still exist" From b1b383e32ffedea87fd33fef869a9c1c34edab82 Mon Sep 17 00:00:00 2001 From: barkha06 Date: Thu, 23 Apr 2026 20:30:42 +0530 Subject: [PATCH 2/5] Added spec files for tests --- client/src/cbltest/api/edgeserver.py | 6 ----- .../QE/edge_server/test_authentication.md | 6 +++-- .../QE/edge_server/test_chaos_scenarios.md | 25 +++++++++++++++++++ spec/tests/QE/edge_server/test_crud.md | 12 +++++++++ 4 files changed, 41 insertions(+), 8 deletions(-) diff --git a/client/src/cbltest/api/edgeserver.py b/client/src/cbltest/api/edgeserver.py index 591ceddce..bfd7f4fb6 100644 --- a/client/src/cbltest/api/edgeserver.py +++ b/client/src/cbltest/api/edgeserver.py @@ -1032,12 +1032,6 @@ async def reset_firewall(self): with self.__tracer.start_as_current_span("reset firewall"): await self._send_request("post", "firewall", session=self.__shell_session) - async def get_ca(self): - with self.__tracer.start_as_current_span("get_ca"): - return await self._send_request( - "get", "get-ca", session=self.__shell_session - ) - async def add_user(self, name, password, role="admin"): with self.__tracer.start_as_current_span("Add user"): await self.kill_server() diff --git a/spec/tests/QE/edge_server/test_authentication.md b/spec/tests/QE/edge_server/test_authentication.md index 67cfd6d36..df11cd861 100644 --- a/spec/tests/QE/edge_server/test_authentication.md +++ b/spec/tests/QE/edge_server/test_authentication.md @@ -12,9 +12,11 @@ Test basic authentication with valid, invalid, and anonymous credentials. 4. Set invalid credentials and verify fetching active tasks fails. 5. Disable auth (anonymous) and verify fetching active tasks fails. -## test_valid_tls +## test_valid_tls_mtls -Test TLS configuration for Edge Server. +Test TLS and MTLS configurations for Edge Server. 1. Configure Edge Server with the `names` dataset using TLS config. 2. Fetch server version information and verify the call succeeds. +3. Re-configure Edge Server with the `names` dataset using MTLS config. +4. Fetch server version information and verify the call succeeds. diff --git a/spec/tests/QE/edge_server/test_chaos_scenarios.md b/spec/tests/QE/edge_server/test_chaos_scenarios.md index 3b1d139b2..0e1b78626 100644 --- a/spec/tests/QE/edge_server/test_chaos_scenarios.md +++ b/spec/tests/QE/edge_server/test_chaos_scenarios.md @@ -33,4 +33,29 @@ Test multi-edge synchronization and recovery with chained replications. 9. Kill Edge Server 1, update documents via Edge Servers 2/3, and verify replication. 10. Kill Edge Servers 1 and 3, update documents via Edge Server 2, and verify replication. +## test_edge_server_offline_sync_and_recovery +Test the offline synchronization and recovery capabilities of a chained Edge Server topology. + +1. Configure Edge Server 1 to replicate from Sync Gateway for the `travel` dataset. +2. Configure Edge Server 2 to replicate from Edge Server 1. +3. Configure Edge Server 3 to replicate from Edge Server 2. +4. Wait for replication to become idle across all Edge Servers. +5. Delete all documents in the `travel.hotels` collection on Edge Server 3 using a bulk delete. +6. Wait for replication to become idle, then kill Edge Server 3. +7. Verify the documents are deleted in Edge Server 1, Edge Server 2, and Sync Gateway. +8. Restart Edge Server 3 and wait for it to come online. +9. Create 10,000 documents in `travel.hotels` via bulk create on Edge Server 3. +10. Verify documents are created on Edge Server 3 and wait for replication to become idle. +11. Kill Edge Server 3 again. +12. Verify the created documents propagated successfully to Edge Server 2, Edge Server 1, and Sync Gateway. + +## test_edge_server_with_concurrent_rest_requests + +Test the stability and consistency of Edge Server under a randomized concurrent REST CRUD workload. + +1. Configure Edge Server with a database named `db`. +2. Seed the database with 10,000 documents using bulk create operations. +3. Verify the initial document load is successful and document counts match. +4. Run a randomized CRUD workload for 1,000 iterations containing a mix of create, update, delete, and read operations. +5. Perform final data consistency validation by verifying the documents on Edge Server match the expected state tracked locally. \ No newline at end of file diff --git a/spec/tests/QE/edge_server/test_crud.md b/spec/tests/QE/edge_server/test_crud.md index b6543faa0..9ffafec76 100644 --- a/spec/tests/QE/edge_server/test_crud.md +++ b/spec/tests/QE/edge_server/test_crud.md @@ -49,3 +49,15 @@ Test CRUD operations with document expiry (TTL). 2. Create a document with TTL=20 seconds and verify retrieval fails after expiry. 3. Create a document with TTL=50 seconds, update it with TTL=20 seconds, and verify retrieval fails after expiry. 4. Create a document with TTL=60 seconds and delete it, then verify retrieval fails. + +## test_multiple_doc_crud + +Test bulk CRUD operations combining document creations, updates, and deletions in a single request. + +1. Configure Edge Server with the `names` database. +2. Fetch existing documents to prepare a list of bulk changes. +3. Prepare a bulk operation containing 10 document creations, 10 document updates, and 10 document deletions. +4. Execute the bulk document operations on the Edge Server. +5. Fetch the specifically created documents by key and verify all 10 exist. +6. Fetch the updated documents by key and verify they exist with updated revisions. +7. Fetch the deleted documents by key and verify they are no longer returned. \ No newline at end of file From 37238c412f996d6eda7031efe75d2083492cce30 Mon Sep 17 00:00:00 2001 From: barkha06 Date: Thu, 23 Apr 2026 20:38:07 +0530 Subject: [PATCH 3/5] Modified certificate property --- environment/aws/common/x509_certificate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/environment/aws/common/x509_certificate.py b/environment/aws/common/x509_certificate.py index cc9973518..21d916076 100644 --- a/environment/aws/common/x509_certificate.py +++ b/environment/aws/common/x509_certificate.py @@ -114,7 +114,7 @@ def create_signed_cert(cn: str, ca: CertKeyPair): .public_key(key.public_key()) .serial_number(random_serial_number()) .not_valid_before(datetime.now(timezone.utc)) - .not_valid_after(datetime.now(timezone.utc) + timedelta(days=1)) + .not_valid_after(datetime.now(timezone.utc) + timedelta(days=365)) .add_extension( ExtendedKeyUsage([ExtendedKeyUsageOID.SERVER_AUTH]), critical=False, @@ -137,7 +137,7 @@ def create_client_cert(cn: str, ca: CertKeyPair): .public_key(key.public_key()) .serial_number(random_serial_number()) .not_valid_before(datetime.now(timezone.utc)) - .not_valid_after(datetime.now(timezone.utc) + timedelta(days=1)) + .not_valid_after(datetime.now(timezone.utc) + timedelta(days=365)) .add_extension( ExtendedKeyUsage([ExtendedKeyUsageOID.CLIENT_AUTH]), critical=False, From 61e9ebaabde8664cd4db79494608f924867823ac Mon Sep 17 00:00:00 2001 From: barkha06 Date: Fri, 24 Apr 2026 11:54:56 +0530 Subject: [PATCH 4/5] Modifed test steps to be more descriptive and match spec --- tests/QE/edge_server/test_chaos_scenarios.py | 22 ++++++++++++-------- tests/QE/edge_server/test_crud.py | 9 ++++---- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/tests/QE/edge_server/test_chaos_scenarios.py b/tests/QE/edge_server/test_chaos_scenarios.py index 2f9183940..727f0b6fe 100644 --- a/tests/QE/edge_server/test_chaos_scenarios.py +++ b/tests/QE/edge_server/test_chaos_scenarios.py @@ -279,7 +279,7 @@ async def test_edge_server_offline_sync_and_recovery( sgw = cblpytest.sync_gateways[0] source_db = sgw.replication_url("travel") - self.mark_test_step("Configure Edge Server with travel dataset") + self.mark_test_step("Configure Edge Server 1 to replicate from Sync Gateway for the `travel` dataset") config_path = f"{SCRIPT_DIR}/config/test_sgw_edge_server.json" with open(config_path, "r") as file: config = json.load(file) @@ -289,7 +289,7 @@ async def test_edge_server_offline_sync_and_recovery( edge_server1 = await cblpytest.edge_servers[0].configure_dataset( db_name="travel", config_file=config_path ) - + self.mark_test_step("Configure Edge Server 2 to replicate from Edge Server 1") config_path2 = f"{SCRIPT_DIR}/config/test_edge_to_edge_server.json" source_db = edge_server1.replication_url("travel") with open(config_path2, "r") as file: @@ -300,7 +300,7 @@ async def test_edge_server_offline_sync_and_recovery( edge_server2 = await cblpytest.edge_servers[1].configure_dataset( db_name="travel", config_file=config_path2 ) - + self.mark_test_step("Configure Edge Server 3 to replicate from Edge Server 2") source_db = edge_server2.replication_url("travel") config["replications"][0]["source"] = source_db with open(config_path2, "w") as file: @@ -309,7 +309,7 @@ async def test_edge_server_offline_sync_and_recovery( db_name="travel", config_file=config_path2 ) - self.mark_test_step("Monitor replication progress") + self.mark_test_step("Wait for replication to become idle across all Edge Servers") await edge_server1.wait_for_idle() await edge_server2.wait_for_idle() await edge_server3.wait_for_idle() @@ -324,12 +324,14 @@ async def test_edge_server_offline_sync_and_recovery( for doc_id, rev in revmap.items() ] await edge_server3.bulk_doc_op(bulk_ops, "travel", "travel", "hotels") + self.mark_test_step("Wait for replication to become idle and validate deletes") await edge_server1.wait_for_idle() await edge_server2.wait_for_idle() await edge_server3.wait_for_idle() + self.mark_test_step("Kill Edge Server 3") await edge_server3.kill_server() - self.mark_test_step("Verify document deleted") + self.mark_test_step("Verify the documents are deleted in Edge Server 1, Edge Server 2, and Sync Gateway") edge2_docs = await edge_server2.get_all_documents( "travel", collection="travel.hotels" ) @@ -346,11 +348,11 @@ async def test_edge_server_offline_sync_and_recovery( f"Collection hotels count mismatch in len(edge1_docs.rows): {len(edge1_docs.rows)} , len(sgw_docs.rows): {len(sgw_docs.rows)}, len(edge2_docs.rows): {len(edge2_docs.rows)}" ) - self.mark_test_step("Start ES3") + self.mark_test_step("Restart Edge Server 3 and wait for it to come online") await edge_server3.start_server() await asyncio.sleep(100) - self.mark_test_step("Test document inserts") + self.mark_test_step("Create 10,000 documents in `travel.hotels` on Edge Server 3") docgen = JSONGenerator(seed=10, size=10000) create_docs = docgen.generate_all_documents() bulk_ops = [ @@ -359,7 +361,7 @@ async def test_edge_server_offline_sync_and_recovery( ] await edge_server3.bulk_doc_op(bulk_ops, "travel", "travel", "hotels") - self.mark_test_step("Verify document inserted") + self.mark_test_step("Verify documents are created on Edge Server 3 and wait for replication to become idle") all_docs = await edge_server3.get_all_documents( db_name="travel", collection="travel.hotels" ) @@ -368,9 +370,10 @@ async def test_edge_server_offline_sync_and_recovery( await edge_server2.wait_for_idle() await edge_server3.wait_for_idle() + self.mark_test_step("Kill Edge Server 3 again") await edge_server3.kill_server() - self.mark_test_step("Verify create propagated to ES2, Es1, SGW") + self.mark_test_step("Verify create propagated to ES2, ES1, SGW") edge2_docs = await edge_server2.get_all_documents( "travel", collection="travel.hotels" ) @@ -477,6 +480,7 @@ async def test_edge_server_with_concurrent_rest_requests( db_name="db", config_file=f"{SCRIPT_DIR}/config/test_edge_server_with_multiple_rest_clients.json", ) + self.mark_test_step("Seed the database with 10,000 documents") docgen = JSONGenerator(seed=10, size=10000) docs = docgen.generate_all_documents() diff --git a/tests/QE/edge_server/test_crud.py b/tests/QE/edge_server/test_crud.py index fbced92c9..4b4896c4a 100644 --- a/tests/QE/edge_server/test_crud.py +++ b/tests/QE/edge_server/test_crud.py @@ -189,12 +189,13 @@ async def test_multiple_doc_crud( config_file=f"{SCRIPT_DIR}/config/adhoc_disabled_config.json", ) - self.mark_test_step("Prepare multiple CRUD operations") + self.mark_test_step("Fetch existing documents to prepare a list of bulk changes") db_name = "names" all_docs = await edge_server.get_all_documents(db_name=db_name) bulk_changes = [] created, deleted, updated = [], [], [] + self.mark_test_step("Prepare a bulk CRUD operation") for i in range(1, 11): new_id = f"doc_{i + 200}" @@ -224,16 +225,16 @@ async def test_multiple_doc_crud( self.mark_test_step("Execute bulk document operations") await edge_server.bulk_doc_op(docs=bulk_changes, db_name=db_name) - self.mark_test_step("Verify created documents") + self.mark_test_step("Validate the specifically created documents") create_task = await edge_server.get_all_documents(db_name=db_name, keys=created) assert len(create_task) == 10, "Created documents missing" - self.mark_test_step("Verify updated documents") + self.mark_test_step("Validate the specifically updated documents") update_task = await edge_server.get_all_documents(db_name=db_name, keys=updated) assert len(update_task) == 10, "Updated documents missing" for row in update_task.rows: assert row.revid.startswith("2") - self.mark_test_step("Verify deleted documents") + self.mark_test_step("Validate the specifically deleted documents") delete_task = await edge_server.get_all_documents(db_name=db_name, keys=deleted) assert len(delete_task) == 0, "Deleted documents still exist" From 54d767032c07932f56a44dd2db416c364fff842b Mon Sep 17 00:00:00 2001 From: barkha06 Date: Fri, 24 Apr 2026 11:56:33 +0530 Subject: [PATCH 5/5] Fixed formatting errors --- tests/QE/edge_server/test_chaos_scenarios.py | 20 +++++++++++++++----- tests/QE/edge_server/test_crud.py | 4 +++- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/tests/QE/edge_server/test_chaos_scenarios.py b/tests/QE/edge_server/test_chaos_scenarios.py index 727f0b6fe..3dc3e1436 100644 --- a/tests/QE/edge_server/test_chaos_scenarios.py +++ b/tests/QE/edge_server/test_chaos_scenarios.py @@ -279,7 +279,9 @@ async def test_edge_server_offline_sync_and_recovery( sgw = cblpytest.sync_gateways[0] source_db = sgw.replication_url("travel") - self.mark_test_step("Configure Edge Server 1 to replicate from Sync Gateway for the `travel` dataset") + self.mark_test_step( + "Configure Edge Server 1 to replicate from Sync Gateway for the `travel` dataset" + ) config_path = f"{SCRIPT_DIR}/config/test_sgw_edge_server.json" with open(config_path, "r") as file: config = json.load(file) @@ -309,7 +311,9 @@ async def test_edge_server_offline_sync_and_recovery( db_name="travel", config_file=config_path2 ) - self.mark_test_step("Wait for replication to become idle across all Edge Servers") + self.mark_test_step( + "Wait for replication to become idle across all Edge Servers" + ) await edge_server1.wait_for_idle() await edge_server2.wait_for_idle() await edge_server3.wait_for_idle() @@ -331,7 +335,9 @@ async def test_edge_server_offline_sync_and_recovery( self.mark_test_step("Kill Edge Server 3") await edge_server3.kill_server() - self.mark_test_step("Verify the documents are deleted in Edge Server 1, Edge Server 2, and Sync Gateway") + self.mark_test_step( + "Verify the documents are deleted in Edge Server 1, Edge Server 2, and Sync Gateway" + ) edge2_docs = await edge_server2.get_all_documents( "travel", collection="travel.hotels" ) @@ -352,7 +358,9 @@ async def test_edge_server_offline_sync_and_recovery( await edge_server3.start_server() await asyncio.sleep(100) - self.mark_test_step("Create 10,000 documents in `travel.hotels` on Edge Server 3") + self.mark_test_step( + "Create 10,000 documents in `travel.hotels` on Edge Server 3" + ) docgen = JSONGenerator(seed=10, size=10000) create_docs = docgen.generate_all_documents() bulk_ops = [ @@ -361,7 +369,9 @@ async def test_edge_server_offline_sync_and_recovery( ] await edge_server3.bulk_doc_op(bulk_ops, "travel", "travel", "hotels") - self.mark_test_step("Verify documents are created on Edge Server 3 and wait for replication to become idle") + self.mark_test_step( + "Verify documents are created on Edge Server 3 and wait for replication to become idle" + ) all_docs = await edge_server3.get_all_documents( db_name="travel", collection="travel.hotels" ) diff --git a/tests/QE/edge_server/test_crud.py b/tests/QE/edge_server/test_crud.py index 4b4896c4a..9887bcb73 100644 --- a/tests/QE/edge_server/test_crud.py +++ b/tests/QE/edge_server/test_crud.py @@ -189,7 +189,9 @@ async def test_multiple_doc_crud( config_file=f"{SCRIPT_DIR}/config/adhoc_disabled_config.json", ) - self.mark_test_step("Fetch existing documents to prepare a list of bulk changes") + self.mark_test_step( + "Fetch existing documents to prepare a list of bulk changes" + ) db_name = "names" all_docs = await edge_server.get_all_documents(db_name=db_name)