add XL blob + short_expiry dataset, for QE tests

vipbhardwaj · vipbhardwaj · commit 54a6022ce432 · 2025-05-23T18:05:49.000+05:30
diff --git a/client/src/cbltest/api/syncgateway.py b/client/src/cbltest/api/syncgateway.py
@@ -427,6 +427,12 @@ def replication_url(self, db_name: str):
         _assert_not_null(db_name, "db_name")
         return urljoin(self.__replication_url, db_name)
 
+    def get_expvars(self):
+        """
+        Gets the expvars for the Sync Gateway
+        """
+        return urljoin(self.__admin_url, "_expvar")
+
     async def _put_database(
         self, db_name: str, payload: PutDatabasePayload, retry_count: int = 0
     ) -> None:
@@ -740,6 +746,54 @@ async def update_documents(
                 JSONDictionary(body),
             )
 
+    async def upsert_documents(
+        self,
+        db_name: str,
+        updates: list[DocumentUpdateEntry],
+        scope: str = "_default",
+        collection: str = "_default",
+    ) -> None:
+        """
+        Upserts a list of documents on Sync Gateway.
+        Its different from update_documents in that it will not overwrite the doc body in case the
+            doc already exists.
+        It will preserve the existing body fields and only add / update whatever is being passed,
+            like the behaviour shown by the function batch_upsert used in CBL updates.
+
+        :param db_name: The name of the DB endpoint to upsert
+        :param updates: A list of upserts to perform
+        :param scope: The scope that the upserts will be applied to (default '_default')
+        :param collection: The collection that the upserts will be applied to (default '_default')
+        """
+        with self.__tracer.start_as_current_span(
+            "update_documents",
+            attributes={
+                "cbl.database.name": db_name,
+                "cbl.scope.name": scope,
+                "cbl.collection.name": collection,
+            },
+        ):
+            merged_updates = []
+            for update in updates:
+                try:
+                    current_doc = await self.get_document(db_name, update.id, scope, collection)
+                    current_body = dict(current_doc.body)
+                    current_body.update(update.to_json())
+                    current_body["_id"] = update.id
+                    if update.rev:
+                        current_body["_rev"] = update.rev
+                except Exception:
+                    current_body = update.to_json()
+                merged_updates.append(DocumentUpdateEntry(update.id, update.rev, current_body))
+
+            await self._rewrite_rev_ids(db_name, merged_updates, scope, collection)
+            body = {"docs": list(u.to_json() for u in merged_updates)}
+            await self._send_request(
+                "post",
+                f"/{db_name}.{scope}.{collection}/_bulk_docs",
+                JSONDictionary(body),
+            )
+
     @deprecated("Only should be used until 4.0 SGW gets close to GA")
     async def _replaced_revid(
         self, doc_id: str, revid: str, db_name: str, scope: str, collection: str
diff --git a/dataset/server/blobs/xl1.jpg b/dataset/server/blobs/xl1.jpg
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:922b6983f5aa5f4ac26306989fa6790985197208f8cdd865c0acb7a440c4c8e2
+size 21200000
diff --git a/dataset/sg/short_expiry-sg-config.json b/dataset/sg/short_expiry-sg-config.json
@@ -0,0 +1,42 @@
+{
+  "config": {
+    "bucket": "posts",
+    "old_rev_expiry_seconds": 60,
+    "scopes": {
+      "_default": {
+        "collections": {
+          "_default": {
+            "sync": "function foo(doc,oldDoc,meta){if(doc._deleted){channel(oldDoc.channels)}else{channel(doc.channels)}}"
+          }
+        }
+      }
+    },
+    "num_index_replicas": 0
+  },
+  "config_options": {
+    "delta_sync": {
+      "delta_sync": {
+        "enabled": true,
+        "rev_max_age_seconds": 60
+      }
+    },
+    "delta_sync_with_expiry": {
+      "delta_sync": {
+        "enabled": true,
+        "rev_max_age_seconds": 60
+      }
+    }
+  },
+  "users": {
+    "user1": {
+      "password": "pass",
+      "collection_access": {
+        "_default": {
+          "_default": {
+            "admin_channels": ["*"]
+          }
+        }
+      }
+    }
+  }
+} 
diff --git a/dataset/sg/short_expiry-sg.json b/dataset/sg/short_expiry-sg.json
@@ -0,0 +1 @@
+{"_id": "doc1", "scope": "_default", "collection": "_default", "channels": ["*"], "type": "test", "value": "This is a test document for short_expiry dataset."} 
diff --git a/jenkins/pipelines/QE/ios/test.sh b/jenkins/pipelines/QE/ios/test.sh
@@ -32,6 +32,6 @@ pushd "${QE_TESTS_DIR}" > /dev/null
 create_venv venv
 . venv/bin/activate
 pip install -r requirements.txt
-pytest -v --no-header -W ignore::DeprecationWarning --config config.json test_delta_sync.py::TestDeltaSync::test_delta_sync_utf8_strings
+pytest -v --no-header -W ignore::DeprecationWarning --config config.json test_replication_eventing.py
 deactivate
 popd > /dev/null
diff --git a/spec/dataset/blobs.md b/spec/dataset/blobs.md
@@ -15,6 +15,12 @@
 | l9.jpg   | image/jpeg   | sha1-sKmH8rJtV3JardoWTE7nJ1yk0gE= | 2838244 |
 | l10.jpg  | image/jpeg   | sha1-TSC2gaaEZbi0kC3zBwdzc7wIOcc= | 1986238 |
 
+## Extra Large Size
+
+| name     | content_type | digest                                      | length    |
+| :------- | ------------ | ------------------------------------------- | --------- |
+| xl1.jpg  | image/jpeg   | sha1-35626dfbdafb966833ee60121cb7edcf9884bd3f | 21200000  |
+
 ## Small Size
 
 | name     | content_type | digest                            | length  | 
diff --git a/spec/dataset/dataset.md b/spec/dataset/dataset.md
@@ -547,4 +547,63 @@ curl --location --request POST 'http://<sg-address>:4985/todo/_role/' \
     "owner" : "user1"  
   }
 }
+```
+
+## 5. short_expiry
+
+### CBL Dataset
+
+| Collections         | #Docs       |
+| :------------------ | ----------- |
+| _default._default   | 1           | doc1         |
+
+### SG Dataset
+
+| Collections         | #Docs       | Doc ID           |
+| :------------------ | ----------- | -----------------|
+| _default._default   | 1           | doc1             |
+
+### SG Config
+
+ | Config      | Value             |
+ | ----------- | ------------------|
+ | Database    | short_expiry      |
+ | Port        | 4984 / 4985       |
+ | Collections | _default._default |
+
+#### Sync Function
+
+```js
+function (doc, oldDoc, meta) {
+  if (doc._deleted) {
+    channel(oldDoc.channels)
+  } else {
+    channel(doc.channels)
+  }
+}
+```
+
+#### Options
+
+ | Name              | Description           |
+ | ----------------- | --------------------- |
+ | delta_sync        | Enabled, 3 min expiry |
+
+#### Users
+
+| Username | Password  | admin_channels |
+| :------- | --------- |----------------|
+| user1    | pass      | ["*"]          |
+
+### Sample Docs
+
+```JSON
+{
+  "_id": "doc1",
+  "scope": "_default",
+  "collection": "_default",
+  "channels": ["*"],
+  "type": "test",
+  "value": "This is a test document for short_expiry dataset."
+}
 ```
diff --git a/spec/tests/QE/test_delta_sync.md b/spec/tests/QE/test_delta_sync.md
@@ -15,6 +15,12 @@ Verify push/pull replication works with large data and delta sync. Ensures that
 7. Start push-pull replication and verify only the updated documents are processed due to delta sync.
 8. Check delta sync stats for number of docs updated.
 
+### Key Assertions
+- The number of processed documents after modification matches the number of changed documents.
+- No unnecessary documents are replicated.
+- The document IDs of updated documents match the expected set.
+- The document count in the database remains consistent before and after delta sync replication.
+
 ## #2 test_delta_sync_nested_doc
 
 ### Description
@@ -30,22 +36,97 @@ Verify delta sync works with nested documents. Ensures that only changed nested
 7. Start push-pull replication and verify only the updated nested document is processed due to delta sync.
 8. Check delta sync stats for number of docs updated.
 
-## Purpose
-This test verifies delta sync functionality in Couchbase Lite and Sync Gateway. It ensures that only changed documents are processed and replicated, and that delta sync bandwidth savings are realized.
-
-## Test Steps
-1. **Setup**: Reset Sync Gateway and load the required dataset with delta sync enabled.
-2. **Database Initialization**: Reset the local database and load the dataset.
-3. **Initial Replication**: Start a replicator and perform initial replication to ensure all documents are present.
-4. **Document Modification**: Modify a subset of documents in Couchbase Lite.
-5. **Delta Sync Replication**: Start replication again and verify that only the modified documents are processed due to delta sync.
-6. **Assertions**:
-   - Only the changed documents are processed during the second replication.
-   - The total document count remains unchanged.
-   - Delta sync is enabled and functioning as expected.
-
-## Key Assertions
+### Key Assertions
 - The number of processed documents after modification matches the number of changed documents.
 - No unnecessary documents are replicated.
 - The document IDs of updated documents match the expected set.
-- The document count in the database remains consistent before and after delta sync replication. 
+- The document count in the database remains consistent before and after delta sync replication.
+
+## #3 test_delta_sync_utf8_strings
+
+### Description
+Verify delta sync works with documents containing large UTF-8 strings. Ensures that delta sync can handle multi-byte and non-ASCII data efficiently.
+
+### Steps
+1. Reset Sync Gateway and load the `travel` dataset with delta sync enabled.
+2. Reset the local database and load the `travel` dataset.
+3. Start a replicator and perform initial replication.
+4. Create a document in Couchbase Lite with a large UTF-8 string field.
+5. Push the document to Sync Gateway.
+6. Update the document in Sync Gateway with a new UTF-8 string value.
+7. Pull the changes to Couchbase Lite.
+8. Verify that only the delta is transferred (bytes transferred < doc size).
+
+### Key Assertions
+- The document is replicated successfully with UTF-8 content.
+- The bytes transferred for the delta are less than the full document size.
+
+## #4 test_delta_sync_enabled_disabled
+
+### Description
+Verify delta sync behavior when toggling delta sync enabled/disabled. Ensures that full documents are transferred when delta sync is disabled and deltas are used when enabled.
+
+### Steps
+1. Reset Sync Gateway and load the `travel` dataset with delta sync enabled.
+2. Reset the local database and load the `travel` dataset.
+3. Create and update documents in Couchbase Lite.
+4. Replicate and record bytes transferred (delta sync enabled).
+5. Reset Sync Gateway and load the `posts` dataset with delta sync disabled.
+6. Reset the local database and load the `posts` dataset.
+7. Create and update documents in Couchbase Lite.
+8. Replicate and record bytes transferred (delta sync disabled).
+9. Compare bytes transferred in both cases.
+
+### Key Assertions
+- When delta sync is enabled, bytes transferred for updates are less than the full doc size.
+- When delta sync is disabled, bytes transferred are close to the full doc size.
+
+## #5 test_delta_sync_within_expiry
+
+### Description
+Verify delta sync behavior within and after the delta revision expiry window. Ensures that deltas are used before expiry and full docs are sent after expiry.
+
+### Steps
+1. Reset Sync Gateway and load a dataset with short delta sync expiry.
+2. Create and update documents in Couchbase Lite.
+3. Replicate to Sync Gateway and record bytes transferred.
+4. Update the document in Sync Gateway.
+5. Wait for the delta revision to expire.
+6. Replicate back to Couchbase Lite and record bytes transferred.
+7. Compare bytes transferred before and after expiry.
+
+### Key Assertions
+- Before expiry, delta sync is used (bytes transferred < doc size).
+- After expiry, a full document is transferred (bytes transferred ≈ doc size).
+
+## #6 test_delta_sync_with_no_deltas
+
+### Description
+Test the case where an update does not produce any changes (empty delta). Ensures that no unnecessary data is transferred and document content remains consistent.
+
+### Steps
+1. Create new documents in Couchbase Lite and/or Sync Gateway.
+2. Replicate documents between CBL and SGW.
+3. Update a document with the same value as the previous revision (no actual change).
+4. Replicate again and verify no unnecessary data is transferred.
+5. Ensure document content matches between CBL and SGW.
+
+### Key Assertions
+- No unnecessary data is transferred for empty delta updates.
+- Document content is consistent between CBL and SGW after all updates.
+
+## #7 test_delta_sync_larger_than_doc
+
+### Description
+Verify delta sync behavior when the delta is larger than the document itself. Ensures that Sync Gateway falls back to sending the full document in such cases.
+
+### Steps
+1. Reset Sync Gateway and load the `travel` dataset with delta sync enabled.
+2. Create and replicate a document in Couchbase Lite.
+3. Update the document in Sync Gateway with a very large change (delta > doc size).
+4. Replicate the document to Couchbase Lite.
+5. Record and compare bytes transferred.
+
+### Key Assertions
+- When the delta is larger than the document, a full document is transferred.
+- The bytes transferred are close to the full document size, not the delta size. 
diff --git a/tests/QE/test_delta_sync.py b/tests/QE/test_delta_sync.py
diff --git a/tests/QE/test_no_conflicts.py b/tests/QE/test_no_conflicts.py
diff --git a/tests/QE/test_replication_eventing.py b/tests/QE/test_replication_eventing.py

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+{"_id": "doc1", "scope": "_default", "collection": "_default", "channels": ["*"], "type": "test", "value": "This is a test document for short_expiry dataset."}`