Merge pull request #267 from launchdarkly-labs/cko_2025

Cko 2025

Author: ahmedhqadri

Diff for: .github/workflows/demo_provisioning_scripts/DemoBuilder.py

@@ -117,6 +117,8 @@ def create_metric_groups(self):
         print("Creating metric groups:")
         print("  - Store Purchases Metric Group")
         self.metgroup_store_purchases()
+        print("  - Shorten Collection Page Metric Group")
+        self.metgroup_shorten_collection_page()
         print("Done")
         self.metric_groups_created = True
 
@@ -142,14 +144,14 @@ def create_flags(self):
         self.flag_api_guarded_release()
         #print("  - C1 - Experiment: AI Models for Chatbot")
         #self.flag_exp_chatbot_ai_models()
+        print("  - D1 - Feature Experiment: Suggested Items Carousel")
+        self.flag_exp_suggestions_carousel()
         print("  - D1 - Funnel Experiment: Promotion Banner")
         self.flag_exp_promotion_banner()
-        print("  - D2 - Feature Experiment: Suggested Items Carousel")
-        self.flag_exp_suggestions_carousel()
-        print("  - D3 - Funnel Experiment: New Shorten Collection Page")
-        self.flag_exp_shorten_collections_page()
-        print("  - D4 - Feature Experiment: New Search Engine")
+        print("  - D3 - Feature Experiment: New Search Engine")
         self.flag_exp_new_search_engine()
+        print("  - D4 - Funnel Experiment: New Shorten Collection Page")
+        self.flag_exp_shorten_collections_page()
         print("  - E1 - Migration: Database (Migration Tool)")
         self.flag_database_migration()
 
@@ -358,7 +360,7 @@ def run_ecommerce_shorten_collection_funnel_experiment(self):
 
     def create_ecommerce_shorten_collection_funnel_experiment(self):
         metrics = [
-            self.ldproject.exp_metric("shorten-collection-page-increase-conversation-metric-group", True), #TODO: need to get metric key for metric group for shorten collection
+            self.ldproject.exp_metric("shorten-collection-page-metric-group", True), 
             self.ldproject.exp_metric("in-cart-total-price", False)
         ]
         res = self.ldproject.create_experiment(
@@ -444,13 +446,29 @@ def create_togglebank_ai_config_experiment(self):
     ##################################################
     # Create all the experiment holdouts   
 
-    # def create_and_run_experiments(self):
-    #     self.run_ecommerce_collection_banner_funnel_experiment()
-    #     self.run_ecommerce_upsell_component_feature_experiment()
-    #     self.run_ecommerce_shorten_collection_funnel_experiment()
-    #     self.run_ecommerce_new_search_engine_feature_experiment()
-    #     self.run_togglebank_ai_config_experiment()
-
+    def create_and_run_holdout(self):
+        self.run_q4_increase_incart_price_holdout()
+     
+    def run_q4_increase_incart_price_holdout(self):
+        metrics = [
+                {
+                "key": "in-cart-total-price",
+                "isGroup": False,
+                "primary": True
+                }
+            ]
+        res = self.ldproject.create_holdout(
+            self=self,
+            holdout_key= "q-4-increase-average-total-in-cart-price",
+            holdout_name="Q4 Increase Average Total Incart Price",
+            holdout_env_key="production",
+            description="This holdout is to see if the new experiments will increase average total cart price and overall revenue.",
+            metrics= metrics,
+            primary_metric_key= "in-cart-total-price",
+            randomization_unit="users",
+            attributes=["tier"],
+            prerequisiteflagkey="release-new-search-engine"
+        )
 ############################################################################################################
 
     ##################################################
@@ -461,7 +479,7 @@ def create_togglebank_ai_config_experiment(self):
     ##################################################
     # Create all the experiment layers 
 
-    # def create_and_run_experiments(self):
+    # def create_and_run_layer(self):
     #     self.run_ecommerce_collection_banner_funnel_experiment()
     #     self.run_ecommerce_upsell_component_feature_experiment()
     #     self.run_ecommerce_shorten_collection_funnel_experiment()
@@ -754,6 +772,18 @@ def metric_store_checkout_completed(self):
             success_criteria="HigherThanBaseline",
             tags=["experiment"]
         )
+
+    def metric_search_engine(self):
+        res = self.ldproject.create_metric(
+            "search-engine-add-to-cart",
+            "New Search Engine Add To Cart is Clicked",
+            "search-engine-add-to-cart",
+            "This metric will track the number of times the new add to cart button within the new search engine is clicked.",
+            numeric=False,
+            unit="",
+            success_criteria="HigherThanBaseline",
+            tags=["experiment"]
+        )
 
 ############################################################################################################
 
@@ -989,6 +1019,19 @@ def metgroup_store_purchases(self):
             description="This metric group will track the store purchases",
         )
 
+    def metgroup_shorten_collection_page(self):
+        res = self.ldproject.create_metric_group(
+            "shorten-collection-page-metric-group",
+            "Shorten Collection Page Metric Group",
+            [
+                {"key": "item-added", "nameInGroup": "1"},
+                {"key": "cart-accessed", "nameInGroup": "2"},
+                {"key": "customer-checkout", "nameInGroup": "3"},
+            ],
+            kind="funnel",
+            description="This metric group will track the store purchases relating to the new shorten collection page.",
+        )
+           
 
 ############################################################################################################
 
@@ -1169,7 +1212,7 @@ def flag_exp_chatbot_ai_models(self):
     def flag_exp_promotion_banner(self):
         res = self.ldproject.create_flag(
             "storeAttentionCallout",
-            "D1 - Funnel Experiment: Promotion Banner",
+            "D2 - Funnel Experiment: Promotion Banner",
             "Releasing New Collection Promotion Banner for the Galaxy Marketplace",
             [
                 {
@@ -1193,7 +1236,7 @@ def flag_exp_promotion_banner(self):
     def flag_exp_suggestions_carousel(self):
         res = self.ldproject.create_flag(
             "cartSuggestedItems",
-            "D2 - Feature Experiment: Suggested Items Carousel",
+            "D1 - Feature Experiment: Suggested Items Carousel",
             "Releasing New Suggested Items Carousel Component for the cart component in Galaxy Marketplace",
             [
                 {

Diff for: .github/workflows/demo_provisioning_scripts/LDPlatform.py

@@ -641,6 +641,51 @@ def create_experiment(
     ##################################################
     # Create a holdout
     ##################################################
+    def create_holdout(
+        self,
+        holdout_key,
+        holdout_name,
+        holdout_env_key,   
+        description,
+        metrics,
+        primary_metric_key,
+        randomization_unit="users",
+        attributes=None,
+        prerequisiteflagkey=""
+    ):
+
+        payload = {
+            "name": holdout_name,
+            "key": holdout_key,
+            "description": description,
+            "randomizationunit": randomization_unit,
+            "attributes": attributes,
+            "holdoutamount": "10",
+            "primarymetrickey": primary_metric_key,
+            "metrics": metrics,
+            "prerequisiteflagkey": prerequisiteflagkey
+        }
+
+        headers = {
+            "Content-Type": "application/json",
+            "Authorization": self.api_key,
+        }
+
+        response = self.getrequest(
+            "POST",
+            "https://app.launchdarkly.com/api/v2/projects/"
+            + self.project_key
+            + "/environments/"
+            + holdout_env_key
+            + "/holdouts",
+            json=payload,
+            headers=headers,
+        )
+
+        data = json.loads(response.text)
+        if "message" in data:
+            print("Error creating holdout: " + data["message"])
+        return response
 
     ##################################################
     # Create a layer

Diff for: pages/marketplace.tsx

@@ -60,9 +60,7 @@ export default function Marketplace() {
   }
 
   const LDClient = useLDClient();
-  const { storeAttentionCallout } = useFlags();
   const releaseNewSearchEngine = useFlags()["release-new-search-engine"]?.includes("new-search-engine");
-  
 
   const [cart, setCart] = useState<InventoryItem[]>([]);
 
@@ -81,6 +79,7 @@ export default function Marketplace() {
     if (releaseNewSearchEngine) {
       addToCart(item);
       LDClient?.track("search-engine-add-to-cart", LDClient.getContext());
+      logLDMetricSent("search-engine-add-to-cart");
       toast({
         title: `${item.item} has been added to your cart!`,
         wrapperStyle: "bg-gradient-experimentation text-white !text-medium font-bold font-sohne",
@@ -98,6 +97,9 @@ export default function Marketplace() {
     if (item.vendor === "boominbox") {
       setOpenBoominBox(true);
     }
+
+    LDClient?.track("search-engine-add-to-cart", LDClient.getContext());
+    logLDMetricSent("search-engine-add-to-cart");
   };
 
   const formatResult = (item: InventoryItem) => {