Fixed related bug where a crawler with multiple extraction rules was …

…only receiving the last-known extraction rule and not all extraction rules
elastic · Feb 26, 2025 · 8cf33a7 · 8cf33a7
1 parent 5ca678d
commit 8cf33a7
Showing 1 changed file with 15 additions and 9 deletions.
diff --git a/notebooks/enterprise-search/elastic-crawler-to-open-crawler-migration.ipynb b/notebooks/enterprise-search/elastic-crawler-to-open-crawler-migration.ipynb
@@ -275,6 +275,15 @@
     "        and domain_oid in inflight_configuration_data[config_oid][\"domains_temp\"]\n",
     "    ):\n",
     "\n",
+    "        # initialize extraction rulesets an empty array if it doesn't exist yet\n",
+    "        if (\n",
+    "            not \"extraction_rulesets\"\n",
+    "            in inflight_configuration_data[config_oid][\"domains_temp\"][domain_oid]\n",
+    "        ):\n",
+    "            inflight_configuration_data[config_oid][\"domains_temp\"][domain_oid][\n",
+    "                \"extraction_rulesets\"\n",
+    "            ] = []\n",
+    "\n",
     "        all_rules = source[\"rules\"]\n",
     "        all_url_filters = source[\"url_filters\"]\n",
     "\n",
@@ -294,7 +303,7 @@
     "            \"extracted\": \"extract\",\n",
     "        }\n",
     "\n",
-    "        ruleset = {}\n",
+    "        ruleset = []\n",
     "        if all_rules:\n",
     "            ruleset = [\n",
     "                {\n",
@@ -309,13 +318,10 @@
     "                }\n",
     "            ]\n",
     "\n",
-    "        # populate the in-memory data structure\n",
-    "        temp_extraction_rulesets = [\n",
-    "            {\n",
-    "                \"url_filters\": url_filters,\n",
-    "                \"rules\": ruleset,\n",
-    "            }\n",
-    "        ]\n",
+    "        temp_extraction_rulesets = {\n",
+    "            \"url_filters\": url_filters,\n",
+    "            \"rules\": ruleset,\n",
+    "        }\n",
     "\n",
     "        print(\n",
     "            f\"{extr_count}.) Crawler {config_oid} has extraction rules {temp_extraction_rulesets}\\n\"\n",
@@ -324,7 +330,7 @@
     "\n",
     "        inflight_configuration_data[config_oid][\"domains_temp\"][domain_oid][\n",
     "            \"extraction_rulesets\"\n",
-    "        ] = temp_extraction_rulesets"
+    "        ].append(temp_extraction_rulesets)"
    ]
   },
   {