aggr round2

Author: james-ua

derby.log

@@ -1,6 +1,6 @@
 ----------------------------------------------------------------
-Tue Apr 02 13:57:11 UTC 2024:
-Booting Derby version The Apache Software Foundation - Apache Derby - 10.14.2.0 - (1828579): instance a816c00e-018e-9f19-7b0a-00000609ec60 
+Wed Apr 03 07:58:40 UTC 2024:
+Booting Derby version The Apache Software Foundation - Apache Derby - 10.14.2.0 - (1828579): instance a816c00e-018e-a2f7-9e04-00000157d978 
 on database directory /workspaces/devcontainer-universal/metastore_db with class loader jdk.internal.loader.ClassLoaders$AppClassLoader@5ffd2b27 
 Loaded from file:/usr/local/sdkman/candidates/spark/3.5.0/jars/derby-10.14.2.0.jar
 java.vendor=Eclipse Adoptium

generate.ipynb

@@ -33,11 +33,12 @@
      "text": [
       "Setting default log level to \"WARN\".\n",
       "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n",
-      "24/04/02 13:57:05 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n",
-      "24/04/02 13:57:10 WARN HiveConf: HiveConf of name hive.stats.jdbc.timeout does not exist\n",
-      "24/04/02 13:57:10 WARN HiveConf: HiveConf of name hive.stats.retries.wait does not exist\n",
-      "24/04/02 13:57:13 WARN ObjectStore: Version information not found in metastore. hive.metastore.schema.verification is not enabled so recording the schema version 2.3.0\n",
-      "24/04/02 13:57:13 WARN ObjectStore: setMetaStoreSchemaVersion called but recording version is disabled: version = 2.3.0, comment = Set by MetaStore [email protected]\n"
+      "24/04/03 07:58:26 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n",
+      "24/04/03 07:58:39 WARN HiveConf: HiveConf of name hive.stats.jdbc.timeout does not exist\n",
+      "24/04/03 07:58:39 WARN HiveConf: HiveConf of name hive.stats.retries.wait does not exist\n",
+      "24/04/03 07:58:44 WARN ObjectStore: Version information not found in metastore. hive.metastore.schema.verification is not enabled so recording the schema version 2.3.0\n",
+      "24/04/03 07:58:44 WARN ObjectStore: setMetaStoreSchemaVersion called but recording version is disabled: version = 2.3.0, comment = Set by MetaStore [email protected]\n",
+      "24/04/03 07:58:45 WARN GarbageCollectionMetrics: To enable non-built-in garbage collector(s) List(G1 Concurrent GC), users should configure it(them) to spark.eventLog.gcMetrics.youngGenerationGarbageCollectors or spark.eventLog.gcMetrics.oldGenerationGarbageCollectors\n"
      ]
     },
     {
@@ -46,13 +47,6 @@
      "text": [
       "[Database(name='default', catalog='spark_catalog', description='Default Hive database', locationUri='file:/workspaces/devcontainer-universal/spark-warehouse')]\n"
      ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "24/04/02 13:57:19 WARN GarbageCollectionMetrics: To enable non-built-in garbage collector(s) List(G1 Concurrent GC), users should configure it(them) to spark.eventLog.gcMetrics.youngGenerationGarbageCollectors or spark.eventLog.gcMetrics.oldGenerationGarbageCollectors\n"
-     ]
     }
    ],
    "source": [
@@ -77,7 +71,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 39,
+   "execution_count": 2,
    "metadata": {
     "tags": [
      "UDFs"
@@ -471,9 +465,17 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 128,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "24/04/03 07:59:13 WARN SparkStringUtils: Truncated the string representation of a plan since it was too large. This behavior can be adjusted by setting 'spark.sql.debug.maxToStringFields'.\n",
+      "24/04/03 07:59:14 WARN SessionState: METASTORE_FILTER_HOOK will be ignored, since hive.security.authorization.manager is set to instance of HiveAuthorizerFactory.\n"
+     ]
+    },
     {
      "name": "stdout",
      "output_type": "stream",
@@ -523,9 +525,19 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 120,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "24/04/03 07:59:19 WARN HiveConf: HiveConf of name hive.internal.ss.authz.settings.applied.marker does not exist\n",
+      "24/04/03 07:59:19 WARN HiveConf: HiveConf of name hive.stats.jdbc.timeout does not exist\n",
+      "24/04/03 07:59:19 WARN HiveConf: HiveConf of name hive.stats.retries.wait does not exist\n",
+      "[Stage 4:>                                                          (0 + 1) / 1]\r"
+     ]
+    },
     {
      "name": "stdout",
      "output_type": "stream",
@@ -557,6 +569,13 @@
       "only showing top 20 rows\n",
       "\n"
      ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "                                                                                \r"
+     ]
     }
    ],
    "source": [
@@ -571,7 +590,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 153,
+   "execution_count": 5,
    "metadata": {},
    "outputs": [
     {
@@ -615,7 +634,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 132,
+   "execution_count": 6,
    "metadata": {},
    "outputs": [
     {
@@ -730,9 +749,16 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 157,
+   "execution_count": 7,
    "metadata": {},
    "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "[Stage 17:>                                                         (0 + 1) / 1]\r"
+     ]
+    },
     {
      "name": "stdout",
      "output_type": "stream",
@@ -755,6 +781,13 @@
       "+-----------------+---------------+---------------+---------------------------------------+---------------------------------------------+----------------------------------------------+--------------------------------------------+-----------------------------------------+--------------------------------------------------+---------------------------------------------------+---------------------------------------------+--------------------------------------------+\n",
       "\n"
      ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "                                                                                \r"
+     ]
     }
    ],
    "source": [
@@ -773,7 +806,8 @@
     "                 consumer_dim.Division_Code_Key, \n",
     "                 consumer_dim.Region_Code_Key,\n",
     "                #  consumer_dim.Affiliate_Code_Key,\n",
-    "                 consumer_dim.Market_Code_Key\n",
+    "                 consumer_dim.Market_Code_Key,\n",
+    "                 consumer_dim.CFDM_Channel_Code_Key\n",
     "            )\n",
     "            .agg(\n",
     "                F.countDistinct(consumer_dim.Consumer_Key).alias('Consumer Aggregate Total Consumer Count'),\n",