feat(DEN-9914) Create hourly table for Onboarding data

[phil-lee70]

Description

This PR adds an hourly view for the firefox_desktop.onboarding data. The growth PMs would like an hourly view. An hourly run would let them check key metrics (impressions, clicks, dismissals) same-day instead of next-day, making it feasible to adjust copy/targeting before users churn. Waiting ~24 h is slowing the feedback loop and decision-making.

Related Tickets & Documents

• DENG-9914

Reviewer, please follow this checklist

[sean-rose]

r+wc

[dataops-ci-bot]

Integration report for "change dag start_date to todays date"

sql.diff

Click to expand!

Only in /tmp/workspace/generated-sql/dags/: bqetl_messaging_system_hourly.py
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_messaging_system_hourly.py /tmp/workspace/generated-sql/dags/bqetl_messaging_system_hourly.py
--- /tmp/workspace/main-generated-sql/dags/bqetl_messaging_system_hourly.py	1970-01-01 00:00:00.000000000 +0000
+++ /tmp/workspace/generated-sql/dags/bqetl_messaging_system_hourly.py	2025-12-15 13:54:42.000000000 +0000
@@ -0,0 +1,67 @@
+# Generated via https://github.com/mozilla/bigquery-etl/blob/main/bigquery_etl/query_scheduling/generate_airflow_dags.py
+
+from airflow import DAG
+from airflow.sensors.external_task import ExternalTaskMarker
+from airflow.sensors.external_task import ExternalTaskSensor
+from airflow.utils.task_group import TaskGroup
+import datetime
+from operators.gcp_container_operator import GKEPodOperator
+from utils.constants import ALLOWED_STATES, FAILED_STATES
+from utils.gcp import bigquery_etl_query, bigquery_dq_check, bigquery_bigeye_check
+
+docs = """
+### bqetl_messaging_system_hourly
+
+Built from bigquery-etl repo, [`dags/bqetl_messaging_system_hourly.py`](https://github.com/mozilla/bigquery-etl/blob/generated-sql/dags/bqetl_messaging_system_hourly.py)
+
+#### Description
+
+Hourly tables for onboarding reporting
+#### Owner
+
+[email protected]
+
+#### Tags
+
+* impact/tier_1
+* repo/bigquery-etl
+"""
+
+
+default_args = {
+    "owner": "[email protected]",
+    "start_date": datetime.datetime(2025, 12, 15, 0, 0),
+    "end_date": None,
+    "email": ["[email protected]", "[email protected]"],
+    "depends_on_past": False,
+    "retry_delay": datetime.timedelta(seconds=600),
+    "email_on_failure": True,
+    "email_on_retry": False,
+    "retries": 1,
+    "max_active_tis_per_dag": None,
+}
+
+tags = ["impact/tier_1", "repo/bigquery-etl"]
+
+with DAG(
+    "bqetl_messaging_system_hourly",
+    default_args=default_args,
+    schedule_interval="@hourly",
+    doc_md=docs,
+    tags=tags,
+    catchup=True,
+) as dag:
+
+    firefox_desktop_derived__onboarding_hourly__v2 = bigquery_etl_query(
+        task_id="firefox_desktop_derived__onboarding_hourly__v2",
+        destination_table='onboarding_hourly_v2${{ (execution_date - macros.timedelta(hours=1)).strftime("%Y%m%d") }}',
+        dataset_id="firefox_desktop_derived",
+        project_id="moz-fx-data-shared-prod",
+        owner="[email protected]",
+        email=["[email protected]", "[email protected]"],
+        date_partition_parameter=None,
+        depends_on_past=False,
+        parameters=[
+            "submission_date:DATE:{{ (execution_date - macros.timedelta(hours=1)).strftime('%Y-%m-%d') }}"
+        ],
+    )
Only in /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop: onboarding_hourly
Only in /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived: onboarding_hourly_v2
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop/onboarding_hourly/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop/onboarding_hourly/metadata.yaml
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop/onboarding_hourly/metadata.yaml	1970-01-01 00:00:00.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop/onboarding_hourly/metadata.yaml	2025-12-15 13:49:22.000000000 +0000
@@ -0,0 +1,16 @@
+friendly_name: Onboarding Hourly
+description: |-
+  Contains CFR specific data extracted from messaging_system_v1 at an hourly cadence
+owners:
+- [email protected]
+labels:
+  owner1: phlee
+workgroup_access:
+- role: roles/bigquery.dataViewer
+  members:
+  - workgroup:mozilla-confidential
+references:
+  view.sql:
+  - moz-fx-data-shared-prod.firefox_desktop_derived.onboarding_hourly_v2
+require_column_descriptions: false
+level: null
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop/onboarding_hourly/view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop/onboarding_hourly/view.sql
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop/onboarding_hourly/view.sql	1970-01-01 00:00:00.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop/onboarding_hourly/view.sql	2025-12-15 13:46:39.000000000 +0000
@@ -0,0 +1,7 @@
+CREATE OR REPLACE VIEW
+  `moz-fx-data-shared-prod.firefox_desktop.onboarding_hourly`
+AS
+SELECT
+  *
+FROM
+  `moz-fx-data-shared-prod.firefox_desktop_derived.onboarding_hourly_v2`
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/onboarding_hourly_v2/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/onboarding_hourly_v2/metadata.yaml
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/onboarding_hourly_v2/metadata.yaml	1970-01-01 00:00:00.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/onboarding_hourly_v2/metadata.yaml	2025-12-15 13:49:23.000000000 +0000
@@ -0,0 +1,39 @@
+friendly_name: Onboarding Hourly
+description: |-
+  Contains CFR specific data extracted from messaging_system_v1 at an hourly cadence
+owners:
+- [email protected]
+labels:
+  incremental: true
+  schedule: hourly
+  owner1: phlee
+  table_type: client_level
+  dag: bqetl_messaging_system_hourly
+scheduling:
+  dag_name: bqetl_messaging_system_hourly
+  date_partition_parameter: null
+  destination_table: onboarding_hourly_v2${{ (execution_date - macros.timedelta(hours=1)).strftime("%Y%m%d")
+    }}
+  parameters:
+  - submission_date:DATE:{{ (execution_date - macros.timedelta(hours=1)).strftime('%Y-%m-%d')
+    }}
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_timestamp
+    require_partition_filter: true
+    expiration_days: 7.0
+  range_partitioning: null
+  clustering:
+    fields:
+    - normalized_channel
+    - sample_id
+workgroup_access:
+- role: roles/bigquery.dataViewer
+  members:
+  - workgroup:mozilla-confidential
+references:
+  query.sql:
+  - moz-fx-data-shared-prod.firefox_desktop_live.messaging_system_v1
+require_column_descriptions: false
+level: null
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/onboarding_hourly_v2/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/onboarding_hourly_v2/query.sql
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/onboarding_hourly_v2/query.sql	1970-01-01 00:00:00.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/onboarding_hourly_v2/query.sql	2025-12-15 13:46:39.000000000 +0000
@@ -0,0 +1,55 @@
+WITH messaging_system_live_deduped AS (
+  SELECT
+    *
+  FROM
+    `moz-fx-data-shared-prod.firefox_desktop_live.messaging_system_v1`
+  WHERE
+    DATE(submission_timestamp) = @submission_date
+    AND metrics.string.messaging_system_ping_type IS NULL
+  QUALIFY
+    ROW_NUMBER() OVER (
+      PARTITION BY
+        DATE(submission_timestamp),
+        document_id
+      ORDER BY
+        submission_timestamp
+    ) = 1
+)
+SELECT
+  submission_timestamp,
+  additional_properties,
+  metrics.string.messaging_system_addon_version AS addon_version,
+  metrics.uuid.messaging_system_client_id AS client_id,
+  document_id,
+  metrics.string.messaging_system_event AS event,
+  metrics.text2.messaging_system_event_context AS event_context,
+  metrics.string.messaging_system_event_page AS event_page,
+  metrics.string.messaging_system_event_reason AS event_reason,
+  metrics.string.messaging_system_event_source AS event_source,
+  metrics.string.messaging_system_locale AS locale,
+  metrics.text2.messaging_system_message_id AS message_id,
+  metadata,
+  normalized_app_name,
+  normalized_channel,
+  normalized_country_code,
+  normalized_os,
+  normalized_os_version,
+  client_info.app_channel AS release_channel,
+  sample_id,
+  client_info.app_display_version AS version,
+  metrics.uuid.messaging_system_browser_session_id AS browser_session_id,
+  ping_info.experiments AS experiments,
+  STRUCT(
+    metrics.string.messaging_system_attribution_campaign AS campaign,
+    metrics.string.messaging_system_attribution_content AS content,
+    metrics.string.messaging_system_attribution_experiment AS experiment,
+    metrics.string.messaging_system_attribution_medium AS medium,
+    metrics.string.messaging_system_attribution_source AS source,
+    metrics.string.messaging_system_attribution_ua AS ua,
+    metrics.string.messaging_system_attribution_variation AS variation,
+    metrics.string.messaging_system_attribution_dltoken AS dltoken,
+    metrics.string.messaging_system_attribution_dlsource AS dlsource,
+    metrics.string.messaging_system_attribution_msstoresignedin AS msstoresignedin
+  ) AS attribution
+FROM
+  messaging_system_live_deduped
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/onboarding_hourly_v2/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/onboarding_hourly_v2/schema.yaml
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/onboarding_hourly_v2/schema.yaml	1970-01-01 00:00:00.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/onboarding_hourly_v2/schema.yaml	2025-12-15 13:46:39.000000000 +0000
@@ -0,0 +1,257 @@
+fields:
+- description: Timestamp when the ping is received on the server side.
+  name: submission_timestamp
+  type: TIMESTAMP
+  mode: NULLABLE
+- description: A JSON string containing any payload properties not present in the
+    schema
+  name: additional_properties
+  type: STRING
+  mode: NULLABLE
+- description: Addon Version
+  name: addon_version
+  type: STRING
+  mode: NULLABLE
+- description: A unique identifier (UUID) for the client.
+  name: client_id
+  type: STRING
+  mode: NULLABLE
+- description: The document ID specified in the URI when the client sent this message.
+  name: document_id
+  type: STRING
+  mode: NULLABLE
+- name: event
+  type: STRING
+  mode: NULLABLE
+- description: A string that describes the context about this event
+  name: event_context
+  type: STRING
+  mode: NULLABLE
+- description: The event_context's page. Almost always "about:welcome".
+  name: event_page
+  type: STRING
+  mode: NULLABLE
+- description: The event_context's reason. Likely something like "welcome-window-closed"
+    or "app-shut-down".
+  name: event_reason
+  type: STRING
+  mode: NULLABLE
+- description: The event_context's source. Likely something like "primary_button".
+  name: event_source
+  type: STRING
+  mode: NULLABLE
+- name: locale
+  type: STRING
+  mode: NULLABLE
+  description: Set of language- and/or country-based preferences for a user interface.
+- name: message_id
+  type: STRING
+  mode: NULLABLE
+  description: Message ID
+- name: metadata
+  type: RECORD
+  mode: NULLABLE
+  fields:
+  - description: Results of a geographic lookup based on the client's IP address
+    name: geo
+    type: RECORD
+    mode: NULLABLE
+    fields:
+    - name: city
+      type: STRING
+      mode: NULLABLE
+    - description: An ISO 3166-1 alpha-2 country code
+      name: country
+      type: STRING
+      mode: NULLABLE
+    - description: The specific geo database version used for this lookup
+      name: db_version
+      type: STRING
+      mode: NULLABLE
+    - description: First major country subdivision, typically a state, province, or
+        county
+      name: subdivision1
+      type: STRING
+      mode: NULLABLE
+    - description: Second major country subdivision; not applicable for most countries
+      name: subdivision2
+      type: STRING
+      mode: NULLABLE
+  - name: header
+    type: RECORD
+    mode: NULLABLE
+    fields:
+    - description: Date HTTP header
+      mode: NULLABLE
+      name: date
+      type: STRING
+    - description: DNT (Do Not Track) HTTP header
+      mode: NULLABLE
+      name: dnt
+      type: STRING
+    - description: X-Debug-Id HTTP header
+      mode: NULLABLE
+      name: x_debug_id
+      type: STRING
+    - description: X-PingSender-Version HTTP header
+      mode: NULLABLE
+      name: x_pingsender_version
+      type: STRING
+    - description: X-Source-Tags HTTP header
+      mode: NULLABLE
+      name: x_source_tags
+      type: STRING
+    - description: X-Telemetry-Agent HTTP header
+      mode: NULLABLE
+      name: x_telemetry_agent
+      type: STRING
+    - description: X-Foxsec-IP-Reputation header
+      mode: NULLABLE
+      name: x_foxsec_ip_reputation
+      type: STRING
+    - description: X-LB-Tags HTTP header
+      mode: NULLABLE
+      name: x_lb_tags
+      type: STRING
+  - description: Results of ISP lookup based on the client's IP address
+    name: isp
+    type: RECORD
+    mode: NULLABLE
+    fields:
+    - description: The specific geo ISP database version used for this lookup
+      mode: NULLABLE
+      name: db_version
+      type: STRING
+    - description: The name of the ISP associated with the client's IP address
+      mode: NULLABLE
+      name: name
+      type: STRING
+    - description: The name of a specific business entity associated with the client's
+        IP address when available; otherwise the ISP name
+      mode: NULLABLE
+      name: organization
+      type: STRING
+  - description: Parsed components of the client's user agent string
+    name: user_agent
+    type: RECORD
+    mode: NULLABLE
+    fields:
+    - name: browser
+      type: STRING
+      mode: NULLABLE
+    - name: os
+      type: STRING
+      mode: NULLABLE
+    - name: version
+      type: STRING
+      mode: NULLABLE
+- description: Set to "Other" if this message contained an unrecognized app name
+  mode: NULLABLE
+  name: normalized_app_name
+  type: STRING
+- description: The normalized channel the application is being distributed on.
+  mode: NULLABLE
+  name: normalized_channel
+  type: STRING
+- description: Code of the country in which the activity took place, as determined
+    by the IP geolocation. Unknown or NULL values are normally stored as '??'.
+  mode: NULLABLE
+  name: normalized_country_code
+  type: STRING
+- description: The normalized name of the operating system running at the client.
+  mode: NULLABLE
+  name: normalized_os
+  type: STRING
+- name: normalized_os_version
+  mode: NULLABLE
+  type: STRING
+- name: release_channel
+  mode: NULLABLE
+  type: STRING
+- name: sample_id
+  mode: NULLABLE
+  type: INTEGER
+  description: A number, 0-99, that samples by client_id and allows filtering data
+    for analysis. It is a pipeline-generated artifact that should match between pings.
+- name: version
+  mode: NULLABLE
+  type: STRING
+  description: User visible version string (e.g. "1.0.3") for the browser.
+- description: A mirror of the browser sessionId, as defined in
+   https://github.com/mozilla-services/mozilla-pipeline-schemas/blob/main/schemas/telemetry/main/main.4.schema.json
+  name: browser_session_id
+  type: STRING
+  mode: NULLABLE
+- name: experiments
+  type: RECORD
+  mode: REPEATED
+  description: Experiment Information
+  fields:
+  - name: key
+    type: STRING
+    mode: NULLABLE
+  - name: value
+    type: RECORD
+    mode: NULLABLE
+    fields:
+    - name: branch
+      type: STRING
+      mode: NULLABLE
+    - name: extra
+      type: RECORD
+      mode: NULLABLE
+      fields:
+      - name: enrollment_id
+        type: STRING
+        mode: NULLABLE
+      - name: type
+        type: STRING
+        mode: NULLABLE
+- name: attribution
+  type: RECORD
+  mode: NULLABLE
+  fields:
+  - description: Identifier of the particular campaign that led to the download of
+      the product.
+    name: campaign
+    type: STRING
+    mode: NULLABLE
+  - description: Identifier to indicate the particular link within a campaign.
+    name: content
+    type: STRING
+    mode: NULLABLE
+  - description: Funnel experiment parameters, see bug 1567339
+    name: experiment
+    type: STRING
+    mode: NULLABLE
+  - description: Category of the source, such as 'organic' for a search engine.
+    name: medium
+    type: STRING
+    mode: NULLABLE
+  - description: Referring partner domain, when install happens via a known partner.
+    name: source
+    type: STRING
+    mode: NULLABLE
+  - description: Derived user agent, see bug 1595063
+    name: ua
+    type: STRING
+    mode: NULLABLE
+  - description: Funnel experiment parameters, see bug 1567339
+    name: variation
+    type: STRING
+    mode: NULLABLE
+  - description: Unique token created at Firefox download time, see bug 1757451
+    name: dltoken
+    type: STRING
+    mode: NULLABLE
+  - description: Identifier that indicates where installations of Firefox originate,
+      see bug 1827238
+    name: dlsource
+    type: STRING
+    mode: NULLABLE
+  - description: Either the string "true" or the string "false" to indicate whether
+      the attributed install came from the Microsoft store and, if so, whether the
+      user was signed in at the time.
+    name: msstoresignedin
+    type: STRING
+    mode: NULLABLE

Link to full diff

[sean-rose]

@phil-lee70 you'll need to manually enable the new bqetl_messaging_system_hourly DAG (though it's good you didn't enable it immediately, which allowed the artifact deployment process to properly create the onboarding_hourly_v2 table with the configured partitioning).