diff --git a/sql/moz-fx-data-shared-prod/amo_glean/fenix_addons_by_client/view.sql b/sql/moz-fx-data-shared-prod/amo_glean/fenix_addons_by_client/view.sql new file mode 100644 index 00000000000..205204bdcb3 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/amo_glean/fenix_addons_by_client/view.sql @@ -0,0 +1,7 @@ +CREATE OR REPLACE VIEW + `moz-fx-data-shared-prod.amo_glean.fenix_addons_by_client` +AS +SELECT + * +FROM + `moz-fx-data-shared-prod.amo_glean_derived.fenix_addons_by_client_v1` diff --git a/sql/moz-fx-data-shared-prod/amo_glean/firefox_desktop_addons_by_client/view.sql b/sql/moz-fx-data-shared-prod/amo_glean/firefox_desktop_addons_by_client/view.sql new file mode 100644 index 00000000000..abedebcaf48 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/amo_glean/firefox_desktop_addons_by_client/view.sql @@ -0,0 +1,7 @@ +CREATE OR REPLACE VIEW + `moz-fx-data-shared-prod.amo_glean.firefox_desktop_addons_by_client` +AS +SELECT + * +FROM + `moz-fx-data-shared-prod.amo_glean_derived.firefox_desktop_addons_by_client_v1` diff --git a/sql/moz-fx-data-shared-prod/amo_glean/firefox_desktop_stats_installs/view.sql b/sql/moz-fx-data-shared-prod/amo_glean/firefox_desktop_stats_installs/view.sql new file mode 100644 index 00000000000..588ffc1deeb --- /dev/null +++ b/sql/moz-fx-data-shared-prod/amo_glean/firefox_desktop_stats_installs/view.sql @@ -0,0 +1,37 @@ +CREATE OR REPLACE VIEW + `moz-fx-data-shared-prod.amo_glean.firefox_desktop_stats_installs` +AS +SELECT + submission_date, + hashed_addon_id, + total_downloads, + ARRAY( + SELECT AS STRUCT + IFNULL(key, 'Unknown') AS key, + value + FROM + UNNEST(downloads_per_campaign) + ) AS downloads_per_campaign, + ARRAY( + SELECT AS STRUCT + IFNULL(key, 'Unknown') AS key, + value + FROM + UNNEST(downloads_per_content) + ) AS downloads_per_content, + ARRAY( + SELECT AS STRUCT + IFNULL(key, 'Unknown') AS key, + value + FROM + UNNEST(downloads_per_source) + ) AS downloads_per_source, + ARRAY( + SELECT AS STRUCT + IFNULL(key, 'Unknown') AS key, + value + FROM + UNNEST(downloads_per_medium) + ) AS downloads_per_medium, +FROM + `moz-fx-data-shared-prod.amo_glean_derived.firefox_desktop_stats_installs_v1` diff --git a/sql/moz-fx-data-shared-prod/amo_glean/stats_dau/view.sql b/sql/moz-fx-data-shared-prod/amo_glean/stats_dau/view.sql new file mode 100644 index 00000000000..b483332bed0 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/amo_glean/stats_dau/view.sql @@ -0,0 +1,51 @@ +CREATE OR REPLACE VIEW + `moz-fx-data-shared-prod.amo_glean.stats_dau` +AS +SELECT + submission_date, + addon_id, + dau, + ARRAY( + SELECT AS STRUCT + IFNULL(key, 'Unknown') AS key, + value + FROM + UNNEST(dau_by_addon_version) + ) AS dau_by_addon_version, + ARRAY( + SELECT AS STRUCT + IFNULL(key, 'Unknown') AS key, + value + FROM + UNNEST(dau_by_app_os) + ) AS dau_by_app_os, + ARRAY( + SELECT AS STRUCT + IFNULL(key, 'Unknown') AS key, + value + FROM + UNNEST(dau_by_app_version) + ) AS dau_by_app_version, + ARRAY( + SELECT AS STRUCT + IFNULL(key, 'Unknown') AS key, + value + FROM + UNNEST(dau_by_fenix_build) + ) AS dau_by_fenix_build, + ARRAY( + SELECT AS STRUCT + IFNULL(key, 'Unknown') AS key, + value + FROM + UNNEST(dau_by_country) + ) AS dau_by_country, + ARRAY( + SELECT AS STRUCT + IFNULL(key, 'Unknown') AS key, + value + FROM + UNNEST(dau_by_locale) + ) AS dau_by_locale, +FROM + `moz-fx-data-shared-prod.amo_glean_derived.stats_dau_v1` diff --git a/sql/moz-fx-data-shared-prod/amo_glean_derived/fenix_addons_by_client_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/amo_glean_derived/fenix_addons_by_client_v1/metadata.yaml new file mode 100644 index 00000000000..4238457b70e --- /dev/null +++ b/sql/moz-fx-data-shared-prod/amo_glean_derived/fenix_addons_by_client_v1/metadata.yaml @@ -0,0 +1,24 @@ +--- +friendly_name: Fenix addons by client +description: >- + Clients_daily-like table on top of the various Firefox for Android channels + that records only the dimensions and addon info necessary to power the daily + amo_stats_dau_v2 query. +owners: + - kik@mozilla.com +labels: + application: amo + incremental: true + schedule: daily + table_table: client_level +scheduling: + dag_name: bqetl_amo_stats +bigquery: + time_partitioning: + type: day + field: submission_date + require_partition_filter: false + expiration_days: 775 + clustering: + fields: + - sample_id diff --git a/sql/moz-fx-data-shared-prod/amo_glean_derived/fenix_addons_by_client_v1/query.sql b/sql/moz-fx-data-shared-prod/amo_glean_derived/fenix_addons_by_client_v1/query.sql new file mode 100644 index 00000000000..371a7e7a896 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/amo_glean_derived/fenix_addons_by_client_v1/query.sql @@ -0,0 +1,99 @@ +CREATE TEMP FUNCTION get_fields(m ANY TYPE) AS ( + STRUCT( + m.submission_timestamp, + m.client_info.client_id, + m.sample_id, + m.metrics.string_list.addons_enabled_addons, + m.normalized_country_code, + m.client_info.locale, + m.normalized_os + ) +); + +WITH unioned AS ( + SELECT + get_fields(release).*, + client_info.app_display_version AS app_version, + FROM + `moz-fx-data-shared-prod.org_mozilla_firefox.metrics` AS release + UNION ALL + SELECT + get_fields(beta).*, + -- Bug 1669516 We choose to show beta versions as 80.0.0b1, etc. + REPLACE(client_info.app_display_version, '-beta.', 'b') AS app_version, + FROM + `moz-fx-data-shared-prod.org_mozilla_firefox_beta.metrics` AS beta + UNION ALL + SELECT + get_fields(nightly).*, + -- Bug 1669516 Nightly versions have app_display_version like "Nightly ", + -- so we take the geckoview version instead. + nightly.metrics.string.geckoview_version AS app_version, + FROM + `moz-fx-data-shared-prod.org_mozilla_fenix.metrics` AS nightly + UNION ALL + SELECT + get_fields(preview_nightly).*, + preview_nightly.metrics.string.geckoview_version AS app_version, + FROM + `moz-fx-data-shared-prod.org_mozilla_fenix_nightly.metrics` AS preview_nightly + UNION ALL + SELECT + get_fields(old_fenix_nightly).*, + old_fenix_nightly.metrics.string.geckoview_version AS app_version, + FROM + `moz-fx-data-shared-prod.org_mozilla_fennec_aurora.metrics` AS old_fenix_nightly +), +cleaned AS ( + SELECT + * REPLACE ( + IF( + -- Accepts formats: 80.0 80.0.0 80.0.0a1 80.0.0b1 + REGEXP_CONTAINS(app_version, r'^(\d+\.\d+(\.\d+)?([ab]\d+)?)$'), + app_version, + NULL + ) AS app_version + ) + FROM + unioned +), +per_client AS ( + SELECT + DATE(submission_timestamp) AS submission_date, + client_id, + sample_id, + ARRAY_CONCAT_AGG(addons_enabled_addons ORDER BY submission_timestamp) AS addons, + -- We always want to take the most recent seen version per + -- https://bugzilla.mozilla.org/show_bug.cgi?id=1693308 + ARRAY_AGG(app_version ORDER BY mozfun.norm.truncate_version(app_version, "minor") DESC)[ + SAFE_OFFSET(0) + ] AS app_version, + `moz-fx-data-shared-prod.udf.mode_last`(ARRAY_AGG(normalized_country_code)) AS country, + `moz-fx-data-shared-prod.udf.mode_last`(ARRAY_AGG(locale)) AS locale, + `moz-fx-data-shared-prod.udf.mode_last`(ARRAY_AGG(normalized_os)) AS app_os, + FROM + cleaned + WHERE + DATE(submission_timestamp) = @submission_date + AND client_id IS NOT NULL + GROUP BY + submission_date, + sample_id, + client_id +) +SELECT + * EXCEPT (addons), + ARRAY( + SELECT AS STRUCT + TRIM(addon) AS addon, + -- As of 2020-07-01, the metrics ping from Fenix contains no data about + -- the version of installed addons, so we inject null and replace with + -- an appropriate placeholder value when we get to the app-facing view. + CAST(NULL AS STRING) AS version, + FROM + UNNEST(addons) AS addon + GROUP BY + TRIM(addon) + ) AS addons +FROM + per_client diff --git a/sql/moz-fx-data-shared-prod/amo_glean_derived/fenix_addons_by_client_v1/schema.yaml b/sql/moz-fx-data-shared-prod/amo_glean_derived/fenix_addons_by_client_v1/schema.yaml new file mode 100644 index 00000000000..0d818315b2b --- /dev/null +++ b/sql/moz-fx-data-shared-prod/amo_glean_derived/fenix_addons_by_client_v1/schema.yaml @@ -0,0 +1,32 @@ +fields: +- name: submission_date + type: DATE + mode: NULLABLE +- name: client_id + type: STRING + mode: NULLABLE +- name: sample_id + type: INTEGER + mode: NULLABLE +- name: app_version + type: STRING + mode: NULLABLE +- name: country + type: STRING + mode: NULLABLE +- name: locale + type: STRING + mode: NULLABLE +- name: app_os + type: STRING + mode: NULLABLE +- name: addons + type: RECORD + mode: REPEATED + fields: + - name: addon + type: STRING + mode: NULLABLE + - name: version + type: STRING + mode: NULLABLE diff --git a/sql/moz-fx-data-shared-prod/amo_glean_derived/firefox_desktop_addons_by_client_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/amo_glean_derived/firefox_desktop_addons_by_client_v1/metadata.yaml new file mode 100644 index 00000000000..6cf488b2cf9 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/amo_glean_derived/firefox_desktop_addons_by_client_v1/metadata.yaml @@ -0,0 +1,23 @@ +--- +friendly_name: Desktop addons by client +description: >- + Clients_daily-like table that records only the dimensions and addon info + necessary to power daily the amo_stats_dau_v2 query. +owners: + - kik@mozilla.com +labels: + application: amo + incremental: true + schedule: daily + table_table: client_level +scheduling: + dag_name: bqetl_amo_stats +bigquery: + time_partitioning: + type: day + field: submission_date + require_partition_filter: false + expiration_days: 775 + clustering: + fields: + - sample_id diff --git a/sql/moz-fx-data-shared-prod/amo_glean_derived/firefox_desktop_addons_by_client_v1/query.sql b/sql/moz-fx-data-shared-prod/amo_glean_derived/firefox_desktop_addons_by_client_v1/query.sql new file mode 100644 index 00000000000..4ed38190e92 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/amo_glean_derived/firefox_desktop_addons_by_client_v1/query.sql @@ -0,0 +1,111 @@ +WITH base AS ( + SELECT + submission_timestamp, + DATE(submission_timestamp) AS submission_date, + client_info.client_id, + sample_id, + metrics.uuid.legacy_telemetry_client_id, + normalized_channel, + normalized_country_code, + client_info.locale, + normalized_os, + client_info.app_display_version, + -- merging active_addons and addons_theme into a single addons object + ARRAY_CONCAT( + -- COALESCE to make sure array concat returns results if active_addons is empty and addons_theme is not + COALESCE(JSON_QUERY_ARRAY(metrics.object.addons_active_addons, '$'), []), + [ + JSON_QUERY(metrics.object.addons_theme, '$') + ] -- wrapping the json object in array to enable array concat + ) AS active_addons, + FROM + `moz-fx-data-shared-prod.firefox_desktop.metrics` + WHERE + DATE(submission_timestamp) = @submission_date + AND client_info.client_id IS NOT NULL + AND ( + ( + metrics.object.addons_active_addons IS NOT NULL + AND ARRAY_LENGTH(JSON_QUERY_ARRAY(metrics.object.addons_active_addons, '$')) > 0 + ) + OR metrics.object.addons_theme IS NOT NULL + ) +), +per_clients_without_addons AS ( + SELECT + submission_date, + client_id, + sample_id, + mozfun.stats.mode_last( + ARRAY_AGG(legacy_telemetry_client_id ORDER BY submission_timestamp) + ) AS legacy_telemetry_client_id, + ARRAY_AGG( + app_display_version + ORDER BY + mozfun.norm.truncate_version(app_display_version, "minor") DESC + )[SAFE_OFFSET(0)] AS app_version, + mozfun.stats.mode_last( + ARRAY_AGG(normalized_country_code ORDER BY submission_timestamp) + ) AS country, + mozfun.stats.mode_last(ARRAY_AGG(locale ORDER BY submission_timestamp)) AS locale, + mozfun.stats.mode_last(ARRAY_AGG(normalized_os ORDER BY submission_timestamp)) AS app_os, + FROM + base + GROUP BY + ALL +), +per_clients_just_addons AS ( + SELECT + submission_date, + client_id, + sample_id, + mozfun.stats.mode_last( + ARRAY_AGG(legacy_telemetry_client_id ORDER BY submission_timestamp) + ) AS legacy_telemetry_client_id, + ARRAY_CONCAT_AGG( + ARRAY( + SELECT AS STRUCT + JSON_VALUE(addon, '$.id') AS id, + JSON_VALUE(addon, '$.version') AS `version` + FROM + UNNEST(active_addons) AS addon + ) + ) AS addons + FROM + base + GROUP BY + ALL +), +per_client AS ( + SELECT + submission_date, + client_id, + sample_id, + legacy_telemetry_client_id, + addons, + app_version, + country, + locale, + app_os, + FROM + per_clients_without_addons + INNER JOIN + per_clients_just_addons + USING (submission_date, client_id, sample_id, legacy_telemetry_client_id) +) +SELECT + * EXCEPT (addons), + ARRAY( + SELECT AS STRUCT + addon.id, + -- Same methodology as for app_version above. + ARRAY_AGG(addon.version ORDER BY mozfun.norm.truncate_version(addon.version, "minor") DESC)[ + SAFE_OFFSET(0) + ] AS `version`, + FROM + UNNEST(addons) AS addon + GROUP BY + addon.id + ) AS addons +FROM + per_client diff --git a/sql/moz-fx-data-shared-prod/amo_glean_derived/firefox_desktop_addons_by_client_v1/schema.yaml b/sql/moz-fx-data-shared-prod/amo_glean_derived/firefox_desktop_addons_by_client_v1/schema.yaml new file mode 100644 index 00000000000..c0eed9eec15 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/amo_glean_derived/firefox_desktop_addons_by_client_v1/schema.yaml @@ -0,0 +1,35 @@ +fields: +- name: submission_date + type: DATE + mode: NULLABLE +- name: client_id + type: STRING + mode: NULLABLE +- name: sample_id + type: INTEGER + mode: NULLABLE +- name: legacy_telemetry_client_id + type: STRING + mode: NULLABLE +- name: app_version + type: STRING + mode: NULLABLE +- name: country + type: STRING + mode: NULLABLE +- name: locale + type: STRING + mode: NULLABLE +- name: app_os + type: STRING + mode: NULLABLE +- name: addons + type: RECORD + mode: REPEATED + fields: + - name: id + type: STRING + mode: NULLABLE + - name: version + type: STRING + mode: NULLABLE diff --git a/sql/moz-fx-data-shared-prod/amo_glean_derived/firefox_desktop_stats_installs_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/amo_glean_derived/firefox_desktop_stats_installs_v1/metadata.yaml new file mode 100644 index 00000000000..2e691eada27 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/amo_glean_derived/firefox_desktop_stats_installs_v1/metadata.yaml @@ -0,0 +1,30 @@ +--- +friendly_name: AMO Stats DAU +description: > + Daily install statistics to power addons.mozilla.org stats pages. + See bug 1654330. Note that this table uses a hashed_addon_id + defined as `TO_HEX(SHA256(addon_id))` because the underlying event + pings have limitations on length of properties attached to events + and addon_id values are sometimes too long. The AMO stats application + looks up records in this table based on the hashed_addon_id. + + The hashed_addon_id is only valid for install_stats: + https://searchfox.org/mozilla-central/rev/5b061cdc4d40d44988dc61aa941cfbd98e31791f/toolkit/components/telemetry/Events.yaml#233-234 +owners: + - kik@mozilla.com +labels: + application: amo + incremental: true + schedule: daily + table_type: aggregate + shredder_mitigation: false +scheduling: + dag_name: bqetl_amo_stats +bigquery: + time_partitioning: + type: day + field: submission_date + require_partition_filter: false + clustering: + fields: + - hashed_addon_id diff --git a/sql/moz-fx-data-shared-prod/amo_glean_derived/firefox_desktop_stats_installs_v1/query.sql b/sql/moz-fx-data-shared-prod/amo_glean_derived/firefox_desktop_stats_installs_v1/query.sql new file mode 100644 index 00000000000..d5a16aa9d40 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/amo_glean_derived/firefox_desktop_stats_installs_v1/query.sql @@ -0,0 +1,152 @@ +CREATE TEMP FUNCTION extract_addon_info(addon_info ANY TYPE) +RETURNS STRUCT< + hashed_addon_id STRING, + utm_content STRING, + utm_campaign STRING, + utm_source STRING, + utm_medium STRING +> AS ( + STRUCT( + mozfun.map.get_key(addon_info, "hashed_addon_id") AS hashed_addon_id, + mozfun.map.get_key(addon_info, "utm_content") AS utm_content, + mozfun.map.get_key(addon_info, "utm_campaign") AS utm_campaign, + mozfun.map.get_key(addon_info, "utm_source") AS utm_source, + mozfun.map.get_key(addon_info, "utm_medium") AS utm_medium + ) +); + +WITH install_stats AS ( + SELECT + DATE(submission_timestamp) AS submission_date, + client_info.client_id, + extract_addon_info(event.extra).*, + FROM + `moz-fx-data-shared-prod.firefox_desktop.events`, + UNNEST(events) AS `event` + WHERE + DATE(submission_timestamp) = @submission_date + AND event.category = "addons_manager" + AND event.name = "install_stats" + GROUP BY + ALL +), +per_source AS ( + SELECT + hashed_addon_id, + submission_date, + ARRAY_AGG(STRUCT(key, value) ORDER BY value DESC) AS downloads_per_source + FROM + ( + SELECT + hashed_addon_id, + submission_date, + utm_source AS key, + COUNT(DISTINCT client_id) AS value + FROM + install_stats + GROUP BY + submission_date, + hashed_addon_id, + key + ) + GROUP BY + submission_date, + hashed_addon_id +), +per_content AS ( + SELECT + hashed_addon_id, + submission_date, + ARRAY_AGG(STRUCT(key, value) ORDER BY value DESC) AS downloads_per_content + FROM + ( + SELECT + hashed_addon_id, + submission_date, + utm_content AS key, + COUNT(DISTINCT client_id) AS value + FROM + install_stats + GROUP BY + submission_date, + hashed_addon_id, + key + ) + GROUP BY + submission_date, + hashed_addon_id +), +per_medium AS ( + SELECT + hashed_addon_id, + submission_date, + ARRAY_AGG(STRUCT(key, value) ORDER BY value DESC) AS downloads_per_medium + FROM + ( + SELECT + hashed_addon_id, + submission_date, + utm_medium AS key, + COUNT(DISTINCT client_id) AS value + FROM + install_stats + GROUP BY + submission_date, + hashed_addon_id, + key + ) + GROUP BY + submission_date, + hashed_addon_id +), +per_campaign AS ( + SELECT + hashed_addon_id, + submission_date, + ARRAY_AGG(STRUCT(key, value) ORDER BY value DESC) AS downloads_per_campaign + FROM + ( + SELECT + hashed_addon_id, + submission_date, + utm_campaign AS key, + COUNT(DISTINCT client_id) AS value + FROM + install_stats + GROUP BY + submission_date, + hashed_addon_id, + key + ) + GROUP BY + submission_date, + hashed_addon_id +), +-- +total_downloads AS ( + SELECT + hashed_addon_id, + submission_date, + COUNT(DISTINCT client_id) AS total_downloads + FROM + install_stats + GROUP BY + hashed_addon_id, + submission_date +) +SELECT + * +FROM + total_downloads +LEFT JOIN + per_source + USING (submission_date, hashed_addon_id) +LEFT JOIN + per_content + USING (submission_date, hashed_addon_id) +LEFT JOIN + per_medium + USING (submission_date, hashed_addon_id) +LEFT JOIN + per_campaign + USING (submission_date, hashed_addon_id) diff --git a/sql/moz-fx-data-shared-prod/amo_glean_derived/firefox_desktop_stats_installs_v1/schema.yaml b/sql/moz-fx-data-shared-prod/amo_glean_derived/firefox_desktop_stats_installs_v1/schema.yaml new file mode 100644 index 00000000000..0618dd7f5b5 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/amo_glean_derived/firefox_desktop_stats_installs_v1/schema.yaml @@ -0,0 +1,50 @@ +fields: +- name: submission_date + type: DATE + mode: NULLABLE +- name: hashed_addon_id + type: STRING + mode: NULLABLE +- name: total_downloads + type: INTEGER + mode: NULLABLE +- name: downloads_per_source + type: RECORD + mode: REPEATED + fields: + - name: key + type: STRING + mode: NULLABLE + - name: value + type: INTEGER + mode: NULLABLE +- name: downloads_per_content + type: RECORD + mode: REPEATED + fields: + - name: key + type: STRING + mode: NULLABLE + - name: value + type: INTEGER + mode: NULLABLE +- name: downloads_per_medium + type: RECORD + mode: REPEATED + fields: + - name: key + type: STRING + mode: NULLABLE + - name: value + type: INTEGER + mode: NULLABLE +- name: downloads_per_campaign + type: RECORD + mode: REPEATED + fields: + - name: key + type: STRING + mode: NULLABLE + - name: value + type: INTEGER + mode: NULLABLE diff --git a/sql/moz-fx-data-shared-prod/amo_glean_derived/stats_dau_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/amo_glean_derived/stats_dau_v1/metadata.yaml new file mode 100644 index 00000000000..6bc1c8e4ac7 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/amo_glean_derived/stats_dau_v1/metadata.yaml @@ -0,0 +1,27 @@ +--- +friendly_name: AMO Stats DAU +description: |- + Daily user statistics to power addons.mozilla.org stats pages. + See bug 1572873. + + Each row in this table represents a particular addon on a particular day + and provides all the information needed to populate the various + "Daily Users" plots for the AMO stats dashboard. +owners: + - kik@mozilla.com +labels: + application: amo + incremental: true + schedule: daily + table_type: aggregate + shredder_mitigation: false +scheduling: + dag_name: bqetl_amo_stats +bigquery: + time_partitioning: + type: day + field: submission_date + require_partition_filter: false + clustering: + fields: + - addon_id diff --git a/sql/moz-fx-data-shared-prod/amo_glean_derived/stats_dau_v1/query.sql b/sql/moz-fx-data-shared-prod/amo_glean_derived/stats_dau_v1/query.sql new file mode 100644 index 00000000000..547a805b452 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/amo_glean_derived/stats_dau_v1/query.sql @@ -0,0 +1,214 @@ +/* + +Daily user statistics to power AMO stats pages. See bug 1572873. + +Each row in this table represents a particular addon on a particular day +and provides all the information needed to populate the various +"Daily Users" plots for the AMO stats dashboard. + +*/ +-- +WITH unioned AS ( + SELECT + *, + 'Desktop' AS app + FROM + `moz-fx-data-shared-prod.amo_glean.firefox_desktop_addons_by_client` + UNION ALL + SELECT + *, + 'Fenix' AS app + FROM + `moz-fx-data-shared-prod.amo_glean.fenix_addons_by_client` +), +unnested AS ( + SELECT + unioned.* EXCEPT (addons), + addon.id AS addon_id, + addon.version AS addon_version, + FROM + unioned + CROSS JOIN + UNNEST(addons) AS addon + WHERE + submission_date = @submission_date + AND addon.id IS NOT NULL +), +-- +per_addon_version AS ( + SELECT + submission_date, + addon_id, + ARRAY_AGG(STRUCT(key, value) ORDER BY value DESC) AS dau_by_addon_version + FROM + ( + SELECT + submission_date, + addon_id, + addon_version AS key, + COUNT(DISTINCT client_id) AS value + FROM + unnested + GROUP BY + submission_date, + addon_id, + key + ) + GROUP BY + submission_date, + addon_id +), +per_app_version AS ( + SELECT + submission_date, + addon_id, + ARRAY_AGG(STRUCT(key, value) ORDER BY value DESC) AS dau_by_app_version + FROM + ( + SELECT + submission_date, + addon_id, + app_version AS key, + COUNT(DISTINCT client_id) AS value + FROM + unnested + WHERE + app = 'Desktop' + GROUP BY + submission_date, + addon_id, + key + ) + GROUP BY + submission_date, + addon_id +), +per_fenix_build AS ( + SELECT + submission_date, + addon_id, + ARRAY_AGG(STRUCT(key, value) ORDER BY value DESC) AS dau_by_fenix_build + FROM + ( + SELECT + submission_date, + addon_id, + app_version AS key, + COUNT(DISTINCT client_id) AS value + FROM + unnested + WHERE + app = 'Fenix' + GROUP BY + submission_date, + addon_id, + key + ) + GROUP BY + submission_date, + addon_id +), +per_locale AS ( + SELECT + submission_date, + addon_id, + ARRAY_AGG(STRUCT(key, value) ORDER BY value DESC) AS dau_by_locale + FROM + ( + SELECT + submission_date, + addon_id, + locale AS key, + COUNT(DISTINCT client_id) AS value + FROM + unnested + GROUP BY + submission_date, + addon_id, + key + ) + GROUP BY + submission_date, + addon_id +), +per_country AS ( + SELECT + submission_date, + addon_id, + ARRAY_AGG(STRUCT(key, value) ORDER BY value DESC) AS dau_by_country + FROM + ( + SELECT + submission_date, + addon_id, + country AS key, + COUNT(DISTINCT client_id) AS value + FROM + unnested + GROUP BY + submission_date, + addon_id, + key + ) + GROUP BY + submission_date, + addon_id +), +per_app_os AS ( + SELECT + submission_date, + addon_id, + ARRAY_AGG(STRUCT(key, value) ORDER BY value DESC) AS dau_by_app_os + FROM + ( + SELECT + submission_date, + addon_id, + app_os AS key, + COUNT(DISTINCT client_id) AS value + FROM + unnested + GROUP BY + submission_date, + addon_id, + key + ) + GROUP BY + submission_date, + addon_id +), +-- +total_dau AS ( + SELECT + submission_date, + addon_id, + COUNT(DISTINCT client_id) AS dau + FROM + unnested + GROUP BY + submission_date, + addon_id +) +-- +SELECT + * +FROM + total_dau +LEFT JOIN + per_addon_version + USING (submission_date, addon_id) +LEFT JOIN + per_app_version + USING (submission_date, addon_id) +LEFT JOIN + per_fenix_build + USING (submission_date, addon_id) +LEFT JOIN + per_locale + USING (submission_date, addon_id) +LEFT JOIN + per_country + USING (submission_date, addon_id) +LEFT JOIN + per_app_os + USING (submission_date, addon_id) diff --git a/sql/moz-fx-data-shared-prod/amo_glean_derived/stats_dau_v1/schema.yaml b/sql/moz-fx-data-shared-prod/amo_glean_derived/stats_dau_v1/schema.yaml new file mode 100644 index 00000000000..bd3478392c6 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/amo_glean_derived/stats_dau_v1/schema.yaml @@ -0,0 +1,72 @@ +fields: +- mode: NULLABLE + name: submission_date + type: DATE + description: The date when the telemetry ping is received on the server side. +- name: addon_id + type: STRING + mode: NULLABLE +- name: dau + type: INTEGER + mode: NULLABLE + description: The number of daily active users. +- name: dau_by_addon_version + type: RECORD + mode: REPEATED + fields: + - name: key + type: STRING + mode: NULLABLE + - name: value + type: INTEGER + mode: NULLABLE +- name: dau_by_app_version + type: RECORD + mode: REPEATED + fields: + - name: key + type: STRING + mode: NULLABLE + - name: value + type: INTEGER + mode: NULLABLE +- name: dau_by_fenix_build + type: RECORD + mode: REPEATED + fields: + - name: key + type: STRING + mode: NULLABLE + - name: value + type: INTEGER + mode: NULLABLE +- name: dau_by_locale + type: RECORD + mode: REPEATED + fields: + - name: key + type: STRING + mode: NULLABLE + - name: value + type: INTEGER + mode: NULLABLE +- name: dau_by_country + type: RECORD + mode: REPEATED + fields: + - name: key + type: STRING + mode: NULLABLE + - name: value + type: INTEGER + mode: NULLABLE +- name: dau_by_app_os + type: RECORD + mode: REPEATED + fields: + - name: key + type: STRING + mode: NULLABLE + - name: value + type: INTEGER + mode: NULLABLE