Merge pull request #209 from HTTPArchive/upgrade-to-new-datamodel

Upgrade to new datamodel

Author: tunetheweb

sql/.sqlfluff

@@ -11,16 +11,18 @@ templater = jinja
 ## Comma separated list of rules to check, or None for all
 rules = None
 ## Comma separated list of rules to exclude, or None
-exclude_rules = AL01,AL04,AL07,AL09,AM03,AM05,CP02,CP03,CV02,LT05,LT09,RF01,RF02,RF03,RF04,ST01,ST02,ST05,ST06,ST07
+exclude_rules = AL01,AL04,AL07,AL09,AM03,AM05,CP02,CP03,CV02,CV12,LT05,LT09,LT14,RF01,RF02,RF03,RF04,ST01,ST02,ST05,ST06,ST07
 # AL04 - Asks for unique table aliases meaning it complains if selecting from two 2021_07_01 tables as implicit alias is table name (not fully qualified) so same.
 # AL07 - Avoid aliases in from and join - why?
 # AM03 - if using DESC in one ORDER BY column, then insist on ASC/DESC for all.
 # AM05 - INNER JOIN must be fully qualified. Probably should use this but not our style.
 # CP02 - Unquoted identifiers (e.g. column names) will be mixed case so don't enforce case
 # CP03 - Function names will be mixed case so don't enforce case
 # CV02 - Use COALESCE instead of IFNULL or NVL. We think ISNULL is clearer.
+# CV12 - Doesn't work with UNNEST. https://github.com/sqlfluff/sqlfluff/issues/6558
 # LT05 - We allow longer lines as some of our queries are complex. Maybe should limit in future?
 # LT09 - Select targets should be on new lines but sub clauses don't always obey this. Maybe revisit in future?
+# LT14 - Keywords on newline. We have some simple, single line joins
 # RF01 - BigQuery uses STRUCTS which can look like incorrect table references
 # RF02 - Asks for qualified columns for ambiguous ones, but we not qualify our columns, and they are not really ambiguous (or BigQuery would complain)
 # RF03 - Insists on references in column names even if not ambiguous. Bit OTT.

sql/.sqlfluffignore

@@ -1 +1,4 @@
 /lens/*/crux_histograms.sql
+/lens/*/crux_timeseries.sql
+/lens/*/histograms.sql
+/lens/*/timeseries.sql

sql/generate_reports.sh

@@ -8,11 +8,14 @@ FROM (
     volume / SUM(volume) OVER (PARTITION BY client) AS pdf
   FROM (
     SELECT
-      _TABLE_SUFFIX AS client,
+      client,
       COUNT(0) AS volume,
-      FLOOR(CAST(IFNULL(JSON_EXTRACT(report, '$.audits.bootup-time.numericValue'), JSON_EXTRACT(report, '$.audits.bootup-time.rawValue')) AS FLOAT64) / 100) / 10 AS bin
+      FLOOR(FLOAT64(IFNULL(lighthouse.audits['bootup-time'].numericValue, lighthouse.audits['bootup-time'].rawValue)) / 100) / 10 AS bin
     FROM
-      `httparchive.lighthouse.${YYYY_MM_DD}_*`
+      `httparchive.crawl.pages`
+    WHERE
+      date = '${YYYY-MM-DD}' AND
+      is_root_page
     GROUP BY
       bin,
       client

sql/histograms/bootupJs.sql

@@ -8,11 +8,14 @@ FROM (
     volume / SUM(volume) OVER (PARTITION BY client) AS pdf
   FROM (
     SELECT
-      _TABLE_SUFFIX AS client,
+      client,
       COUNT(0) AS volume,
-      CAST(FLOOR(bytesCSS / 10240) * 10 AS INT64) AS bin
+      CAST(FLOOR(FLOAT64(summary.bytesCss) / 10240) * 10 AS INT64) AS bin
     FROM
-      `httparchive.summary_pages.${YYYY_MM_DD}_*`
+      `httparchive.crawl.pages`
+    WHERE
+      date = '${YYYY-MM-DD}' AND
+      is_root_page
     GROUP BY
       bin,
       client

sql/histograms/bytesCss.sql

@@ -8,11 +8,14 @@ FROM (
     volume / SUM(volume) OVER (PARTITION BY client) AS pdf
   FROM (
     SELECT
-      _TABLE_SUFFIX AS client,
+      client,
       COUNT(0) AS volume,
-      CAST(FLOOR(bytesFont / 10240) * 10 AS INT64) AS bin
+      CAST(FLOOR(FLOAT64(summary.bytesFont) / 10240) * 10 AS INT64) AS bin
     FROM
-      `httparchive.summary_pages.${YYYY_MM_DD}_*`
+      `httparchive.crawl.pages`
+    WHERE
+      date = '${YYYY-MM-DD}' AND
+      is_root_page
     GROUP BY
       bin,
       client

sql/histograms/bytesFont.sql

@@ -8,11 +8,14 @@ FROM (
     volume / SUM(volume) OVER (PARTITION BY client) AS pdf
   FROM (
     SELECT
-      _TABLE_SUFFIX AS client,
+      client,
       COUNT(0) AS volume,
-      CAST(FLOOR(bytesHtml / 10240) * 10 AS INT64) AS bin
+      CAST(FLOOR(FLOAT64(summary.bytesHtml) / 10240) * 10 AS INT64) AS bin
     FROM
-      `httparchive.summary_pages.${YYYY_MM_DD}_*`
+      `httparchive.crawl.pages`
+    WHERE
+      date = '${YYYY-MM-DD}' AND
+      is_root_page
     GROUP BY
       bin,
       client

sql/histograms/bytesHtml.sql

@@ -8,11 +8,14 @@ FROM (
     volume / SUM(volume) OVER (PARTITION BY client) AS pdf
   FROM (
     SELECT
-      _TABLE_SUFFIX AS client,
+      client,
       COUNT(0) AS volume,
-      CAST(FLOOR(bytesImg / 102400) * 100 AS INT64) AS bin
+      CAST(FLOOR(FLOAT64(summary.bytesImg) / 102400) * 100 AS INT64) AS bin
     FROM
-      `httparchive.summary_pages.${YYYY_MM_DD}_*`
+      `httparchive.crawl.pages`
+    WHERE
+      date = '${YYYY-MM-DD}' AND
+      is_root_page
     GROUP BY
       bin,
       client

sql/histograms/bytesImg.sql

@@ -8,11 +8,14 @@ FROM (
     volume / SUM(volume) OVER (PARTITION BY client) AS pdf
   FROM (
     SELECT
-      _TABLE_SUFFIX AS client,
+      client,
       COUNT(0) AS volume,
-      CAST(FLOOR(bytesJS / 10240) * 10 AS INT64) AS bin
+      CAST(FLOOR(FLOAT64(summary.bytesJS) / 10240) * 10 AS INT64) AS bin
     FROM
-      `httparchive.summary_pages.${YYYY_MM_DD}_*`
+      `httparchive.crawl.pages`
+    WHERE
+      date = '${YYYY-MM-DD}' AND
+      is_root_page
     GROUP BY
       bin,
       client

sql/histograms/bytesJs.sql

@@ -8,11 +8,14 @@ FROM (
     volume / SUM(volume) OVER (PARTITION BY client) AS pdf
   FROM (
     SELECT
-      _TABLE_SUFFIX AS client,
+      client,
       COUNT(0) AS volume,
-      CAST(FLOOR(bytesOther / 10240) * 10 AS INT64) AS bin
+      CAST(FLOOR(FLOAT64(summary.bytesOther) / 10240) * 10 AS INT64) AS bin
     FROM
-      `httparchive.summary_pages.${YYYY_MM_DD}_*`
+      `httparchive.crawl.pages`
+    WHERE
+      date = '${YYYY-MM-DD}' AND
+      is_root_page
     GROUP BY
       bin,
       client