From 40b5d50deacc66e65a117e50a16b6461cf6c594c Mon Sep 17 00:00:00 2001 From: "Aaron T. Grogg" Date: Tue, 13 Jan 2026 11:10:24 -0800 Subject: [PATCH 01/13] Add SQL query for flexbox and grid adoption analysis --- sql/2025/css/flexbox_grid.sql | 40 +++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 sql/2025/css/flexbox_grid.sql diff --git a/sql/2025/css/flexbox_grid.sql b/sql/2025/css/flexbox_grid.sql new file mode 100644 index 00000000000..baeb22c8635 --- /dev/null +++ b/sql/2025/css/flexbox_grid.sql @@ -0,0 +1,40 @@ +#standardSQL +# flexbox and grid adoption +WITH totals AS ( + SELECT + date, + client, + COUNT(DISTINCT root_page) AS total + FROM + `httparchive.crawl.pages` + WHERE + date IN ('2025-07-01', '2024-06-01', '2023-07-01', '2022-06-01', '2021-07-01', '2020-08-01', '2019-07-01') + GROUP BY + date, + client +) + +SELECT + SUBSTR(CAST(date AS STRING), 0, 4) AS year, + client, + IF(feat.feature = 'CSSFlexibleBox', 'flexbox', 'grid') AS layout, + COUNT(DISTINCT root_page) AS freq, + total, + COUNT(DISTINCT root_page) / total AS pct +FROM + `httparchive.crawl.pages`, + UNNEST (features) AS feat +JOIN + totals +USING (date, client) +WHERE + date IN ('2025-07-01', '2024-06-01', '2023-07-01', '2022-06-01', '2021-07-01', '2020-08-01', '2019-07-01') AND + feature IN ('CSSFlexibleBox', 'CSSGridLayout') +GROUP BY + year, + client, + layout, + total +ORDER BY + year DESC, + pct DESC From 7383f00bed4db9df3036b6e250efbddd5c8c17d6 Mon Sep 17 00:00:00 2001 From: "Aaron T. Grogg" Date: Tue, 13 Jan 2026 12:20:02 -0800 Subject: [PATCH 02/13] Add function to analyze CSS color formats This SQL file defines a temporary function to analyze color formats in CSS, including handling various color representations and calculating their usage statistics. --- sql/2025/css/color_alpha_functions.sql | 232 +++++++++++++++++++++++++ 1 file changed, 232 insertions(+) create mode 100644 sql/2025/css/color_alpha_functions.sql diff --git a/sql/2025/css/color_alpha_functions.sql b/sql/2025/css/color_alpha_functions.sql new file mode 100644 index 00000000000..118b3791c12 --- /dev/null +++ b/sql/2025/css/color_alpha_functions.sql @@ -0,0 +1,232 @@ +#standardSQL +CREATE TEMPORARY FUNCTION getColorFormats(css JSON) +RETURNS ARRAY> +LANGUAGE js +OPTIONS (library = "gs://httparchive/lib/css-utils.js") +AS r''' +try { + function compute(ast) { + let usage = { + hex: { + "3": 0, "4": 0, + "6": 0, "8": 0 + }, + functions: {}, + alpha: {}, + keywords: {}, + system: {}, + currentcolor: 0, + transparent: 0, + args: {commas: 0, nocommas: 0}, + spaces: {}, + p3: {sRGB_in: 0, sRGB_out: 0} + }; + + const keywords = [ + "aliceblue", "antiquewhite", "aqua", "aquamarine", "azure", "beige", "bisque", "black", "blanchedalmond", "blue", "blueviolet", "brown", "burlywood", "cadetblue", "chartreuse", + "chocolate", "coral", "cornflowerblue", "cornsilk", "crimson", "cyan", "darkblue", "darkcyan", "darkgoldenrod", "darkgray", "darkgreen", "darkgrey", "darkkhaki", "darkmagenta", + "darkolivegreen", "darkorange", "darkorchid", "darkred", "darksalmon", "darkseagreen", "darkslateblue", "darkslategray", "darkslategrey", "darkturquoise", "darkviolet", + "deeppink", "deepskyblue", "dimgray", "dimgrey", "dodgerblue", "firebrick", "floralwhite", "forestgreen", "fuchsia", "gainsboro", "ghostwhite", "gold", "goldenrod", "gray", + "green", "greenyellow", "grey", "honeydew", "hotpink", "indianred", "indigo", "ivory", "khaki", "lavender", "lavenderblush", "lawngreen", "lemonchiffon", "lightblue", "lightcoral", + "lightcyan", "lightgoldenrodyellow", "lightgray", "lightgreen", "lightgrey", "lightpink", "lightsalmon", "lightseagreen", "lightskyblue", "lightslategray", "lightslategrey", + "lightsteelblue", "lightyellow", "lime", "limegreen", "linen", "magenta", "maroon", "mediumaquamarine", "mediumblue", "mediumorchid", "mediumpurple", "mediumseagreen", + "mediumslateblue", "mediumspringgreen", "mediumturquoise", "mediumvioletred", "midnightblue", "mintcream", "mistyrose", "moccasin", "navajowhite", "navy", "oldlace", + "olive", "olivedrab", "orange", "orangered", "orchid", "palegoldenrod", "palegreen", "paleturquoise", "palevioletred", "papayawhip", "peachpuff", "peru", "pink", "plum", + "powderblue", "purple", "rebeccapurple", "red", "rosybrown", "royalblue", "saddlebrown", "salmon", "sandybrown", "seagreen", "seashell", "sienna", "silver", "skyblue", + "slateblue", "slategray", "slategrey", "snow", "springgreen", "steelblue", "tan", "teal", "thistle", "tomato", "turquoise", "violet", "wheat", "white", "whitesmoke", + "yellow", "yellowgreen" + ]; + + const system = [ + "ActiveBorder", "ActiveCaption", "AppWorkspace", "Background", "ButtonFace", "ButtonHighlight", "ButtonShadow", "ButtonText", "CaptionText", + "GrayText", "Highlight", "HighlightText", "InactiveBorder", "InactiveCaption", "InactiveCaptionText", "InfoBackground", "InfoText", + "Menu", "MenuText", "Scrollbar", "ThreeDDarkShadow", "ThreeDFace", "ThreeDHighlight", "ThreeDLightShadow", "ThreeDShadow", "Window", "WindowFrame", "WindowText" + ]; + + // Lookbehind to prevent matching on e.g. var(--color-red) + const keywordRegex = RegExp(`\\b(? c >= 0 && c <= 1); + } + + // given an array of display-p3 RGB values in range [0-1], + // undo gamma correction to get linear light values + function linearize_p3 (P3) { + return P3.map(val => { + if (val < 0.04045) { + return val / 12.92; + } + return ((val + 0.055) / 1.055) ** 2.4; + }); + } + + // given an array of linear-light display-p3 RGB values in range [0-1], + // convert to CIE XYZ and then to linear-light sRGB + // The two linear operations are combined into a single matrix. + // The matrix multiply is hard-coded, for efficiency + function lin_P3_to_sRGB (linP3) { + let [r, g, b] = linP3; + + return [ + 1.2247452561927687 * r + -0.22490435913073928 * g + 1.8500279863609137e-8 * b, + -0.04205792199232122 * r + 1.0420810071506164 * g + -1.585738278880866e-8 * b, + -0.019642279587426013 * r + -0.07865491660582305 * g + 1.098537193883219 * b + ]; + } + + walkDeclarations(ast, ({property, value}) => { + if (value.length > 1000) return; + // First remove url() references to avoid them mucking the results + for (let f of extractFunctionCalls(value, {names: "url"})) { + let [start, end] = f.pos; + value = value.substring(0, start) + "url()" + " ".repeat(end - start - 5) + value.substring(end); + } + + usage.hex[3] += countMatches(value, /#[a-f0-9]{3}\\b/gi); + usage.hex[4] += countMatches(value, /#[a-f0-9]{4}\\b/gi); + usage.hex[6] += countMatches(value, /#[a-f0-9]{6}\\b/gi); + usage.hex[8] += countMatches(value, /#[a-f0-9]{8}\\b/gi); + + for (let f of extractFunctionCalls(value, {names: functionNames})) { + let {name, args} = f; + + incrementByKey(usage.functions, name); + incrementByKey(usage.args, (args.indexOf(",") > -1? "" : "no") + "commas"); + + switch (name) { + case 'rgba': + case 'hsla': + // The function name implies that they use alpha. + incrementByKey(usage.alpha, name) + break; + case 'rgb': + case 'hsl': + case 'color': + case 'lab': + case 'lch': + case 'hwb': + // Check if the function uses the special "/" syntax or fourth arg for alpha. + if (args.includes('/') || args.trim().split(/[\s+,/]+/).length == 4) { + incrementByKey(usage.alpha, name); + } + break; + } + + if (name === "color") { + // Let's look at color() more closely + let match = args.match(/^(?[\w-]+)\s+(?[-\d\\s.%\/]+)$/); + + if (match) { + let {space, params} = match.groups; + + incrementByKey(usage.spaces, space); + + if (space === "display-p3") { + let percents = params.indexOf("%") > -1; + let coords = params.trim().split(/\s+/).map(c => parseFloat(c) / (percents? 100 : 1)); + + usage.p3["sRGB_" + (P3inSRGB(coords)? "in" : "out")]++; + } + } + } + } + + for (let match of value.matchAll(keywordRegex)) { + incrementByKey(usage.keywords, match[0].toLowerCase()); + } + + for (let match of value.matchAll(systemRegex)) { + incrementByKey(usage.system, system.find(kw => kw.toLowerCase() == match[0].toLowerCase())); + } + + for (let match of value.matchAll(/\b(? Date: Tue, 13 Jan 2026 12:37:58 -0800 Subject: [PATCH 03/13] Add getColorFormats function for CSS color analysis --- sql/2025/css/color_formats.sql | 221 +++++++++++++++++++++++++++++++++ 1 file changed, 221 insertions(+) create mode 100644 sql/2025/css/color_formats.sql diff --git a/sql/2025/css/color_formats.sql b/sql/2025/css/color_formats.sql new file mode 100644 index 00000000000..b1de94f4f55 --- /dev/null +++ b/sql/2025/css/color_formats.sql @@ -0,0 +1,221 @@ +#standardSQL +CREATE TEMPORARY FUNCTION getColorFormats(css JSON) +RETURNS ARRAY> +LANGUAGE js +OPTIONS (library = "gs://httparchive/lib/css-utils.js") +AS ''' +try { + function compute(ast) { + let usage = { + hex: { + "3": 0, "4": 0, + "6": 0, "8": 0 + }, + functions: {}, + keywords: {}, + system: {}, + currentcolor: 0, + transparent: 0, + args: {commas: 0, nocommas: 0}, + spaces: {}, + p3: {sRGB_in: 0, sRGB_out: 0} + }; + + const keywords = [ + "aliceblue", "antiquewhite", "aqua", "aquamarine", "azure", "beige", "bisque", "black", "blanchedalmond", "blue", "blueviolet", "brown", "burlywood", "cadetblue", "chartreuse", + "chocolate", "coral", "cornflowerblue", "cornsilk", "crimson", "cyan", "darkblue", "darkcyan", "darkgoldenrod", "darkgray", "darkgreen", "darkgrey", "darkkhaki", "darkmagenta", + "darkolivegreen", "darkorange", "darkorchid", "darkred", "darksalmon", "darkseagreen", "darkslateblue", "darkslategray", "darkslategrey", "darkturquoise", "darkviolet", + "deeppink", "deepskyblue", "dimgray", "dimgrey", "dodgerblue", "firebrick", "floralwhite", "forestgreen", "fuchsia", "gainsboro", "ghostwhite", "gold", "goldenrod", "gray", + "green", "greenyellow", "grey", "honeydew", "hotpink", "indianred", "indigo", "ivory", "khaki", "lavender", "lavenderblush", "lawngreen", "lemonchiffon", "lightblue", "lightcoral", + "lightcyan", "lightgoldenrodyellow", "lightgray", "lightgreen", "lightgrey", "lightpink", "lightsalmon", "lightseagreen", "lightskyblue", "lightslategray", "lightslategrey", + "lightsteelblue", "lightyellow", "lime", "limegreen", "linen", "magenta", "maroon", "mediumaquamarine", "mediumblue", "mediumorchid", "mediumpurple", "mediumseagreen", + "mediumslateblue", "mediumspringgreen", "mediumturquoise", "mediumvioletred", "midnightblue", "mintcream", "mistyrose", "moccasin", "navajowhite", "navy", "oldlace", + "olive", "olivedrab", "orange", "orangered", "orchid", "palegoldenrod", "palegreen", "paleturquoise", "palevioletred", "papayawhip", "peachpuff", "peru", "pink", "plum", + "powderblue", "purple", "rebeccapurple", "red", "rosybrown", "royalblue", "saddlebrown", "salmon", "sandybrown", "seagreen", "seashell", "sienna", "silver", "skyblue", + "slateblue", "slategray", "slategrey", "snow", "springgreen", "steelblue", "tan", "teal", "thistle", "tomato", "turquoise", "violet", "wheat", "white", "whitesmoke", + "yellow", "yellowgreen" + ]; + + const system = [ + "ActiveBorder", "ActiveCaption", "AppWorkspace", "Background", "ButtonFace", "ButtonHighlight", "ButtonShadow", "ButtonText", "CaptionText", + "GrayText", "Highlight", "HighlightText", "InactiveBorder", "InactiveCaption", "InactiveCaptionText", "InfoBackground", "InfoText", + "Menu", "MenuText", "Scrollbar", "ThreeDDarkShadow", "ThreeDFace", "ThreeDHighlight", "ThreeDLightShadow", "ThreeDShadow", "Window", "WindowFrame", "WindowText" + ]; + + // Lookbehind to prevent matching on e.g. var(--color-red) + const keywordRegex = RegExp(`\\\\b(? c >= 0 && c <= 1); + } + + // given an array of display-p3 RGB values in range [0-1], + // undo gamma correction to get linear light values + function linearize_p3 (P3) { + return P3.map(val => { + if (val < 0.04045) { + return val / 12.92; + } + return ((val + 0.055) / 1.055) ** 2.4; + }); + } + + // given an array of linear-light display-p3 RGB values in range [0-1], + // convert to CIE XYZ and then to linear-light sRGB + // The two linear operations are combined into a single matrix. + // The matrix multiply is hard-coded, for efficiency + function lin_P3_to_sRGB (linP3) { + let [r, g, b] = linP3; + + return [ + 1.2247452561927687 * r + -0.22490435913073928 * g + 1.8500279863609137e-8 * b, + -0.04205792199232122 * r + 1.0420810071506164 * g + -1.585738278880866e-8 * b, + -0.019642279587426013 * r + -0.07865491660582305 * g + 1.098537193883219 * b + ]; + } + + walkDeclarations(ast, ({property, value}) => { + if (value.length > 1000) return; + // First remove url() references to avoid them mucking the results + for (let f of extractFunctionCalls(value, {names: "url"})) { + let [start, end] = f.pos; + value = value.substring(0, start) + "url()" + " ".repeat(end - start - 5) + value.substring(end); + } + + usage.hex[3] += countMatches(value, /#[a-f0-9]{3}\\b/gi); + usage.hex[4] += countMatches(value, /#[a-f0-9]{4}\\b/gi); + usage.hex[6] += countMatches(value, /#[a-f0-9]{6}\\b/gi); + usage.hex[8] += countMatches(value, /#[a-f0-9]{8}\\b/gi); + + for (let f of extractFunctionCalls(value, {names: functionNames})) { + let {name, args} = f; + + incrementByKey(usage.functions, name); + incrementByKey(usage.args, (args.indexOf(",") > -1? "" : "no") + "commas"); + + if (name === "color") { + // Let's look at color() more closely + let match = args.match(/^(?[\\w-]+)\\s+(?[-\\d\\s.%\\/]+)$/); + + if (match) { + let {space, params} = match.groups; + + incrementByKey(usage.spaces, space); + + if (space === "display-p3") { + let percents = params.indexOf("%") > -1; + let coords = params.trim().split(/\\s+/).map(c => parseFloat(c) / (percents? 100 : 1)); + + usage.p3["sRGB_" + (P3inSRGB(coords)? "in" : "out")]++; + } + } + } + } + + for (let match of value.matchAll(keywordRegex)) { + incrementByKey(usage.keywords, match[0].toLowerCase()); + } + + for (let match of value.matchAll(systemRegex)) { + incrementByKey(usage.system, system.find(kw => kw.toLowerCase() == match[0].toLowerCase())); + } + + for (let match of value.matchAll(/\\b(? total += i, 0)}, + {name: 'system', value: Object.values(color.system).reduce((total, i) => total += i, 0)}, + {name: 'currentColor', value: color.currentcolor}, + {name: 'transparent', value: color.transparent} + ]; +} catch (e) { + return []; +} +'''; + +WITH totals AS ( + SELECT + client, + COUNT(0) AS total_pages + FROM + `httparchive.crawl.pages` + WHERE + date = '2025-07-01' AND + rank <= 1000000 AND + is_root_page -- remove if wanna look at home pages AND inner pages. Old tables only had home pages. + GROUP BY + client +) + +SELECT + client, + name AS format, + COUNT(DISTINCT page) AS pages, + ANY_VALUE(total_pages) AS total_pages, + COUNT(DISTINCT page) / ANY_VALUE(total_pages) AS pct_pages, + SUM(value) AS freq, + SUM(SUM(value)) OVER (PARTITION BY client) AS total, + SUM(value) / SUM(SUM(value)) OVER (PARTITION BY client) AS pct +FROM ( + SELECT + client, + page, + format.name, + format.value + FROM + `httparchive.crawl.parsed_css`, + UNNEST(getColorFormats(css)) AS format + WHERE + date = '2025-07-01' AND + rank <= 1000000 AND + format.value IS NOT NULL +) +JOIN + totals +USING (client) +GROUP BY + client, + format +ORDER BY + pct DESC From dfb4646f82b13ccbcd4a2d6df831228396a761b0 Mon Sep 17 00:00:00 2001 From: "Aaron T. Grogg" Date: Tue, 13 Jan 2026 12:44:35 -0800 Subject: [PATCH 04/13] Change getColorKeywords parameter type to JSON --- sql/2025/css/color_keywords.sql | 186 ++++++++++++++++++++++++++++++++ 1 file changed, 186 insertions(+) create mode 100644 sql/2025/css/color_keywords.sql diff --git a/sql/2025/css/color_keywords.sql b/sql/2025/css/color_keywords.sql new file mode 100644 index 00000000000..9d50082b6db --- /dev/null +++ b/sql/2025/css/color_keywords.sql @@ -0,0 +1,186 @@ +#standardSQL +CREATE TEMPORARY FUNCTION getColorKeywords(css JSON) +RETURNS ARRAY> +LANGUAGE js +OPTIONS (library = "gs://httparchive/lib/css-utils.js") +AS ''' +try { + function compute(ast) { + let usage = { + hex: { + "3": 0, "4": 0, + "6": 0, "8": 0 + }, + functions: {}, + keywords: {}, + system: {}, + currentcolor: 0, + transparent: 0, + args: {commas: 0, nocommas: 0}, + spaces: {}, + p3: {sRGB_in: 0, sRGB_out: 0} + }; + + const keywords = [ + "aliceblue", "antiquewhite", "aqua", "aquamarine", "azure", "beige", "bisque", "black", "blanchedalmond", "blue", "blueviolet", "brown", "burlywood", "cadetblue", "chartreuse", + "chocolate", "coral", "cornflowerblue", "cornsilk", "crimson", "cyan", "darkblue", "darkcyan", "darkgoldenrod", "darkgray", "darkgreen", "darkgrey", "darkkhaki", "darkmagenta", + "darkolivegreen", "darkorange", "darkorchid", "darkred", "darksalmon", "darkseagreen", "darkslateblue", "darkslategray", "darkslategrey", "darkturquoise", "darkviolet", + "deeppink", "deepskyblue", "dimgray", "dimgrey", "dodgerblue", "firebrick", "floralwhite", "forestgreen", "fuchsia", "gainsboro", "ghostwhite", "gold", "goldenrod", "gray", + "green", "greenyellow", "grey", "honeydew", "hotpink", "indianred", "indigo", "ivory", "khaki", "lavender", "lavenderblush", "lawngreen", "lemonchiffon", "lightblue", "lightcoral", + "lightcyan", "lightgoldenrodyellow", "lightgray", "lightgreen", "lightgrey", "lightpink", "lightsalmon", "lightseagreen", "lightskyblue", "lightslategray", "lightslategrey", + "lightsteelblue", "lightyellow", "lime", "limegreen", "linen", "magenta", "maroon", "mediumaquamarine", "mediumblue", "mediumorchid", "mediumpurple", "mediumseagreen", + "mediumslateblue", "mediumspringgreen", "mediumturquoise", "mediumvioletred", "midnightblue", "mintcream", "mistyrose", "moccasin", "navajowhite", "navy", "oldlace", + "olive", "olivedrab", "orange", "orangered", "orchid", "palegoldenrod", "palegreen", "paleturquoise", "palevioletred", "papayawhip", "peachpuff", "peru", "pink", "plum", + "powderblue", "purple", "rebeccapurple", "red", "rosybrown", "royalblue", "saddlebrown", "salmon", "sandybrown", "seagreen", "seashell", "sienna", "silver", "skyblue", + "slateblue", "slategray", "slategrey", "snow", "springgreen", "steelblue", "tan", "teal", "thistle", "tomato", "turquoise", "violet", "wheat", "white", "whitesmoke", + "yellow", "yellowgreen" + ]; + + const system = [ + "ActiveBorder", "ActiveCaption", "AppWorkspace", "Background", "ButtonFace", "ButtonHighlight", "ButtonShadow", "ButtonText", "CaptionText", + "GrayText", "Highlight", "HighlightText", "InactiveBorder", "InactiveCaption", "InactiveCaptionText", "InfoBackground", "InfoText", + "Menu", "MenuText", "Scrollbar", "ThreeDDarkShadow", "ThreeDFace", "ThreeDHighlight", "ThreeDLightShadow", "ThreeDShadow", "Window", "WindowFrame", "WindowText" + ]; + + // Lookbehind to prevent matching on e.g. var(--color-red) + const keywordRegex = RegExp(`\\\\b(? c >= 0 && c <= 1); + } + + // given an array of display-p3 RGB values in range [0-1], + // undo gamma correction to get linear light values + function linearize_p3 (P3) { + return P3.map(val => { + if (val < 0.04045) { + return val / 12.92; + } + return ((val + 0.055) / 1.055) ** 2.4; + }); + } + + // given an array of linear-light display-p3 RGB values in range [0-1], + // convert to CIE XYZ and then to linear-light sRGB + // The two linear operations are combined into a single matrix. + // The matrix multiply is hard-coded, for efficiency + function lin_P3_to_sRGB (linP3) { + let [r, g, b] = linP3; + + return [ + 1.2247452561927687 * r + -0.22490435913073928 * g + 1.8500279863609137e-8 * b, + -0.04205792199232122 * r + 1.0420810071506164 * g + -1.585738278880866e-8 * b, + -0.019642279587426013 * r + -0.07865491660582305 * g + 1.098537193883219 * b + ]; + } + + walkDeclarations(ast, ({property, value}) => { + if (value.length > 1000) return; + // First remove url() references to avoid them mucking the results + for (let f of extractFunctionCalls(value, {names: "url"})) { + let [start, end] = f.pos; + value = value.substring(0, start) + "url()" + " ".repeat(end - start - 5) + value.substring(end); + } + + usage.hex[3] += countMatches(value, /#[a-f0-9]{3}\b/gi); + usage.hex[4] += countMatches(value, /#[a-f0-9]{4}\b/gi); + usage.hex[6] += countMatches(value, /#[a-f0-9]{6}\b/gi); + usage.hex[8] += countMatches(value, /#[a-f0-9]{8}\b/gi); + + for (let f of extractFunctionCalls(value, {names: functionNames})) { + let {name, args} = f; + + incrementByKey(usage.functions, name); + incrementByKey(usage.args, (args.indexOf(",") > -1? "" : "no") + "commas"); + + if (name === "color") { + // Let's look at color() more closely + let match = args.match(/^(?[\\w-]+)\\s+(?[-\\d\\s.%\\/]+)$/); + + if (match) { + let {space, params} = match.groups; + + incrementByKey(usage.spaces, space); + + if (space === "display-p3") { + let percents = params.indexOf("%") > -1; + let coords = params.trim().split(/\\s+/).map(c => parseFloat(c) / (percents? 100 : 1)); + + usage.p3["sRGB_" + (P3inSRGB(coords)? "in" : "out")]++; + } + } + } + } + + for (let match of value.matchAll(keywordRegex)) { + incrementByKey(usage.keywords, match[0].toLowerCase()); + } + + for (let match of value.matchAll(systemRegex)) { + incrementByKey(usage.system, system.find(kw => kw.toLowerCase() == match[0].toLowerCase())); + } + + for (let match of value.matchAll(/\\b(? ({name, value}))) + .concat(Object.entries(color.system).map(([name, value]) => ({name, value}))); +} catch (e) { + return []; +} +'''; + +SELECT + client, + name AS keyword, + SUM(value) AS freq, + SUM(SUM(value)) OVER (PARTITION BY client) AS total, + SUM(value) / SUM(SUM(value)) OVER (PARTITION BY client) AS pct +FROM ( + SELECT + client, + keyword.name, + keyword.value + FROM + `httparchive.crawl.parsed_css`, + UNNEST(getColorKeywords(css)) AS keyword + WHERE + date = '2025-07-01' AND + rank <= 1000000 +) +GROUP BY + client, + keyword +ORDER BY + pct DESC From 71350f3be3ee5ba4d5a6950f6442cf679739ba09 Mon Sep 17 00:00:00 2001 From: "Aaron T. Grogg" Date: Tue, 13 Jan 2026 12:52:28 -0800 Subject: [PATCH 05/13] Add function to count color-mix declarations in CSS --- sql/2025/css/color_mix.sql | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 sql/2025/css/color_mix.sql diff --git a/sql/2025/css/color_mix.sql b/sql/2025/css/color_mix.sql new file mode 100644 index 00000000000..5f3efa7129b --- /dev/null +++ b/sql/2025/css/color_mix.sql @@ -0,0 +1,30 @@ +CREATE TEMPORARY FUNCTION countColorMixDeclarations(css JSON) +RETURNS NUMERIC +LANGUAGE js +OPTIONS (library = "gs://httparchive/lib/css-utils.js") +AS r''' +try { + return countDeclarations(css.stylesheet.rules, {values: /color-mix\(.*\)/}); +} catch (e) { + return null; +} +'''; + +SELECT + client, + COUNT(DISTINCT IF(declarations > 0, page, NULL)) AS pages, + COUNT(DISTINCT page) AS total, + COUNT(DISTINCT IF(declarations > 0, page, NULL)) / COUNT(DISTINCT page) AS pct_pages +FROM ( + SELECT + client, + page, + countColorMixDeclarations(css) AS declarations + FROM + `httparchive.crawl.parsed_css` + WHERE + date = '2025-07-01' AND + rank <= 1000000 +) +GROUP BY + client From e1be18782ee9ae0d18597dc07e0c804cc2b12d73 Mon Sep 17 00:00:00 2001 From: "Aaron T. Grogg" Date: Tue, 13 Jan 2026 13:05:00 -0800 Subject: [PATCH 06/13] Create color_p3.sql --- sql/2025/css/color_p3.sql | 204 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 204 insertions(+) create mode 100644 sql/2025/css/color_p3.sql diff --git a/sql/2025/css/color_p3.sql b/sql/2025/css/color_p3.sql new file mode 100644 index 00000000000..d0d14ee08ca --- /dev/null +++ b/sql/2025/css/color_p3.sql @@ -0,0 +1,204 @@ +#standardSQL +CREATE TEMPORARY FUNCTION getP3Usage(css JSON) +RETURNS ARRAY> +LANGUAGE js +OPTIONS (library = "gs://httparchive/lib/css-utils.js") +AS ''' +try { + function compute(ast) { + let usage = { + hex: { + "3": 0, "4": 0, + "6": 0, "8": 0 + }, + functions: {}, + keywords: {}, + system: {}, + currentcolor: 0, + transparent: 0, + args: {commas: 0, nocommas: 0}, + spaces: {}, + p3: {sRGB_in: 0, sRGB_out: 0} + }; + + const keywords = [ + "aliceblue", "antiquewhite", "aqua", "aquamarine", "azure", "beige", "bisque", "black", "blanchedalmond", "blue", "blueviolet", "brown", "burlywood", "cadetblue", "chartreuse", + "chocolate", "coral", "cornflowerblue", "cornsilk", "crimson", "cyan", "darkblue", "darkcyan", "darkgoldenrod", "darkgray", "darkgreen", "darkgrey", "darkkhaki", "darkmagenta", + "darkolivegreen", "darkorange", "darkorchid", "darkred", "darksalmon", "darkseagreen", "darkslateblue", "darkslategray", "darkslategrey", "darkturquoise", "darkviolet", + "deeppink", "deepskyblue", "dimgray", "dimgrey", "dodgerblue", "firebrick", "floralwhite", "forestgreen", "fuchsia", "gainsboro", "ghostwhite", "gold", "goldenrod", "gray", + "green", "greenyellow", "grey", "honeydew", "hotpink", "indianred", "indigo", "ivory", "khaki", "lavender", "lavenderblush", "lawngreen", "lemonchiffon", "lightblue", "lightcoral", + "lightcyan", "lightgoldenrodyellow", "lightgray", "lightgreen", "lightgrey", "lightpink", "lightsalmon", "lightseagreen", "lightskyblue", "lightslategray", "lightslategrey", + "lightsteelblue", "lightyellow", "lime", "limegreen", "linen", "magenta", "maroon", "mediumaquamarine", "mediumblue", "mediumorchid", "mediumpurple", "mediumseagreen", + "mediumslateblue", "mediumspringgreen", "mediumturquoise", "mediumvioletred", "midnightblue", "mintcream", "mistyrose", "moccasin", "navajowhite", "navy", "oldlace", + "olive", "olivedrab", "orange", "orangered", "orchid", "palegoldenrod", "palegreen", "paleturquoise", "palevioletred", "papayawhip", "peachpuff", "peru", "pink", "plum", + "powderblue", "purple", "rebeccapurple", "red", "rosybrown", "royalblue", "saddlebrown", "salmon", "sandybrown", "seagreen", "seashell", "sienna", "silver", "skyblue", + "slateblue", "slategray", "slategrey", "snow", "springgreen", "steelblue", "tan", "teal", "thistle", "tomato", "turquoise", "violet", "wheat", "white", "whitesmoke", + "yellow", "yellowgreen" + ]; + + const system = [ + "ActiveBorder", "ActiveCaption", "AppWorkspace", "Background", "ButtonFace", "ButtonHighlight", "ButtonShadow", "ButtonText", "CaptionText", + "GrayText", "Highlight", "HighlightText", "InactiveBorder", "InactiveCaption", "InactiveCaptionText", "InfoBackground", "InfoText", + "Menu", "MenuText", "Scrollbar", "ThreeDDarkShadow", "ThreeDFace", "ThreeDHighlight", "ThreeDLightShadow", "ThreeDShadow", "Window", "WindowFrame", "WindowText" + ]; + + // Lookbehind to prevent matching on e.g. var(--color-red) + const keywordRegex = RegExp(`\\\\b(? c >= 0 && c <= 1); + } + + // given an array of display-p3 RGB values in range [0-1], + // undo gamma correction to get linear light values + function linearize_p3 (P3) { + return P3.map(val => { + if (val < 0.04045) { + return val / 12.92; + } + return ((val + 0.055) / 1.055) ** 2.4; + }); + } + + // given an array of linear-light display-p3 RGB values in range [0-1], + // convert to CIE XYZ and then to linear-light sRGB + // The two linear operations are combined into a single matrix. + // The matrix multiply is hard-coded, for efficiency + function lin_P3_to_sRGB (linP3) { + let [r, g, b] = linP3; + + return [ + 1.2247452561927687 * r + -0.22490435913073928 * g + 1.8500279863609137e-8 * b, + -0.04205792199232122 * r + 1.0420810071506164 * g + -1.585738278880866e-8 * b, + -0.019642279587426013 * r + -0.07865491660582305 * g + 1.098537193883219 * b + ]; + } + + walkDeclarations(ast, ({property, value}) => { + if (value.length > 1000) return; + // First remove url() references to avoid them mucking the results + for (let f of extractFunctionCalls(value, {names: "url"})) { + let [start, end] = f.pos; + value = value.substring(0, start) + "url()" + " ".repeat(end - start - 5) + value.substring(end); + } + + usage.hex[3] += countMatches(value, /#[a-f0-9]{3}\\b/gi); + usage.hex[4] += countMatches(value, /#[a-f0-9]{4}\\b/gi); + usage.hex[6] += countMatches(value, /#[a-f0-9]{6}\\b/gi); + usage.hex[8] += countMatches(value, /#[a-f0-9]{8}\\b/gi); + + for (let f of extractFunctionCalls(value, {names: functionNames})) { + let {name, args} = f; + + incrementByKey(usage.functions, name); + incrementByKey(usage.args, (args.indexOf(",") > -1? "" : "no") + "commas"); + + if (name === "color") { + // Let's look at color() more closely + let match = args.match(/^(?[\\w-]+)\\s+(?[-\\d\\s.%\\/]+)$/); + + if (match) { + let {space, params} = match.groups; + + incrementByKey(usage.spaces, space); + + if (space === "display-p3") { + let percents = params.indexOf("%") > -1; + let coords = params.trim().split(/\\s+/).map(c => parseFloat(c) / (percents? 100 : 1)); + + usage.p3["sRGB_" + (P3inSRGB(coords)? "in" : "out")]++; + } + } + } + } + + for (let match of value.matchAll(keywordRegex)) { + incrementByKey(usage.keywords, match[0].toLowerCase()); + } + + for (let match of value.matchAll(systemRegex)) { + incrementByKey(usage.system, system.find(kw => kw.toLowerCase() == match[0].toLowerCase())); + } + + for (let match of value.matchAll(/\\b(? ({name, value})); +} catch (e) { + return []; +} +'''; + +WITH totals AS ( + SELECT + client, + COUNT(0) AS total_pages + FROM + `httparchive.crawl.pages` + WHERE + date = '2025-07-01' AND + rank <= 1000000 AND + is_root_page -- remove if wanna look at home pages AND inner pages. Old tables only had home pages. + GROUP BY + client +) + +SELECT + client, + name AS p3, + COUNT(DISTINCT page) AS pages, + ANY_VALUE(total_pages) AS total_pages, + COUNT(DISTINCT page) / ANY_VALUE(total_pages) AS pct_pages, + SUM(value) AS freq, + SUM(SUM(value)) OVER (PARTITION BY client) AS total, + SAFE_DIVIDE(SUM(value), SUM(SUM(value)) OVER (PARTITION BY client)) AS pct +FROM ( + SELECT + client, + page, + p3.name, + p3.value + FROM + `httparchive.crawl.parsed_css`, + UNNEST(getP3Usage(css)) AS p3 + WHERE + date = '2025-07-01' AND + rank <= 1000000 +) +JOIN + totals +USING (client) +GROUP BY + client, + p3 +ORDER BY + pct DESC From b92ab903842252dd14c51c0665710e490daacc38 Mon Sep 17 00:00:00 2001 From: "Aaron T. Grogg" Date: Tue, 13 Jan 2026 14:00:16 -0800 Subject: [PATCH 07/13] Add SQL query for custom property adoption analysis --- sql/2025/css/custom_property_adoption.sql | 59 +++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 sql/2025/css/custom_property_adoption.sql diff --git a/sql/2025/css/custom_property_adoption.sql b/sql/2025/css/custom_property_adoption.sql new file mode 100644 index 00000000000..b1b8494c8f2 --- /dev/null +++ b/sql/2025/css/custom_property_adoption.sql @@ -0,0 +1,59 @@ +#standardSQL +# % of sites that use custom properties. +# Same query as 2019, to compare trend +CREATE TEMPORARY FUNCTION usesCustomProps(css JSON) +RETURNS BOOLEAN LANGUAGE js AS ''' +try { + var reduceValues = (values, rule) => { + if ('rules' in rule) { + return rule.rules.reduce(reduceValues, values); + } + if (!('declarations' in rule)) { + return values; + } + + return values.concat(rule.declarations.filter(d => d.property.startsWith(`--`))); + }; + return css.stylesheet.rules.reduce(reduceValues, []).length > 0; +} catch (e) { + return false; +} +'''; + +SELECT + client, + COUNTIF(num_stylesheets > 0) AS freq, + total, + COUNTIF(num_stylesheets > 0) / total AS pct +FROM ( + SELECT + client, + page, + COUNTIF(usesCustomProps(css)) AS num_stylesheets + FROM + `httparchive.crawl.parsed_css` + WHERE + date = '2025-07-01' AND + rank <= 1000000 AND + is_root_page -- remove if wanna look at home pages AND inner pages. Old tables only had home pages. + GROUP BY + client, + page +) +JOIN ( + SELECT + client, + COUNT(0) AS total + FROM + `httparchive.crawl.pages` + WHERE + date = '2025-07-01' AND + rank <= 1000000 AND + is_root_page -- remove if wanna look at home pages AND inner pages. Old tables only had home pages. + GROUP BY + client +) +USING (client) +GROUP BY + client, + total From 27604e00b712f0cc21f8e414aeffc66a99d1a160 Mon Sep 17 00:00:00 2001 From: "Aaron T. Grogg" Date: Tue, 13 Jan 2026 14:03:53 -0800 Subject: [PATCH 08/13] Add rank filter to SQL query for page ranking --- sql/2025/css/color_alpha_functions.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/sql/2025/css/color_alpha_functions.sql b/sql/2025/css/color_alpha_functions.sql index 118b3791c12..c2d39fd93aa 100644 --- a/sql/2025/css/color_alpha_functions.sql +++ b/sql/2025/css/color_alpha_functions.sql @@ -194,6 +194,7 @@ WITH totals AS ( `httparchive.crawl.pages` WHERE date = '2025-07-01' AND + rank <= 1000000 AND is_root_page -- remove if wanna look at home pages AND inner pages. Old tables only had home pages. GROUP BY client From 897d6452f2cbac97e0f88c4e839645fa7a2a575d Mon Sep 17 00:00:00 2001 From: "Aaron T. Grogg" Date: Tue, 13 Jan 2026 14:48:20 -0800 Subject: [PATCH 09/13] Add SQL function to analyze custom property depths --- sql/2025/css/custom_property_depth.sql | 125 +++++++++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 sql/2025/css/custom_property_depth.sql diff --git a/sql/2025/css/custom_property_depth.sql b/sql/2025/css/custom_property_depth.sql new file mode 100644 index 00000000000..23dd45ceccf --- /dev/null +++ b/sql/2025/css/custom_property_depth.sql @@ -0,0 +1,125 @@ +#standardSQL +CREATE TEMPORARY FUNCTION getCustomPropertyLengths(vars JSON) +RETURNS ARRAY> +LANGUAGE js +OPTIONS (library = "gs://httparchive/lib/css-utils.js") +AS ''' +try { + function compute(vars) { + function walkElements(node, callback, parent) { + if (Array.isArray(node)) { + for (let n of node) { + walkElements(n, callback, parent); + } + } + else { + callback(node, parent); + + if (node.children) { + walkElements(node.children, callback, node); + } + } + } + + let ret = { + depths: {} + }; + + function countDependencyLength(node, property) { + if (!node) { + return 0; + } + + let declarations = node.declarations; + + if (!declarations || !(property in declarations)) { + return countDependencyLength(node.parent, property); + } + + let o = declarations[property]; + + if (!o.references || o.references.length === 0) { + return 0; + } + + let lengths = o.references.map(p => countDependencyLength(node, p)); + + return 1 + Math.max(...lengths); + } + + walkElements(vars.computed, (node, parent) => { + if (parent && !node.parent) { + node.parent = parent; + } + + if (node.declarations) { + for (let property in node.declarations) { + + let o = node.declarations[property]; + if (o.computed && o.computed.trim() !== o.value.trim() && (o.computed === "initial" || o.computed === "null")) { + // Cycle or missing ref + incrementByKey(ret, "cycles_or_initial"); + } + else { + let depth = countDependencyLength(node, property); + + incrementByKey(ret.depths, depth); + } + } + } + }); + + return ret; + } + if (!vars || !vars.computed) return null; + var custom_props = compute(vars); + return Object.entries(custom_props.depths).map(([depth, freq]) => ({depth, freq})) +} catch (e) { + return []; +} +'''; + +WITH totals AS ( + SELECT + client, + COUNT(0) AS total_pages + FROM + `httparchive.crawl.pages` + WHERE + date = '2025-07-01' AND + is_root_page + GROUP BY + client +) + +SELECT + client, + depth, + COUNT(DISTINCT page) AS pages, + ANY_VALUE(total_pages) AS total_pages, + COUNT(DISTINCT page) / ANY_VALUE(total_pages) AS pct_pages, + SUM(freq) AS freq, + SUM(SUM(freq)) OVER (PARTITION BY client) AS total, + SUM(freq) / SUM(SUM(freq)) OVER (PARTITION BY client) AS pct +FROM ( + SELECT + client, + page, + custom_properties.depth, + custom_properties.freq + FROM + `httparchive.crawl.pages`, + UNNEST(getCustomPropertyLengths(custom_metrics.css_variables)) AS custom_properties + WHERE + date = '2025-07-01' AND + is_root_page +) +JOIN + totals +USING (client) +GROUP BY + client, + depth +ORDER BY + depth, + client From 29afbf8d510ec2c570e1f096fe1f20e9cc5d0326 Mon Sep 17 00:00:00 2001 From: "Aaron T. Grogg" Date: Tue, 13 Jan 2026 15:12:11 -0800 Subject: [PATCH 10/13] Add custom property functions SQL script --- sql/2025/css/custom_property_functions.sql | 131 +++++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100644 sql/2025/css/custom_property_functions.sql diff --git a/sql/2025/css/custom_property_functions.sql b/sql/2025/css/custom_property_functions.sql new file mode 100644 index 00000000000..e3f6f7af044 --- /dev/null +++ b/sql/2025/css/custom_property_functions.sql @@ -0,0 +1,131 @@ +#standardSQL +CREATE TEMPORARY FUNCTION getCustomPropertyFunctions(css JSON) +RETURNS ARRAY +LANGUAGE js +OPTIONS (library = "gs://httparchive/lib/css-utils.js") +AS ''' +try { + function compute(ast) { + let ret = { + properties: {}, + functions: {}, + supports: {}, + "pseudo-classes": {}, + fallback: { + none: 0, + literal: 0, + var: 0 + }, + initial: 0 + }; + + walkRules(ast, rule => { + for (let match of rule.supports.matchAll(/\\(--(?[\\w-]+)\\s*:/g)) { + incrementByKey(ret.supports, match.groups.name); + } + }, {type: "supports"}); + + let parsedSelectors = {}; + + walkDeclarations(ast, ({property, value}, rule) => { + if (matches(value, /\\bvar\\(\\s*--/)) { + if (!property.startsWith("--")) { + incrementByKey(ret.properties, property); + } + + for (let call of extractFunctionCalls(value)) { + if (call.name === "var") { + let fallback = call.args.split(",").slice(1).join(","); + + if (matches(fallback, /\\bvar\\(\\s*--/)) { + ret.fallback.var++; + } + else if (fallback) { + ret.fallback.literal++; + } + else { + ret.fallback.none++; + } + } + else if (call.args.includes("var(--")) { + incrementByKey(ret.functions, call.name); + } + } + } + + if (property.startsWith("--")) { + if (value === "initial") { + ret.initial++; + } + + if (rule.selectors) { + for (let selector of rule.selectors) { + let sast = parsedSelectors[selector] = parsedSelectors[selector] || parsel.parse(selector); + parsel.walk(sast, node => { + if (node.type === "pseudo-class") { + incrementByKey(ret["pseudo-classes"], node.name); + } + }) + } + } + + } + }); + + for (let type in ret) { + ret[type] = sortObject(ret[type]); + } + + return ret; + } + + let custom_property = compute(css); + return Object.keys(custom_property.functions); +} catch (e) { + return []; +} +'''; + +SELECT + client, + function, + COUNT(DISTINCT page) AS pages, + total, + COUNT(DISTINCT page) / total AS pct +FROM ( + SELECT DISTINCT + client, + page, + LOWER(function) AS function + FROM + `httparchive.crawl.parsed_css` + LEFT JOIN + UNNEST(getCustomPropertyFunctions(css)) AS function + WHERE + function IS NOT NULL AND + date = '2025-07-01' AND + rank <= 1000000 AND + is_root_page -- remove if wanna look at home pages AND inner pages. Old tables only had home pages. +) +JOIN ( + SELECT + client, + COUNT(0) AS total + FROM + `httparchive.crawl.pages` + WHERE + date = '2025-07-01' AND + rank <= 1000000 AND + is_root_page -- remove if wanna look at home pages AND inner pages. Old tables only had home pages. + GROUP BY + client +) +USING (client) +GROUP BY + client, + total, + function +HAVING + pages >= 100 +ORDER BY + pct DESC From ec7a6998869803a93c07ee7438e56d12ffff7059 Mon Sep 17 00:00:00 2001 From: "Aaron T. Grogg" Date: Tue, 13 Jan 2026 15:17:33 -0800 Subject: [PATCH 11/13] Add rank filter to custom property depth query --- sql/2025/css/custom_property_depth.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sql/2025/css/custom_property_depth.sql b/sql/2025/css/custom_property_depth.sql index 23dd45ceccf..63b16755df5 100644 --- a/sql/2025/css/custom_property_depth.sql +++ b/sql/2025/css/custom_property_depth.sql @@ -87,6 +87,7 @@ WITH totals AS ( `httparchive.crawl.pages` WHERE date = '2025-07-01' AND + rank <= 1000000 AND is_root_page GROUP BY client @@ -112,6 +113,7 @@ FROM ( UNNEST(getCustomPropertyLengths(custom_metrics.css_variables)) AS custom_properties WHERE date = '2025-07-01' AND + rank <= 1000000 AND is_root_page ) JOIN From 5e61effd4c82b4a7ede82e15ef656e8a059999d7 Mon Sep 17 00:00:00 2001 From: "Aaron T. Grogg" Date: Wed, 14 Jan 2026 11:30:55 -0800 Subject: [PATCH 12/13] Making linter happy --- sql/2025/css/custom_property_adoption.sql | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/sql/2025/css/custom_property_adoption.sql b/sql/2025/css/custom_property_adoption.sql index b1b8494c8f2..ff17fb24fee 100644 --- a/sql/2025/css/custom_property_adoption.sql +++ b/sql/2025/css/custom_property_adoption.sql @@ -41,16 +41,16 @@ FROM ( page ) JOIN ( - SELECT - client, - COUNT(0) AS total - FROM - `httparchive.crawl.pages` + SELECT + client, + COUNT(0) AS total + FROM + `httparchive.crawl.pages` WHERE date = '2025-07-01' AND rank <= 1000000 AND is_root_page -- remove if wanna look at home pages AND inner pages. Old tables only had home pages. - GROUP BY + GROUP BY client ) USING (client) From e4f455e4e136af1db65956ba1049165fe06bbf4e Mon Sep 17 00:00:00 2001 From: "Aaron T. Grogg" Date: Wed, 14 Jan 2026 11:31:40 -0800 Subject: [PATCH 13/13] Making linter happy --- sql/2025/css/flexbox_grid.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/2025/css/flexbox_grid.sql b/sql/2025/css/flexbox_grid.sql index baeb22c8635..7d0a0ee4550 100644 --- a/sql/2025/css/flexbox_grid.sql +++ b/sql/2025/css/flexbox_grid.sql @@ -23,7 +23,7 @@ SELECT COUNT(DISTINCT root_page) / total AS pct FROM `httparchive.crawl.pages`, - UNNEST (features) AS feat + UNNEST(features) AS feat JOIN totals USING (date, client)