From 1647eb8c704d9f6e8e22bda639af27889069df1e Mon Sep 17 00:00:00 2001 From: Nurullah Demir Date: Thu, 15 Aug 2024 13:35:22 +0200 Subject: [PATCH 01/53] initial commit --- .../breakdown_of_pages_using_async_defer.sql | 20 ++++ ...ipts_using_async_defer_module_nomodule.sql | 39 ++++++++ sql/2024/javascript/bytes_2019.sql | 15 +++ sql/2024/javascript/bytes_2020.sql | 15 +++ sql/2024/javascript/bytes_2021.sql | 15 +++ sql/2024/javascript/bytes_2022.sql | 15 +++ sql/2024/javascript/bytes_by_3p.sql | 33 +++++++ sql/2024/javascript/compression_method.sql | 47 ++++++++++ .../javascript/compression_method_by_3p.sql | 52 +++++++++++ .../javascript/compression_none_by_bytes.sql | 55 +++++++++++ .../distribution_of_injected_scripts.sql | 56 ++++++++++++ .../distribution_of_long_tasks_time.sql | 56 ++++++++++++ .../distribution_of_number_of_long_tasks.sql | 56 ++++++++++++ sql/2024/javascript/dynamic_import.sql | 16 ++++ sql/2024/javascript/fid.sql | 30 ++++++ .../frameworks_bytes_by_framework.sql | 35 +++++++ sql/2024/javascript/frameworks_libraries.sql | 30 ++++++ .../frameworks_libraries_by_version.sql | 32 +++++++ .../frameworks_libraries_combos.sql | 43 +++++++++ sql/2024/javascript/injected_scripts.sql | 49 ++++++++++ sql/2024/javascript/inp.sql | 30 ++++++ .../javascript/lighthouse_unminified_js.sql | 23 +++++ .../lighthouse_unminified_js_by_3p.sql | 42 +++++++++ .../lighthouse_unminified_js_bytes.sql | 37 ++++++++ .../javascript/lighthouse_vulnerabilities.sql | 22 +++++ .../lighthouse_vulnerable_libraries.sql | 39 ++++++++ sql/2024/javascript/module_and_nomodule.sql | 26 ++++++ .../javascript/percent_of_dns_prefetch.sql | 84 +++++++++++++++++ .../percent_of_dns_prefetch_by_host.sql | 91 +++++++++++++++++++ sql/2024/javascript/percent_of_preconnect.sql | 84 +++++++++++++++++ .../percent_of_preconnect_by_host.sql | 91 +++++++++++++++++++ sql/2024/javascript/posttask_scheduler.sql | 16 ++++ .../javascript/render_blocking_javascript.sql | 33 +++++++ .../render_blocking_javascript_by_rank.sql | 58 ++++++++++++ sql/2024/javascript/requests_2019.sql | 15 +++ sql/2024/javascript/requests_2020.sql | 15 +++ sql/2024/javascript/requests_2021.sql | 15 +++ sql/2024/javascript/requests_2022.sql | 15 +++ sql/2024/javascript/requests_by_3p.sql | 33 +++++++ ...fetch-preload-modulepreload-percentage.sql | 52 +++++++++++ ...ad-prefetch-modulepreload-distribution.sql | 60 ++++++++++++ sql/2024/javascript/resource_hints.sql | 48 ++++++++++ .../javascript/resource_hints_per_page.sql | 56 ++++++++++++ sql/2024/javascript/sourcemap_header.sql | 48 ++++++++++ .../javascript/sourcemap_header_by_3p.sql | 54 +++++++++++ sql/2024/javascript/sourcemaps.sql | 43 +++++++++ sql/2024/javascript/sync_requests.sql | 16 ++++ sql/2024/javascript/tbt.sql | 25 +++++ sql/2024/javascript/time_to_interactive.sql | 19 ++++ .../unused_js_bytes_distribution.sql | 15 +++ .../javascript/usage_of_document_write.sql | 56 ++++++++++++ .../javascript/usage_of_legacy_javascript.sql | 56 ++++++++++++ sql/2024/javascript/usage_of_parcel.sql | 37 ++++++++ .../javascript/usage_of_parcel_by_rank.sql | 61 +++++++++++++ .../usage_of_typescript_and_babel.sql | 47 ++++++++++ .../usage_of_typescript_and_babel_by_rank.sql | 71 +++++++++++++++ sql/2024/javascript/usage_of_webpack.sql | 37 ++++++++ .../javascript/usage_of_webpack_by_rank.sql | 61 +++++++++++++ .../web_components_is_attribute.sql | 33 +++++++ sql/2024/javascript/web_components_pct.sql | 24 +++++ sql/2024/javascript/web_components_specs.sql | 23 +++++ sql/2024/javascript/web_workers.sql | 17 ++++ sql/2024/javascript/worklets.sql | 22 +++++ 63 files changed, 2459 insertions(+) create mode 100644 sql/2024/javascript/breakdown_of_pages_using_async_defer.sql create mode 100644 sql/2024/javascript/breakdown_of_scripts_using_async_defer_module_nomodule.sql create mode 100644 sql/2024/javascript/bytes_2019.sql create mode 100644 sql/2024/javascript/bytes_2020.sql create mode 100644 sql/2024/javascript/bytes_2021.sql create mode 100644 sql/2024/javascript/bytes_2022.sql create mode 100644 sql/2024/javascript/bytes_by_3p.sql create mode 100644 sql/2024/javascript/compression_method.sql create mode 100644 sql/2024/javascript/compression_method_by_3p.sql create mode 100644 sql/2024/javascript/compression_none_by_bytes.sql create mode 100644 sql/2024/javascript/distribution_of_injected_scripts.sql create mode 100644 sql/2024/javascript/distribution_of_long_tasks_time.sql create mode 100644 sql/2024/javascript/distribution_of_number_of_long_tasks.sql create mode 100644 sql/2024/javascript/dynamic_import.sql create mode 100644 sql/2024/javascript/fid.sql create mode 100644 sql/2024/javascript/frameworks_bytes_by_framework.sql create mode 100644 sql/2024/javascript/frameworks_libraries.sql create mode 100644 sql/2024/javascript/frameworks_libraries_by_version.sql create mode 100644 sql/2024/javascript/frameworks_libraries_combos.sql create mode 100644 sql/2024/javascript/injected_scripts.sql create mode 100644 sql/2024/javascript/inp.sql create mode 100644 sql/2024/javascript/lighthouse_unminified_js.sql create mode 100644 sql/2024/javascript/lighthouse_unminified_js_by_3p.sql create mode 100644 sql/2024/javascript/lighthouse_unminified_js_bytes.sql create mode 100644 sql/2024/javascript/lighthouse_vulnerabilities.sql create mode 100644 sql/2024/javascript/lighthouse_vulnerable_libraries.sql create mode 100644 sql/2024/javascript/module_and_nomodule.sql create mode 100644 sql/2024/javascript/percent_of_dns_prefetch.sql create mode 100644 sql/2024/javascript/percent_of_dns_prefetch_by_host.sql create mode 100644 sql/2024/javascript/percent_of_preconnect.sql create mode 100644 sql/2024/javascript/percent_of_preconnect_by_host.sql create mode 100644 sql/2024/javascript/posttask_scheduler.sql create mode 100644 sql/2024/javascript/render_blocking_javascript.sql create mode 100644 sql/2024/javascript/render_blocking_javascript_by_rank.sql create mode 100644 sql/2024/javascript/requests_2019.sql create mode 100644 sql/2024/javascript/requests_2020.sql create mode 100644 sql/2024/javascript/requests_2021.sql create mode 100644 sql/2024/javascript/requests_2022.sql create mode 100644 sql/2024/javascript/requests_by_3p.sql create mode 100644 sql/2024/javascript/resource-hints-prefetch-preload-modulepreload-percentage.sql create mode 100644 sql/2024/javascript/resource-hints-preload-prefetch-modulepreload-distribution.sql create mode 100644 sql/2024/javascript/resource_hints.sql create mode 100644 sql/2024/javascript/resource_hints_per_page.sql create mode 100644 sql/2024/javascript/sourcemap_header.sql create mode 100644 sql/2024/javascript/sourcemap_header_by_3p.sql create mode 100644 sql/2024/javascript/sourcemaps.sql create mode 100644 sql/2024/javascript/sync_requests.sql create mode 100644 sql/2024/javascript/tbt.sql create mode 100644 sql/2024/javascript/time_to_interactive.sql create mode 100644 sql/2024/javascript/unused_js_bytes_distribution.sql create mode 100644 sql/2024/javascript/usage_of_document_write.sql create mode 100644 sql/2024/javascript/usage_of_legacy_javascript.sql create mode 100644 sql/2024/javascript/usage_of_parcel.sql create mode 100644 sql/2024/javascript/usage_of_parcel_by_rank.sql create mode 100644 sql/2024/javascript/usage_of_typescript_and_babel.sql create mode 100644 sql/2024/javascript/usage_of_typescript_and_babel_by_rank.sql create mode 100644 sql/2024/javascript/usage_of_webpack.sql create mode 100644 sql/2024/javascript/usage_of_webpack_by_rank.sql create mode 100644 sql/2024/javascript/web_components_is_attribute.sql create mode 100644 sql/2024/javascript/web_components_pct.sql create mode 100644 sql/2024/javascript/web_components_specs.sql create mode 100644 sql/2024/javascript/web_workers.sql create mode 100644 sql/2024/javascript/worklets.sql diff --git a/sql/2024/javascript/breakdown_of_pages_using_async_defer.sql b/sql/2024/javascript/breakdown_of_pages_using_async_defer.sql new file mode 100644 index 00000000000..ae9176f3e33 --- /dev/null +++ b/sql/2024/javascript/breakdown_of_pages_using_async_defer.sql @@ -0,0 +1,20 @@ +#standardSQL +# Breakdown of scripts using Async, Defer, Module or NoModule attributes. Also breakdown of inline vs external scripts +SELECT + _TABLE_SUFFIX AS client, + COUNTIF(CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.script_tags.async') AS INT64) > 0) AS async, + COUNTIF(CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.script_tags.defer') AS INT64) > 0) AS defer, + COUNTIF(CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.script_tags.async_and_defer') AS INT64) > 0) AS async_and_defer, + COUNTIF(CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.script_tags.type_module') AS INT64) > 0) AS module, + COUNTIF(CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.script_tags.nomodule') AS INT64) > 0) AS nomodule, + COUNTIF(CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.script_tags.async') AS INT64) > 0) / COUNT(0) AS async_pct, + COUNTIF(CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.script_tags.defer') AS INT64) > 0) / COUNT(0) AS defer_pct, + COUNTIF(CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.script_tags.async_and_defer') AS INT64) > 0) / COUNT(0) AS async_and_defer_pct, + COUNTIF(CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.script_tags.type_module') AS INT64) > 0) / COUNT(0) AS module_pct, + COUNTIF(CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.script_tags.nomodule') AS INT64) > 0) / COUNT(0) AS nomodule_pct, + COUNTIF(CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.script_tags.async') AS INT64) = 0 AND CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.script_tags.defer') AS INT64) = 0) AS neither, + COUNTIF(CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.script_tags.async') AS INT64) = 0 AND CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.script_tags.defer') AS INT64) = 0) / COUNT(0) AS neither_pct +FROM + `httparchive.pages.2022_06_01_*` +GROUP BY + client diff --git a/sql/2024/javascript/breakdown_of_scripts_using_async_defer_module_nomodule.sql b/sql/2024/javascript/breakdown_of_scripts_using_async_defer_module_nomodule.sql new file mode 100644 index 00000000000..afa405f6311 --- /dev/null +++ b/sql/2024/javascript/breakdown_of_scripts_using_async_defer_module_nomodule.sql @@ -0,0 +1,39 @@ +#standardSQL +# Breakdown of scripts using Async, Defer, Module or NoModule attributes. Also breakdown of inline vs external scripts +CREATE TEMPORARY FUNCTION getScripts(payload STRING) +RETURNS STRUCT +LANGUAGE js AS ''' +try { + var $ = JSON.parse(payload); + var javascript = JSON.parse($._javascript); + return javascript.script_tags; +} catch (e) { + return {}; +} +'''; + +SELECT + client, + SUM(script.total) AS total_scripts, + SUM(script.inline) AS inline_script, + SUM(script.src) AS external_script, + SUM(script.src) / SUM(script.total) AS pct_external_script, + SUM(script.inline) / SUM(script.total) AS pct_inline_script, + SUM(script.async) AS async, + SUM(script.defer) AS defer, + SUM(script.async_and_defer) AS async_and_defer, + SUM(script.type_module) AS module, + SUM(script.nomodule) AS nomodule, + SUM(script.async) / SUM(script.src) AS pct_external_async, + SUM(script.defer) / SUM(script.src) AS pct_external_defer, + SUM(script.async_and_defer) / SUM(script.src) AS pct_external_async_defer, + SUM(script.type_module) / SUM(script.src) AS pct_external_module, + SUM(script.nomodule) / SUM(script.src) AS pct_external_nomodule +FROM ( + SELECT + _TABLE_SUFFIX AS client, + getScripts(payload) AS script + FROM + `httparchive.pages.2022_06_01_*`) +GROUP BY + client diff --git a/sql/2024/javascript/bytes_2019.sql b/sql/2024/javascript/bytes_2019.sql new file mode 100644 index 00000000000..554e9b5bdfa --- /dev/null +++ b/sql/2024/javascript/bytes_2019.sql @@ -0,0 +1,15 @@ +#standardSQL +# Sum of JS request bytes per page (2019) +SELECT + percentile, + _TABLE_SUFFIX AS client, + APPROX_QUANTILES(bytesJs / 1024, 1000)[OFFSET(percentile * 10)] AS js_kilobytes +FROM + `httparchive.summary_pages.2019_07_01_*`, + UNNEST([10, 25, 50, 75, 90, 100]) AS percentile +GROUP BY + percentile, + client +ORDER BY + client, + percentile diff --git a/sql/2024/javascript/bytes_2020.sql b/sql/2024/javascript/bytes_2020.sql new file mode 100644 index 00000000000..a251649b66f --- /dev/null +++ b/sql/2024/javascript/bytes_2020.sql @@ -0,0 +1,15 @@ +#standardSQL +# Sum of JS request bytes per page (2020) +SELECT + percentile, + _TABLE_SUFFIX AS client, + APPROX_QUANTILES(bytesJs / 1024, 1000)[OFFSET(percentile * 10)] AS js_kilobytes +FROM + `httparchive.summary_pages.2020_08_01_*`, + UNNEST([10, 25, 50, 75, 90, 100]) AS percentile +GROUP BY + percentile, + client +ORDER BY + client, + percentile diff --git a/sql/2024/javascript/bytes_2021.sql b/sql/2024/javascript/bytes_2021.sql new file mode 100644 index 00000000000..70c2ce430c8 --- /dev/null +++ b/sql/2024/javascript/bytes_2021.sql @@ -0,0 +1,15 @@ +#standardSQL +# Sum of JS request bytes per page (2021) +SELECT + percentile, + _TABLE_SUFFIX AS client, + APPROX_QUANTILES(bytesJs / 1024, 1000)[OFFSET(percentile * 10)] AS js_kilobytes +FROM + `httparchive.summary_pages.2021_07_01_*`, + UNNEST([10, 25, 50, 75, 90, 100]) AS percentile +GROUP BY + percentile, + client +ORDER BY + client, + percentile diff --git a/sql/2024/javascript/bytes_2022.sql b/sql/2024/javascript/bytes_2022.sql new file mode 100644 index 00000000000..ed7db881706 --- /dev/null +++ b/sql/2024/javascript/bytes_2022.sql @@ -0,0 +1,15 @@ +#standardSQL +# Sum of JS request bytes per page (2022) +SELECT + percentile, + _TABLE_SUFFIX AS client, + APPROX_QUANTILES(bytesJs / 1024, 1000)[OFFSET(percentile * 10)] AS js_kilobytes +FROM + `httparchive.summary_pages.2022_06_01_*`, + UNNEST([10, 25, 50, 75, 90, 100]) AS percentile +GROUP BY + percentile, + client +ORDER BY + client, + percentile diff --git a/sql/2024/javascript/bytes_by_3p.sql b/sql/2024/javascript/bytes_by_3p.sql new file mode 100644 index 00000000000..c2eb6329230 --- /dev/null +++ b/sql/2024/javascript/bytes_by_3p.sql @@ -0,0 +1,33 @@ +#standardSQL +# Distribution of 1P/3P JS bytes +SELECT + percentile, + client, + host, + APPROX_QUANTILES(kbytes, 1000)[OFFSET(percentile * 10)] AS kbytes +FROM ( + SELECT + client, + page, + IF(NET.HOST(url) IN ( + SELECT domain FROM `httparchive.almanac.third_parties` WHERE date = '2022-06-01' AND category != 'hosting' + ), 'third party', 'first party') AS host, + SUM(respSize) / 1024 AS kbytes + FROM + `httparchive.almanac.requests` + WHERE + date = '2022-06-01' AND + type = 'script' + GROUP BY + client, + page, + host), + UNNEST([10, 25, 50, 75, 90, 100]) AS percentile +GROUP BY + percentile, + client, + host +ORDER BY + client, + percentile, + host diff --git a/sql/2024/javascript/compression_method.sql b/sql/2024/javascript/compression_method.sql new file mode 100644 index 00000000000..09da63c4e53 --- /dev/null +++ b/sql/2024/javascript/compression_method.sql @@ -0,0 +1,47 @@ +#standardSQL +CREATE TEMPORARY FUNCTION getHeader(headers STRING, headername STRING) +RETURNS STRING +DETERMINISTIC +LANGUAGE js AS ''' + const parsed_headers = JSON.parse(headers); + const matching_headers = parsed_headers.filter(h => h.name.toLowerCase() == headername.toLowerCase()); + if (matching_headers.length > 0) { + return matching_headers[0].value; + } + return null; +'''; + +SELECT + client, + compression, + COUNT(DISTINCT page) AS pages, + ANY_VALUE(total_pages) AS total_pages, + COUNT(DISTINCT page) / ANY_VALUE(total_pages) AS pct_pages, + COUNT(0) AS js_requests, + SUM(COUNT(0)) OVER (PARTITION BY client) AS total_js_requests, + COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY client) AS pct_js_requests +FROM ( + SELECT + client, + page, + getHeader(JSON_EXTRACT(payload, '$.response.headers'), 'Content-Encoding') AS compression + FROM + `httparchive.almanac.requests` + WHERE + date = '2022-06-01' AND + type = 'script') +JOIN ( + SELECT + _TABLE_SUFFIX AS client, + COUNT(0) AS total_pages + FROM + `httparchive.summary_pages.2022_06_01_*` + GROUP BY + client) +USING + (client) +GROUP BY + client, + compression +ORDER BY + pct_js_requests DESC diff --git a/sql/2024/javascript/compression_method_by_3p.sql b/sql/2024/javascript/compression_method_by_3p.sql new file mode 100644 index 00000000000..2ad5fe79c61 --- /dev/null +++ b/sql/2024/javascript/compression_method_by_3p.sql @@ -0,0 +1,52 @@ +#standardSQL +CREATE TEMPORARY FUNCTION getHeader(headers STRING, headername STRING) +RETURNS STRING +DETERMINISTIC +LANGUAGE js AS ''' + const parsed_headers = JSON.parse(headers); + const matching_headers = parsed_headers.filter(h => h.name.toLowerCase() == headername.toLowerCase()); + if (matching_headers.length > 0) { + return matching_headers[0].value; + } + return null; +'''; + +SELECT + client, + host, + compression, + COUNT(DISTINCT page) AS pages, + ANY_VALUE(total_pages) AS total_pages, + COUNT(DISTINCT page) / ANY_VALUE(total_pages) AS pct_pages, + COUNT(0) AS js_requests, + SUM(COUNT(0)) OVER (PARTITION BY client, host) AS total, + COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY client, host) AS pct +FROM ( + SELECT + client, + page, + IF(NET.HOST(url) IN ( + SELECT domain FROM `httparchive.almanac.third_parties` WHERE date = '2022-06-01' AND category != 'hosting' + ), 'third party', 'first party') AS host, + getHeader(JSON_EXTRACT(payload, '$.response.headers'), 'Content-Encoding') AS compression + FROM + `httparchive.almanac.requests` + WHERE + date = '2022-06-01' AND + type = 'script') +JOIN ( + SELECT + _TABLE_SUFFIX AS client, + COUNT(0) AS total_pages + FROM + `httparchive.summary_pages.2022_06_01_*` + GROUP BY + client) +USING + (client) +GROUP BY + client, + host, + compression +ORDER BY + pct DESC diff --git a/sql/2024/javascript/compression_none_by_bytes.sql b/sql/2024/javascript/compression_none_by_bytes.sql new file mode 100644 index 00000000000..1a3081cd2ff --- /dev/null +++ b/sql/2024/javascript/compression_none_by_bytes.sql @@ -0,0 +1,55 @@ +#standardSQL +CREATE TEMPORARY FUNCTION getHeader(headers STRING, headername STRING) +RETURNS STRING +DETERMINISTIC +LANGUAGE js AS ''' + const parsed_headers = JSON.parse(headers); + const matching_headers = parsed_headers.filter(h => h.name.toLowerCase() == headername.toLowerCase()); + if (matching_headers.length > 0) { + return matching_headers[0].value; + } + return null; +'''; + +SELECT + client, + host, + IF(kbytes < 100, FLOOR(kbytes / 5) * 5, 100) AS kbytes, + COUNT(DISTINCT page) AS pages, + ANY_VALUE(total_pages) AS total_pages, + COUNT(DISTINCT page) / ANY_VALUE(total_pages) AS pct_pages, + COUNT(0) AS js_requests, + SUM(COUNT(0)) OVER (PARTITION BY client, host) AS total, + COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY client, host) AS pct +FROM ( + SELECT + client, + page, + IF(NET.HOST(url) IN ( + SELECT domain FROM `httparchive.almanac.third_parties` WHERE date = '2022-06-01' AND category != 'hosting' + ), 'third party', 'first party') AS host, + respSize / 1024 AS kbytes + FROM + `httparchive.almanac.requests` + WHERE + date = '2022-06-01' AND + type = 'script' AND + getHeader(JSON_EXTRACT(payload, '$.response.headers'), 'Content-Encoding') IS NULL) +JOIN ( + SELECT + _TABLE_SUFFIX AS client, + COUNT(0) AS total_pages + FROM + `httparchive.summary_pages.2022_06_01_*` + GROUP BY + client) +USING + (client) +GROUP BY + client, + host, + kbytes +ORDER BY + client, + host, + kbytes diff --git a/sql/2024/javascript/distribution_of_injected_scripts.sql b/sql/2024/javascript/distribution_of_injected_scripts.sql new file mode 100644 index 00000000000..ac938f6361a --- /dev/null +++ b/sql/2024/javascript/distribution_of_injected_scripts.sql @@ -0,0 +1,56 @@ +#standardSQL +# Number of injected scripts per page. +# Pages without any scripts are excluded + +# returns the number of injected scripts and inline +CREATE TEMPORARY FUNCTION getScripts(payload STRING) +RETURNS STRUCT +LANGUAGE js AS ''' +try { + const $ = JSON.parse(payload); + const javascript = JSON.parse($._javascript); + + if (javascript) { + // server-generated scripts + const { scripts, inlineScripts } = javascript.document; + + // all scripts including injected scripts + const { inline, src } = javascript.script_tags; + + return ({ + documentScripts: scripts, + documentInline: inlineScripts, + scripts: src, + inline: inline, + injectedScripts: src - scripts, + injectedInline: inline - inlineScripts + }); + } + + return null; +} catch (e) { + return null; +} +'''; + +SELECT + client, + percentile, + APPROX_QUANTILES(IF(scripts.scripts > 0, scripts.injectedScripts / scripts.scripts, 0), 1000)[OFFSET(percentile * 10)] AS pct_injected_scripts_per_page +FROM ( + SELECT + _TABLE_SUFFIX AS client, + url AS page, + getScripts(payload) AS scripts + FROM + `httparchive.pages.2022_06_01_*` +), +UNNEST([10, 25, 50, 75, 90, 100]) AS percentile +WHERE + scripts IS NOT NULL +GROUP BY + percentile, + client +ORDER BY + client, + percentile diff --git a/sql/2024/javascript/distribution_of_long_tasks_time.sql b/sql/2024/javascript/distribution_of_long_tasks_time.sql new file mode 100644 index 00000000000..9234775164c --- /dev/null +++ b/sql/2024/javascript/distribution_of_long_tasks_time.sql @@ -0,0 +1,56 @@ +#standardSQL +# Distribution of number of long tasks per page + +CREATE TEMPORARY FUNCTION getLongTasks(payload STRING) +RETURNS ARRAY +LANGUAGE js AS ''' +try { + const $ = JSON.parse(payload); + const longTasks = $._longTasks; + + if (longTasks) { + return longTasks.map(n => n[1] - n[0]); + } + + return []; + +} catch (e) { + return []; +} +'''; + +WITH long_tasks_pages AS ( + SELECT + _TABLE_SUFFIX AS client, + url, + _longTasks AS long_tasks + FROM + `httparchive.pages.2022_06_01_*`, + UNNEST(getLongTasks(payload)) AS _longTasks +), + +long_tasks_by_page AS ( + SELECT + client, + url AS page, + SUM(long_tasks) AS long_tasks_ms + FROM + long_tasks_pages + GROUP BY + client, + url +) + +SELECT + client, + percentile, + APPROX_QUANTILES(IF(long_tasks_ms IS NOT NULL, long_tasks_ms, 0), 1000)[OFFSET(percentile * 10)] AS long_tasks_ms_per_page +FROM + long_tasks_by_page, + UNNEST([10, 25, 50, 75, 90, 100]) AS percentile +GROUP BY + client, + percentile +ORDER BY + client, + percentile diff --git a/sql/2024/javascript/distribution_of_number_of_long_tasks.sql b/sql/2024/javascript/distribution_of_number_of_long_tasks.sql new file mode 100644 index 00000000000..16bb14763ce --- /dev/null +++ b/sql/2024/javascript/distribution_of_number_of_long_tasks.sql @@ -0,0 +1,56 @@ +#standardSQL +# Distribution of number of long tasks per page + +CREATE TEMPORARY FUNCTION getLongTasks(payload STRING) +RETURNS ARRAY +LANGUAGE js AS ''' +try { + const $ = JSON.parse(payload); + const longTasks = $._longTasks; + + if (longTasks) { + return longTasks.map(n => n[1] - n[0]); + } + + return []; + +} catch (e) { + return []; +} +'''; + +WITH long_tasks_pages AS ( + SELECT + _TABLE_SUFFIX AS client, + url, + _longTasks AS long_tasks + FROM + `httparchive.pages.2022_06_01_*`, + UNNEST(getLongTasks(payload)) AS _longTasks +), + +long_tasks_by_page AS ( + SELECT + client, + url AS page, + COUNT(0) AS long_tasks + FROM + long_tasks_pages + GROUP BY + client, + url +) + +SELECT + client, + percentile, + APPROX_QUANTILES(IF(long_tasks IS NOT NULL, long_tasks, 0), 1000)[OFFSET(percentile * 10)] AS number_of_long_tasks_per_page +FROM + long_tasks_by_page, + UNNEST([10, 25, 50, 75, 90, 100]) AS percentile +GROUP BY + client, + percentile +ORDER BY + client, + percentile diff --git a/sql/2024/javascript/dynamic_import.sql b/sql/2024/javascript/dynamic_import.sql new file mode 100644 index 00000000000..7a865a0aa9a --- /dev/null +++ b/sql/2024/javascript/dynamic_import.sql @@ -0,0 +1,16 @@ +# standardSQL +# usage of dynamic import using blink features from usage counters +SELECT + client, + pct_urls +FROM + `httparchive.blink_features.usage` +WHERE + yyyymmdd = '20220601' AND + feature = 'DynamicImportModuleScript' +GROUP BY + pct_urls, + client +ORDER BY + pct_urls, + client diff --git a/sql/2024/javascript/fid.sql b/sql/2024/javascript/fid.sql new file mode 100644 index 00000000000..92f2aae3dd0 --- /dev/null +++ b/sql/2024/javascript/fid.sql @@ -0,0 +1,30 @@ +#standardSQL +# FID by device + +WITH +base AS ( + SELECT + device, + p75_fid, + p75_fid_origin + FROM + `chrome-ux-report.materialized.device_summary` + WHERE + device IN ('desktop', 'phone') AND + date IN ('2022-06-01') +) + +SELECT + device, + percentile, + APPROX_QUANTILES(p75_fid, 1000 IGNORE NULLS)[OFFSET(percentile * 10)] AS p75_fid, + APPROX_QUANTILES(p75_fid_origin, 1000 IGNORE NULLS)[OFFSET(percentile * 10)] AS p75_fid_origin +FROM + base, + UNNEST([10, 25, 50, 75, 90, 100]) AS percentile +GROUP BY + device, + percentile +ORDER BY + device, + percentile diff --git a/sql/2024/javascript/frameworks_bytes_by_framework.sql b/sql/2024/javascript/frameworks_bytes_by_framework.sql new file mode 100644 index 00000000000..a15cd9398da --- /dev/null +++ b/sql/2024/javascript/frameworks_bytes_by_framework.sql @@ -0,0 +1,35 @@ +#standardSQL +# Sum of JS request bytes per page by framework +SELECT + percentile, + client, + app AS js_framework, + COUNT(DISTINCT page) AS pages, + APPROX_QUANTILES(bytesJs / 1024, 1000)[OFFSET(percentile * 10)] AS js_kilobytes +FROM ( + SELECT + _TABLE_SUFFIX AS client, + url AS page, + bytesJs + FROM + `httparchive.summary_pages.2022_06_01_*`) +JOIN ( + SELECT DISTINCT + _TABLE_SUFFIX AS client, + url AS page, + app + FROM + `httparchive.technologies.2022_06_01_*` + WHERE + category = 'JavaScript frameworks') +USING + (client, page), + UNNEST([10, 25, 50, 75, 90]) AS percentile +GROUP BY + percentile, + client, + js_framework +ORDER BY + client, + percentile, + pages DESC diff --git a/sql/2024/javascript/frameworks_libraries.sql b/sql/2024/javascript/frameworks_libraries.sql new file mode 100644 index 00000000000..288d9972772 --- /dev/null +++ b/sql/2024/javascript/frameworks_libraries.sql @@ -0,0 +1,30 @@ +#standardSQL +# Top JS frameworks and libraries +SELECT + _TABLE_SUFFIX AS client, + category, + app, + COUNT(DISTINCT url) AS pages, + total, + COUNT(DISTINCT url) / total AS pct +FROM + `httparchive.technologies.2022_06_01_*` +JOIN ( + SELECT + _TABLE_SUFFIX, + COUNT(0) AS total + FROM + `httparchive.summary_pages.2022_06_01_*` + GROUP BY + _TABLE_SUFFIX) +USING + (_TABLE_SUFFIX) +WHERE + category IN ('JavaScript frameworks', 'JavaScript libraries') +GROUP BY + client, + category, + app, + total +ORDER BY + pct DESC diff --git a/sql/2024/javascript/frameworks_libraries_by_version.sql b/sql/2024/javascript/frameworks_libraries_by_version.sql new file mode 100644 index 00000000000..8735ddce2d7 --- /dev/null +++ b/sql/2024/javascript/frameworks_libraries_by_version.sql @@ -0,0 +1,32 @@ +#standardSQL +# Top JS frameworks and libraries by version +SELECT + _TABLE_SUFFIX AS client, + category, + app, + info AS version, + COUNT(DISTINCT url) AS pages, + total, + COUNT(DISTINCT url) / total AS pct +FROM + `httparchive.technologies.2022_06_01_*` +JOIN ( + SELECT + _TABLE_SUFFIX, + COUNT(0) AS total + FROM + `httparchive.summary_pages.2022_06_01_*` + GROUP BY + _TABLE_SUFFIX) +USING + (_TABLE_SUFFIX) +WHERE + app IN ('jQuery', 'jQuery Migrate', 'jQuery UI', 'Modernizr', 'FancyBox', 'Slick', 'Lightbox', 'Moment.js', 'Underscore.js', 'Lodash', 'React') +GROUP BY + client, + category, + app, + info, + total +ORDER BY + pct DESC diff --git a/sql/2024/javascript/frameworks_libraries_combos.sql b/sql/2024/javascript/frameworks_libraries_combos.sql new file mode 100644 index 00000000000..7ce666da60f --- /dev/null +++ b/sql/2024/javascript/frameworks_libraries_combos.sql @@ -0,0 +1,43 @@ +#standardSQL +# Top JS frameworks and libraries combinations +SELECT + * +FROM ( + SELECT + client, + apps, + COUNT(DISTINCT page) AS pages, + total, + COUNT(DISTINCT page) / total AS pct + FROM ( + SELECT + _TABLE_SUFFIX AS client, + url AS page, + total, + ARRAY_TO_STRING(ARRAY_AGG(app ORDER BY app), ', ') AS apps + FROM + `httparchive.technologies.2022_06_01_*` + JOIN ( + SELECT + _TABLE_SUFFIX, + COUNT(0) AS total + FROM + `httparchive.summary_pages.2022_06_01_*` + GROUP BY + _TABLE_SUFFIX) + USING + (_TABLE_SUFFIX) + WHERE + category IN ('JavaScript frameworks', 'JavaScript libraries') + GROUP BY + client, + url, + total) + GROUP BY + client, + apps, + total) +WHERE + pages >= 10000 +ORDER BY + pct DESC diff --git a/sql/2024/javascript/injected_scripts.sql b/sql/2024/javascript/injected_scripts.sql new file mode 100644 index 00000000000..8696e065458 --- /dev/null +++ b/sql/2024/javascript/injected_scripts.sql @@ -0,0 +1,49 @@ +#standardSQL +# Number of injected scripts per page. + +# returns the number of injected scripts and inline +CREATE TEMPORARY FUNCTION getScripts(payload STRING) +RETURNS STRUCT +LANGUAGE js AS ''' +try { + const $ = JSON.parse(payload); + const javascript = JSON.parse($._javascript); + + if (javascript) { + // server-generated scripts + const { scripts, inlineScripts } = javascript.document; + + // all scripts including injected scripts + const { inline, src } = javascript.script_tags; + + return ({ + documentScripts: scripts, + documentInline: inlineScripts, + scripts: src, + inline: inline, + injectedScripts: src - scripts, + injectedInline: inline - inlineScripts + }); + } + + return null; +} catch (e) { + return null; +} +'''; + +SELECT + client, + COUNTIF(scripts.injectedScripts > 0) AS has_injected_scripts, + COUNT(0) AS total_pages, + COUNTIF(scripts.injectedScripts > 0) / COUNT(0) AS pct_injected_scripts +FROM ( + SELECT + _TABLE_SUFFIX AS client, + url AS page, + getScripts(payload) AS scripts + FROM + `httparchive.pages.2022_06_01_*` +) +GROUP BY + client diff --git a/sql/2024/javascript/inp.sql b/sql/2024/javascript/inp.sql new file mode 100644 index 00000000000..61f8be138d3 --- /dev/null +++ b/sql/2024/javascript/inp.sql @@ -0,0 +1,30 @@ +#standardSQL +# INP by device + +WITH +base AS ( + SELECT + device, + p75_inp, + p75_inp_origin + FROM + `chrome-ux-report.materialized.device_summary` + WHERE + device IN ('desktop', 'phone') AND + date IN ('2022-06-01') +) + +SELECT + device, + percentile, + APPROX_QUANTILES(p75_inp, 1000 IGNORE NULLS)[OFFSET(percentile * 10)] AS p75_inp, + APPROX_QUANTILES(p75_inp_origin, 1000 IGNORE NULLS)[OFFSET(percentile * 10)] AS p75_inp_origin +FROM + base, + UNNEST([10, 25, 50, 75, 90, 100]) AS percentile +GROUP BY + device, + percentile +ORDER BY + device, + percentile diff --git a/sql/2024/javascript/lighthouse_unminified_js.sql b/sql/2024/javascript/lighthouse_unminified_js.sql new file mode 100644 index 00000000000..3af42232b97 --- /dev/null +++ b/sql/2024/javascript/lighthouse_unminified_js.sql @@ -0,0 +1,23 @@ +#standardSQL +# Pages with unminified JS +SELECT + client, + score, + COUNT(0) AS pages, + SUM(COUNT(0)) OVER (PARTITION BY client) AS total, + COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY client) AS pct +FROM ( + SELECT + _TABLE_SUFFIX AS client, + JSON_EXTRACT_SCALAR(report, "$.audits['unminified-javascript'].score") AS score + FROM + `httparchive.lighthouse.2022_06_01_*` + ) +WHERE + score IS NOT NULL +GROUP BY + client, + score +ORDER BY + client, + score diff --git a/sql/2024/javascript/lighthouse_unminified_js_by_3p.sql b/sql/2024/javascript/lighthouse_unminified_js_by_3p.sql new file mode 100644 index 00000000000..a9094a92ffd --- /dev/null +++ b/sql/2024/javascript/lighthouse_unminified_js_by_3p.sql @@ -0,0 +1,42 @@ +#standardSQL +# Pages with unminified JS by 1P/3P +CREATE TEMPORARY FUNCTION getUnminifiedJsUrls(audit STRING) +RETURNS ARRAY> LANGUAGE js AS ''' +try { + var $ = JSON.parse(audit); + return $.details.items.map(({url, wastedBytes}) => { + return {url, wastedBytes}; + }); +} catch (e) { + return []; +} +'''; + +SELECT + client, + AVG(pct_1p_wasted_bytes) AS avg_pct_1p_wasted_bytes, + AVG(pct_3p_wasted_bytes) AS avg_pct_3p_wasted_bytes +FROM ( + SELECT + client, + page, + SUM(IF(is_3p, 0, wasted_bytes)) / SUM(wasted_bytes) AS pct_1p_wasted_bytes, + SUM(IF(is_3p, wasted_bytes, 0)) / SUM(wasted_bytes) AS pct_3p_wasted_bytes + FROM ( + SELECT + _TABLE_SUFFIX AS client, + lighthouse.url AS page, + NET.HOST(unminified.url) IS NOT NULL AND NET.HOST(unminified.url) IN ( + SELECT domain FROM `httparchive.almanac.third_parties` WHERE date = '2022-06-01' AND category != 'hosting' + ) AS is_3p, + unminified.wastedBytes AS wasted_bytes + FROM + `httparchive.lighthouse.2022_06_01_*` AS lighthouse, + UNNEST(getUnminifiedJsUrls(JSON_EXTRACT(report, "$.audits['unminified-javascript']"))) AS unminified + ) + GROUP BY + client, + page + ) +GROUP BY + client diff --git a/sql/2024/javascript/lighthouse_unminified_js_bytes.sql b/sql/2024/javascript/lighthouse_unminified_js_bytes.sql new file mode 100644 index 00000000000..1555efc80e2 --- /dev/null +++ b/sql/2024/javascript/lighthouse_unminified_js_bytes.sql @@ -0,0 +1,37 @@ +#standardSQL +# Histogram of wasted bytes per page +CREATE TEMPORARY FUNCTION getUnminifiedJsBytes(audit STRING) +RETURNS ARRAY LANGUAGE js AS ''' +try { + var $ = JSON.parse(audit); + return $.details.items.map(({wastedBytes}) => wastedBytes); +} catch (e) { + return []; +} +'''; + +SELECT + client, + IF(unminified_js_kbytes <= 200, CEIL(unminified_js_kbytes / 10) * 10, 200) AS unminified_js_kbytes, + COUNT(0) AS pages, + SUM(COUNT(0)) OVER (PARTITION BY client) AS total, + COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY client) AS pct +FROM ( + SELECT + _TABLE_SUFFIX AS client, + lighthouse.url AS page, + SUM(IFNULL(unminified_js_bytes, 0)) / 1024 AS unminified_js_kbytes + FROM + `httparchive.lighthouse.2022_06_01_*` AS lighthouse + LEFT JOIN + UNNEST(getUnminifiedJsBytes(JSON_EXTRACT(report, "$.audits['unminified-javascript']"))) AS unminified_js_bytes + GROUP BY + client, + page + ) +GROUP BY + client, + unminified_js_kbytes +ORDER BY + client, + unminified_js_kbytes diff --git a/sql/2024/javascript/lighthouse_vulnerabilities.sql b/sql/2024/javascript/lighthouse_vulnerabilities.sql new file mode 100644 index 00000000000..37992a5ae3b --- /dev/null +++ b/sql/2024/javascript/lighthouse_vulnerabilities.sql @@ -0,0 +1,22 @@ +#standardSQL +# Pages with vulnerable libraries +SELECT + client, + score, + COUNT(0) AS pages, + SUM(COUNT(0)) OVER (PARTITION BY client) AS total, + COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY client) AS pct +FROM ( + SELECT + _TABLE_SUFFIX AS client, + JSON_EXTRACT_SCALAR(report, "$.audits['no-vulnerable-libraries'].score") AS score + FROM + `httparchive.lighthouse.2022_06_01_*`) +WHERE + score IS NOT NULL +GROUP BY + client, + score +ORDER BY + client, + score diff --git a/sql/2024/javascript/lighthouse_vulnerable_libraries.sql b/sql/2024/javascript/lighthouse_vulnerable_libraries.sql new file mode 100644 index 00000000000..28f6f14d403 --- /dev/null +++ b/sql/2024/javascript/lighthouse_vulnerable_libraries.sql @@ -0,0 +1,39 @@ +#standardSQL +# Most frequent vulnerable libraries +CREATE TEMPORARY FUNCTION getVulnerabilities(audit STRING) +RETURNS ARRAY LANGUAGE js AS ''' +try { + var $ = JSON.parse(audit); + return $.details.items.map(i => i.detectedLib.text.split('@')[0]); +} catch(e) { + return []; +} +'''; + +SELECT + _TABLE_SUFFIX AS client, + lib, + COUNT(0) AS freq, + total, + COUNT(0) / total AS pct +FROM + `httparchive.lighthouse.2022_06_01_*` AS lighthouse, + UNNEST(getVulnerabilities(JSON_EXTRACT(report, "$.audits['no-vulnerable-libraries']"))) AS lib +JOIN ( + SELECT + _TABLE_SUFFIX AS client, + COUNT(DISTINCT url) AS total + FROM + `httparchive.lighthouse.2022_06_01_*` + GROUP BY + client +) +ON + lighthouse._TABLE_SUFFIX = client +GROUP BY + client, + lib, + total +ORDER BY + client, + freq DESC diff --git a/sql/2024/javascript/module_and_nomodule.sql b/sql/2024/javascript/module_and_nomodule.sql new file mode 100644 index 00000000000..76a5df03a1b --- /dev/null +++ b/sql/2024/javascript/module_and_nomodule.sql @@ -0,0 +1,26 @@ +SELECT + client, + COUNT(DISTINCT IF(module, page, NULL)) AS module, + COUNT(DISTINCT IF(nomodule, page, NULL)) AS nomodule, + COUNT(DISTINCT IF(nomodule AND module, page, NULL)) AS both, + COUNT(DISTINCT page) AS total, + COUNT(DISTINCT IF(module, page, NULL)) / COUNT(DISTINCT page) AS pct_module, + COUNT(DISTINCT IF(nomodule, page, NULL)) / COUNT(DISTINCT page) AS pct_nomodule, + COUNT(DISTINCT IF(module AND nomodule, page, NULL)) / COUNT(DISTINCT page) AS pct_both +FROM ( + SELECT + client, + page, + script, + REGEXP_CONTAINS(script, r'(?i)\bmodule\b') AS module, + REGEXP_CONTAINS(script, r'(?i)\bnomodule\b') AS nomodule + FROM + `httparchive.almanac.summary_response_bodies` + LEFT JOIN + UNNEST(REGEXP_EXTRACT_ALL(body, r'(?i)(]*>)')) AS script + WHERE + date = '2022-06-01' AND + firstHtml + ) +GROUP BY + client diff --git a/sql/2024/javascript/percent_of_dns_prefetch.sql b/sql/2024/javascript/percent_of_dns_prefetch.sql new file mode 100644 index 00000000000..7e5b22829e2 --- /dev/null +++ b/sql/2024/javascript/percent_of_dns_prefetch.sql @@ -0,0 +1,84 @@ +#standardSQL +# Percentage of requests for script files which are preceded by a dns-prefetch + +CREATE TEMPORARY FUNCTION getResourceHintsHrefs(payload STRING, hint STRING) +RETURNS ARRAY +LANGUAGE js AS ''' +try { + var $ = JSON.parse(payload); + var almanac = JSON.parse($._almanac); + return almanac['link-nodes'].nodes.filter(link => link.rel.toLowerCase() == hint).map(n => n.href); +} catch (e) { + return []; +} +'''; + +WITH resource_hints AS ( + SELECT + client, + page, + host + FROM ( + SELECT + _TABLE_SUFFIX AS client, + url AS page, + NET.HOST(href) AS host + FROM + `httparchive.pages.2022_06_01_*`, + UNNEST(getResourceHintsHrefs(payload, 'dns-prefetch')) AS href + ) + GROUP BY + client, + page, + host +), + +third_party_domains AS ( + SELECT DISTINCT + NET.HOST(domain) AS host + FROM + `httparchive.almanac.third_parties` +), + +requests AS ( + SELECT + client, + page, + NET.HOST(url) AS host + FROM + `httparchive.almanac.requests` + INNER JOIN + third_party_domains + ON + (third_party_domains.host = NET.HOST(url)) + WHERE + date = '2022-06-01' AND + type = 'script' + GROUP BY + client, + page, + host +) + +SELECT + client, + freq, + total, + pct +FROM ( + SELECT + client, + COUNTIF(resource_hints.host IS NOT NULL) AS freq, + SUM(COUNT(0)) OVER (PARTITION BY client) AS total, + COUNTIF(resource_hints.host IS NOT NULL) / SUM(COUNT(0)) OVER (PARTITION BY client) AS pct + FROM + requests + LEFT OUTER JOIN + resource_hints + USING + (client, page, host) + GROUP BY + client +) +ORDER BY + client diff --git a/sql/2024/javascript/percent_of_dns_prefetch_by_host.sql b/sql/2024/javascript/percent_of_dns_prefetch_by_host.sql new file mode 100644 index 00000000000..fecc2ee4052 --- /dev/null +++ b/sql/2024/javascript/percent_of_dns_prefetch_by_host.sql @@ -0,0 +1,91 @@ +#standardSQL +# Percentage of requests for script files which are preceded by a dns-prefetch + +CREATE TEMPORARY FUNCTION getResourceHintsHrefs(payload STRING, hint STRING) +RETURNS ARRAY +LANGUAGE js AS ''' +try { + var $ = JSON.parse(payload); + var almanac = JSON.parse($._almanac); + return almanac['link-nodes'].nodes.filter(link => link.rel.toLowerCase() == hint).map(n => n.href); +} catch (e) { + return []; +} +'''; + +WITH resource_hints AS ( + SELECT + client, + page, + host + FROM ( + SELECT + _TABLE_SUFFIX AS client, + url AS page, + NET.HOST(href) AS host + FROM + `httparchive.pages.2022_06_01_*`, + UNNEST(getResourceHintsHrefs(payload, 'dns-prefetch')) AS href + ) + GROUP BY + client, + page, + host +), + +third_party_domains AS ( + SELECT DISTINCT + NET.HOST(domain) AS host + FROM + `httparchive.almanac.third_parties` +), + +requests AS ( + SELECT + client, + page, + NET.HOST(url) AS host + FROM + `httparchive.almanac.requests` + INNER JOIN + third_party_domains + ON + (third_party_domains.host = NET.HOST(url)) + WHERE + date = '2022-06-01' AND + type = 'script' + GROUP BY + client, + page, + host +) + +SELECT + client, + host, + freq, + total, + pct +FROM ( + SELECT + client, + host, + COUNTIF(resource_hints.host IS NOT NULL) AS freq, + SUM(COUNT(0)) OVER (PARTITION BY client, host) AS total, + COUNTIF(resource_hints.host IS NOT NULL) / SUM(COUNT(0)) OVER (PARTITION BY client, host) AS pct, + RANK() OVER (PARTITION BY client ORDER BY COUNTIF(resource_hints.host IS NOT NULL) DESC) AS resource_hint_rank + FROM + requests + LEFT OUTER JOIN + resource_hints + USING + (client, page, host) + GROUP BY + client, + host +) +WHERE + resource_hint_rank < 100 +ORDER BY + client, + total DESC diff --git a/sql/2024/javascript/percent_of_preconnect.sql b/sql/2024/javascript/percent_of_preconnect.sql new file mode 100644 index 00000000000..74686f1778e --- /dev/null +++ b/sql/2024/javascript/percent_of_preconnect.sql @@ -0,0 +1,84 @@ +#standardSQL +# Percentage of requests for script files which are preceded by a preconnect + +CREATE TEMPORARY FUNCTION getResourceHintsHrefs(payload STRING, hint STRING) +RETURNS ARRAY +LANGUAGE js AS ''' +try { + var $ = JSON.parse(payload); + var almanac = JSON.parse($._almanac); + return almanac['link-nodes'].nodes.filter(link => link.rel.toLowerCase() == hint).map(n => n.href); +} catch (e) { + return []; +} +'''; + +WITH resource_hints AS ( + SELECT + client, + page, + host + FROM ( + SELECT + _TABLE_SUFFIX AS client, + url AS page, + NET.HOST(href) AS host + FROM + `httparchive.pages.2022_06_01_*`, + UNNEST(getResourceHintsHrefs(payload, 'preconnect')) AS href + ) + GROUP BY + client, + page, + host +), + +third_party_domains AS ( + SELECT DISTINCT + NET.HOST(domain) AS host + FROM + `httparchive.almanac.third_parties` +), + +requests AS ( + SELECT + client, + page, + NET.HOST(url) AS host + FROM + `httparchive.almanac.requests` + INNER JOIN + third_party_domains + ON + (third_party_domains.host = NET.HOST(url)) + WHERE + date = '2022-06-01' AND + type = 'script' + GROUP BY + client, + page, + host +) + +SELECT + client, + freq, + total, + pct +FROM ( + SELECT + client, + COUNTIF(resource_hints.host IS NOT NULL) AS freq, + SUM(COUNT(0)) OVER (PARTITION BY client) AS total, + COUNTIF(resource_hints.host IS NOT NULL) / SUM(COUNT(0)) OVER (PARTITION BY client) AS pct + FROM + requests + LEFT OUTER JOIN + resource_hints + USING + (client, page, host) + GROUP BY + client +) +ORDER BY + client diff --git a/sql/2024/javascript/percent_of_preconnect_by_host.sql b/sql/2024/javascript/percent_of_preconnect_by_host.sql new file mode 100644 index 00000000000..5a85ec875b2 --- /dev/null +++ b/sql/2024/javascript/percent_of_preconnect_by_host.sql @@ -0,0 +1,91 @@ +#standardSQL +# Percentage of requests for script files which are preceded by a preconnect + +CREATE TEMPORARY FUNCTION getResourceHintsHrefs(payload STRING, hint STRING) +RETURNS ARRAY +LANGUAGE js AS ''' +try { + var $ = JSON.parse(payload); + var almanac = JSON.parse($._almanac); + return almanac['link-nodes'].nodes.filter(link => link.rel.toLowerCase() == hint).map(n => n.href); +} catch (e) { + return []; +} +'''; + +WITH resource_hints AS ( + SELECT + client, + page, + host + FROM ( + SELECT + _TABLE_SUFFIX AS client, + url AS page, + NET.HOST(href) AS host + FROM + `httparchive.pages.2022_06_01_*`, + UNNEST(getResourceHintsHrefs(payload, 'preconnect')) AS href + ) + GROUP BY + client, + page, + host +), + +third_party_domains AS ( + SELECT DISTINCT + NET.HOST(domain) AS host + FROM + `httparchive.almanac.third_parties` +), + +requests AS ( + SELECT + client, + page, + NET.HOST(url) AS host + FROM + `httparchive.almanac.requests` + INNER JOIN + third_party_domains + ON + (third_party_domains.host = NET.HOST(url)) + WHERE + date = '2022-06-01' AND + type = 'script' + GROUP BY + client, + page, + host +) + +SELECT + client, + host, + freq, + total, + pct +FROM ( + SELECT + client, + host, + COUNTIF(resource_hints.host IS NOT NULL) AS freq, + SUM(COUNT(0)) OVER (PARTITION BY client, host) AS total, + COUNTIF(resource_hints.host IS NOT NULL) / SUM(COUNT(0)) OVER (PARTITION BY client, host) AS pct, + RANK() OVER (PARTITION BY client ORDER BY COUNTIF(resource_hints.host IS NOT NULL) DESC) AS resource_hint_rank + FROM + requests + LEFT OUTER JOIN + resource_hints + USING + (client, page, host) + GROUP BY + client, + host +) +WHERE + resource_hint_rank < 100 +ORDER BY + client, + total DESC diff --git a/sql/2024/javascript/posttask_scheduler.sql b/sql/2024/javascript/posttask_scheduler.sql new file mode 100644 index 00000000000..42ef918c323 --- /dev/null +++ b/sql/2024/javascript/posttask_scheduler.sql @@ -0,0 +1,16 @@ +# standardSQL +# usage of PostTask API using blink features from usage counters +SELECT + client, + pct_urls +FROM + `httparchive.blink_features.usage` +WHERE + yyyymmdd = '20220601' AND + feature = 'SchedulerPostTask' +GROUP BY + pct_urls, + client +ORDER BY + pct_urls, + client diff --git a/sql/2024/javascript/render_blocking_javascript.sql b/sql/2024/javascript/render_blocking_javascript.sql new file mode 100644 index 00000000000..8b469b62d3d --- /dev/null +++ b/sql/2024/javascript/render_blocking_javascript.sql @@ -0,0 +1,33 @@ +#standardSQL +# Percent of pages using render-blocking JavaScript. + +CREATE TEMPORARY FUNCTION getRenderBlockingScripts(payload STRING) +RETURNS INT64 +LANGUAGE js AS ''' +try { + var $ = JSON.parse(payload); + var renderBlockingJS = $._renderBlockingJS; + return renderBlockingJS; +} catch (e) { + return 0; +} +'''; + +WITH render_blocking_scripts AS ( + SELECT + _TABLE_SUFFIX AS client, + url, + getRenderBlockingScripts(payload) AS number_of_render_blocking_scripts + FROM + `httparchive.pages.2022_06_01_*` +) + +SELECT + client, + COUNTIF(number_of_render_blocking_scripts > 0) AS pages_with_render_blocking_scripts, + COUNT(0) AS total_pages, + COUNTIF(number_of_render_blocking_scripts > 0) / COUNT(0) AS pct_pages_with_render_blocking_scripts +FROM + render_blocking_scripts +GROUP BY + client diff --git a/sql/2024/javascript/render_blocking_javascript_by_rank.sql b/sql/2024/javascript/render_blocking_javascript_by_rank.sql new file mode 100644 index 00000000000..312fd09bb6c --- /dev/null +++ b/sql/2024/javascript/render_blocking_javascript_by_rank.sql @@ -0,0 +1,58 @@ +#standardSQL +# Percent of pages using render-blocking JavaScript. + +CREATE TEMPORARY FUNCTION getRenderBlockingScripts(payload STRING) +RETURNS INT64 +LANGUAGE js AS ''' +try { + var $ = JSON.parse(payload); + var renderBlockingJS = $._renderBlockingJS; + return renderBlockingJS; +} catch (e) { + return 0; +} +'''; + +WITH render_blocking_scripts AS ( + SELECT + client, + url, + rank, + number_of_render_blocking_scripts + FROM ( + SELECT + _TABLE_SUFFIX AS client, + url, + getRenderBlockingScripts(payload) AS number_of_render_blocking_scripts + FROM + `httparchive.pages.2022_06_01_*` + ) + JOIN ( + SELECT + _TABLE_SUFFIX AS client, + url, + rank + FROM + `httparchive.summary_pages.2022_06_01_*` + ) + USING + (client, url) +) + +SELECT + client, + rank_grouping, + COUNTIF(number_of_render_blocking_scripts > 0) AS pages_with_render_blocking_scripts, + COUNT(0) AS total_pages, + COUNTIF(number_of_render_blocking_scripts > 0) / COUNT(0) AS pct_pages_with_render_blocking_scripts +FROM + render_blocking_scripts, + UNNEST([1000, 10000, 100000, 1000000, 10000000]) AS rank_grouping +WHERE + rank <= rank_grouping +GROUP BY + client, + rank_grouping +ORDER BY + client, + rank_grouping diff --git a/sql/2024/javascript/requests_2019.sql b/sql/2024/javascript/requests_2019.sql new file mode 100644 index 00000000000..293939c5e7b --- /dev/null +++ b/sql/2024/javascript/requests_2019.sql @@ -0,0 +1,15 @@ +#standardSQL +# Sum of JS requests per page (2019) +SELECT + percentile, + _TABLE_SUFFIX AS client, + APPROX_QUANTILES(reqJS, 1000)[OFFSET(percentile * 10)] AS js_requests +FROM + `httparchive.summary_pages.2019_07_01_*`, + UNNEST([10, 25, 50, 75, 90, 100]) AS percentile +GROUP BY + percentile, + client +ORDER BY + client, + percentile diff --git a/sql/2024/javascript/requests_2020.sql b/sql/2024/javascript/requests_2020.sql new file mode 100644 index 00000000000..12785430a68 --- /dev/null +++ b/sql/2024/javascript/requests_2020.sql @@ -0,0 +1,15 @@ +#standardSQL +# Sum of JS requests per page (2020) +SELECT + percentile, + _TABLE_SUFFIX AS client, + APPROX_QUANTILES(reqJS, 1000)[OFFSET(percentile * 10)] AS js_requests +FROM + `httparchive.summary_pages.2020_08_01_*`, + UNNEST([10, 25, 50, 75, 90, 100]) AS percentile +GROUP BY + percentile, + client +ORDER BY + client, + percentile diff --git a/sql/2024/javascript/requests_2021.sql b/sql/2024/javascript/requests_2021.sql new file mode 100644 index 00000000000..c6ca85c79ca --- /dev/null +++ b/sql/2024/javascript/requests_2021.sql @@ -0,0 +1,15 @@ +#standardSQL +# Sum of JS requests per page (2021) +SELECT + percentile, + _TABLE_SUFFIX AS client, + APPROX_QUANTILES(reqJS, 1000)[OFFSET(percentile * 10)] AS js_requests +FROM + `httparchive.summary_pages.2021_07_01_*`, + UNNEST([10, 25, 50, 75, 90, 100]) AS percentile +GROUP BY + percentile, + client +ORDER BY + client, + percentile diff --git a/sql/2024/javascript/requests_2022.sql b/sql/2024/javascript/requests_2022.sql new file mode 100644 index 00000000000..55c9977d5f6 --- /dev/null +++ b/sql/2024/javascript/requests_2022.sql @@ -0,0 +1,15 @@ +#standardSQL +# Sum of JS requests per page (2022) +SELECT + percentile, + _TABLE_SUFFIX AS client, + APPROX_QUANTILES(reqJS, 1000)[OFFSET(percentile * 10)] AS js_requests +FROM + `httparchive.summary_pages.2022_06_01_*`, + UNNEST([10, 25, 50, 75, 90, 100]) AS percentile +GROUP BY + percentile, + client +ORDER BY + client, + percentile diff --git a/sql/2024/javascript/requests_by_3p.sql b/sql/2024/javascript/requests_by_3p.sql new file mode 100644 index 00000000000..b4f04717aa1 --- /dev/null +++ b/sql/2024/javascript/requests_by_3p.sql @@ -0,0 +1,33 @@ +#standardSQL +# Distribution of 1P/3P JS requests +SELECT + percentile, + client, + host, + APPROX_QUANTILES(requests, 1000)[OFFSET(percentile * 10)] AS num_requests +FROM ( + SELECT + client, + page, + IF(NET.HOST(url) IN ( + SELECT domain FROM `httparchive.almanac.third_parties` WHERE date = '2022-06-01' AND category != 'hosting' + ), 'third party', 'first party') AS host, + COUNT(0) AS requests + FROM + `httparchive.almanac.requests` + WHERE + date = '2022-06-01' AND + type = 'script' + GROUP BY + client, + page, + host), + UNNEST([10, 25, 50, 75, 90, 100]) AS percentile +GROUP BY + percentile, + client, + host +ORDER BY + client, + percentile, + host diff --git a/sql/2024/javascript/resource-hints-prefetch-preload-modulepreload-percentage.sql b/sql/2024/javascript/resource-hints-prefetch-preload-modulepreload-percentage.sql new file mode 100644 index 00000000000..6ed7cb9765e --- /dev/null +++ b/sql/2024/javascript/resource-hints-prefetch-preload-modulepreload-percentage.sql @@ -0,0 +1,52 @@ +#standardSQL +CREATE TEMPORARY FUNCTION getResourceHintAttrs(payload STRING) +RETURNS ARRAY> +LANGUAGE js AS ''' +var hints = new Set(['preload', 'prefetch', 'modulepreload']); +var attributes = ['as']; +try { + var $ = JSON.parse(payload); + var almanac = JSON.parse($._almanac); + return almanac['link-nodes'].nodes.reduce((results, link) => { + var hint = link.rel.toLowerCase(); + if (!hints.has(hint)) { + return results; + } + attributes.forEach(attribute => { + var value = link[attribute]; + results.push({ + name: hint, + attribute: attribute, + // Support empty strings. + value: typeof value == 'string' ? value : null + }); + }); + return results; + }, []); +} catch (e) { + return []; +} +'''; + +SELECT + client, + COUNT(DISTINCT IF(prefetch_hint, page, NULL)) AS prefetch_pages, + COUNT(DISTINCT page) AS total, + COUNT(DISTINCT IF(prefetch_hint, page, NULL)) / COUNT(DISTINCT page) AS prefetch_pct, + COUNT(DISTINCT IF(preload_hint, page, NULL)) AS preload_pages, + COUNT(DISTINCT IF(preload_hint, page, NULL)) / COUNT(DISTINCT page) AS preload_pct, + COUNT(DISTINCT IF(modulepreload_hint, page, NULL)) AS modulepreload_pages, + COUNT(DISTINCT IF(modulepreload_hint, page, NULL)) / COUNT(DISTINCT page) AS modulepreload_pct +FROM ( + SELECT + _TABLE_SUFFIX AS client, + url AS page, + hint.name = 'prefetch' AND hint.value = 'script' AS prefetch_hint, + hint.name = 'preload' AND hint.value = 'script' AS preload_hint, + hint.name = 'modulepreload' AND hint.value = 'script' AS modulepreload_hint + FROM + `httparchive.pages.2022_06_01_*` + LEFT JOIN + UNNEST(getResourceHintAttrs(payload)) AS hint) +GROUP BY + client diff --git a/sql/2024/javascript/resource-hints-preload-prefetch-modulepreload-distribution.sql b/sql/2024/javascript/resource-hints-preload-prefetch-modulepreload-distribution.sql new file mode 100644 index 00000000000..e806dd0e191 --- /dev/null +++ b/sql/2024/javascript/resource-hints-preload-prefetch-modulepreload-distribution.sql @@ -0,0 +1,60 @@ +#standardSQL +CREATE TEMPORARY FUNCTION getResourceHintAttrs(payload STRING) +RETURNS ARRAY> +LANGUAGE js AS ''' +var hints = new Set(['preload', 'prefetch', 'modulepreload']); +var attributes = ['as']; +try { + var $ = JSON.parse(payload); + var almanac = JSON.parse($._almanac); + return almanac['link-nodes'].nodes.reduce((results, link) => { + var hint = link.rel.toLowerCase(); + if (!hints.has(hint)) { + return results; + } + attributes.forEach(attribute => { + var value = link[attribute]; + results.push({ + name: hint, + attribute: attribute, + // Support empty strings. + value: typeof value == 'string' ? value : null + }); + }); + return results; + }, []); +} catch (e) { + return []; +} +'''; + +SELECT + percentile, + client, + APPROX_QUANTILES(prefetch_hint, 1000)[OFFSET(percentile * 10)] AS prefetch_hints_per_page, + APPROX_QUANTILES(IF(prefetch_hint = 0, NULL, prefetch_hint), 1000 IGNORE NULLS)[OFFSET(percentile * 10)] AS prefetch_hints_per_page_with_hints, + APPROX_QUANTILES(preload_hint, 1000)[OFFSET(percentile * 10)] AS preload_hints_per_page, + APPROX_QUANTILES(IF(preload_hint = 0, NULL, preload_hint), 1000 IGNORE NULLS)[OFFSET(percentile * 10)] AS preload_hints_per_page_with_hints, + APPROX_QUANTILES(modulepreload_hint, 1000)[OFFSET(percentile * 10)] AS modulepreload_hints_per_page, + APPROX_QUANTILES(IF(modulepreload_hint = 0, NULL, modulepreload_hint), 1000 IGNORE NULLS)[OFFSET(percentile * 10)] AS modulepreload_hints_per_page_with_hints +FROM ( + SELECT + _TABLE_SUFFIX AS client, + url AS page, + COUNTIF(hint.name = 'prefetch' AND hint.value = 'script') AS prefetch_hint, + COUNTIF(hint.name = 'preload' AND hint.value = 'script') AS preload_hint, + COUNTIF(hint.name = 'modulepreload' AND hint.value = 'script') AS modulepreload_hint + FROM + `httparchive.pages.2022_06_01_*` + LEFT JOIN + UNNEST(getResourceHintAttrs(payload)) AS hint + GROUP BY + client, + page), + UNNEST([10, 25, 50, 75, 90, 100]) AS percentile +GROUP BY + percentile, + client +ORDER BY + client, + percentile diff --git a/sql/2024/javascript/resource_hints.sql b/sql/2024/javascript/resource_hints.sql new file mode 100644 index 00000000000..73b236c045c --- /dev/null +++ b/sql/2024/javascript/resource_hints.sql @@ -0,0 +1,48 @@ +#standardSQL +# Returns the number of pages which have preload, prefetch or modulepreload for scripts + +CREATE TEMPORARY FUNCTION getResourceHintAttrs(payload STRING) +RETURNS ARRAY> +LANGUAGE js AS ''' +var hints = new Set(['preload', 'prefetch', 'modulepreload']); +var attributes = ['as']; +try { + var $ = JSON.parse(payload); + var almanac = JSON.parse($._almanac); + return almanac['link-nodes'].nodes.reduce((results, link) => { + var hint = link.rel.toLowerCase(); + if (!hints.has(hint)) { + return results; + } + attributes.forEach(attribute => { + var value = link[attribute]; + results.push({ + name: hint, + attribute: attribute, + // Support empty strings. + value: typeof value == 'string' ? value : null + }); + }); + return results; + }, []); +} catch (e) { + return []; +} +'''; + +SELECT + client, + COUNT(DISTINCT IF(script_hint, page, NULL)) AS pages, + COUNT(DISTINCT page) AS total, + COUNT(DISTINCT IF(script_hint, page, NULL)) / COUNT(DISTINCT page) AS pct +FROM ( + SELECT + _TABLE_SUFFIX AS client, + url AS page, + hint.name IN ('prefetch', 'preload', 'modulepreload') AND hint.value = 'script' AS script_hint + FROM + `httparchive.pages.2022_06_01_*` + LEFT JOIN + UNNEST(getResourceHintAttrs(payload)) AS hint) +GROUP BY + client diff --git a/sql/2024/javascript/resource_hints_per_page.sql b/sql/2024/javascript/resource_hints_per_page.sql new file mode 100644 index 00000000000..14686eac410 --- /dev/null +++ b/sql/2024/javascript/resource_hints_per_page.sql @@ -0,0 +1,56 @@ +#standardSQL +# Returns the number of resource hints per page which are preload, prefetch or modulepreload + +CREATE TEMPORARY FUNCTION getResourceHintAttrs(payload STRING) +RETURNS ARRAY> +LANGUAGE js AS ''' +var hints = new Set(['preload', 'prefetch', 'modulepreload']); +var attributes = ['as']; +try { + var $ = JSON.parse(payload); + var almanac = JSON.parse($._almanac); + return almanac['link-nodes'].nodes.reduce((results, link) => { + var hint = link.rel.toLowerCase(); + if (!hints.has(hint)) { + return results; + } + attributes.forEach(attribute => { + var value = link[attribute]; + results.push({ + name: hint, + attribute: attribute, + // Support empty strings. + value: typeof value == 'string' ? value : null + }); + }); + return results; + }, []); +} catch (e) { + return []; +} +'''; + +SELECT + percentile, + client, + APPROX_QUANTILES(script_hint, 1000)[OFFSET(percentile * 10)] AS hints_per_page, + APPROX_QUANTILES(IF(script_hint = 0, NULL, script_hint), 1000 IGNORE NULLS)[OFFSET(percentile * 10)] AS hints_per_page_with_hints +FROM ( + SELECT + _TABLE_SUFFIX AS client, + url AS page, + COUNTIF(hint.name IN ('prefetch', 'preload', 'modulepreload') AND hint.value = 'script') AS script_hint + FROM + `httparchive.pages.2022_06_01_*` + LEFT JOIN + UNNEST(getResourceHintAttrs(payload)) AS hint + GROUP BY + client, + page), + UNNEST([10, 25, 50, 75, 90, 100]) AS percentile +GROUP BY + percentile, + client +ORDER BY + client, + percentile diff --git a/sql/2024/javascript/sourcemap_header.sql b/sql/2024/javascript/sourcemap_header.sql new file mode 100644 index 00000000000..93b59b5b4f6 --- /dev/null +++ b/sql/2024/javascript/sourcemap_header.sql @@ -0,0 +1,48 @@ +#standardSQL +CREATE TEMPORARY FUNCTION getHeader(headers STRING, headername STRING) +RETURNS STRING +DETERMINISTIC +LANGUAGE js AS ''' + const parsed_headers = JSON.parse(headers); + const matching_headers = parsed_headers.filter(h => h.name.toLowerCase() == headername.toLowerCase()); + if (matching_headers.length > 0) { + return matching_headers[0].value; + } + return null; +'''; + +SELECT + client, + has_sourcemap_header, + COUNT(DISTINCT page) AS pages, + ANY_VALUE(total_pages) AS total_pages, + COUNT(DISTINCT page) / ANY_VALUE(total_pages) AS pct_pages, + COUNT(0) AS js_requests, + SUM(COUNT(0)) OVER (PARTITION BY client) AS total_js_requests, + COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY client) AS pct_js_requests +FROM ( + SELECT + client, + page, + getHeader(JSON_EXTRACT(payload, '$.response.headers'), 'SourceMap') IS NOT NULL AS has_sourcemap_header + FROM + `httparchive.almanac.requests` + WHERE + date = '2022-06-01' AND + type = 'script') +JOIN ( + SELECT + _TABLE_SUFFIX AS client, + COUNT(0) AS total_pages + FROM + `httparchive.summary_pages.2022_06_01_*` + GROUP BY + client) +USING + (client) +GROUP BY + client, + has_sourcemap_header +ORDER BY + client, + has_sourcemap_header diff --git a/sql/2024/javascript/sourcemap_header_by_3p.sql b/sql/2024/javascript/sourcemap_header_by_3p.sql new file mode 100644 index 00000000000..7937b123806 --- /dev/null +++ b/sql/2024/javascript/sourcemap_header_by_3p.sql @@ -0,0 +1,54 @@ +#standardSQL +CREATE TEMPORARY FUNCTION getHeader(headers STRING, headername STRING) +RETURNS STRING +DETERMINISTIC +LANGUAGE js AS ''' + const parsed_headers = JSON.parse(headers); + const matching_headers = parsed_headers.filter(h => h.name.toLowerCase() == headername.toLowerCase()); + if (matching_headers.length > 0) { + return matching_headers[0].value; + } + return null; +'''; + +SELECT + client, + host, + has_sourcemap_header, + COUNT(DISTINCT page) AS pages, + ANY_VALUE(total_pages) AS total_pages, + COUNT(DISTINCT page) / ANY_VALUE(total_pages) AS pct_pages, + COUNT(0) AS js_requests, + SUM(COUNT(0)) OVER (PARTITION BY client) AS total_js_requests, + COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY client) AS pct_js_requests +FROM ( + SELECT + client, + page, + IF(NET.HOST(url) IN ( + SELECT domain FROM `httparchive.almanac.third_parties` WHERE date = '2022-06-01' AND category != 'hosting' + ), 'third party', 'first party') AS host, + getHeader(JSON_EXTRACT(payload, '$.response.headers'), 'SourceMap') IS NOT NULL AS has_sourcemap_header + FROM + `httparchive.almanac.requests` + WHERE + date = '2022-06-01' AND + type = 'script') +JOIN ( + SELECT + _TABLE_SUFFIX AS client, + COUNT(0) AS total_pages + FROM + `httparchive.summary_pages.2022_06_01_*` + GROUP BY + client) +USING + (client) +GROUP BY + client, + host, + has_sourcemap_header +ORDER BY + client, + host, + has_sourcemap_header diff --git a/sql/2024/javascript/sourcemaps.sql b/sql/2024/javascript/sourcemaps.sql new file mode 100644 index 00000000000..e9182b37199 --- /dev/null +++ b/sql/2024/javascript/sourcemaps.sql @@ -0,0 +1,43 @@ +#standardSQL +# Number of pages with publicly available sourcemaps + +# returns boolean whether the page has sourcemaps or not +CREATE TEMPORARY FUNCTION getSourceMaps(payload STRING) +RETURNS STRUCT +LANGUAGE js AS ''' +try { + const $ = JSON.parse(payload); + const javascript = JSON.parse($._javascript); + + if (javascript && javascript.sourceMaps) { + const { sourceMaps } = javascript; + + return ({ + hasSourceMaps: Boolean(sourceMaps.count), + isPublic: Boolean(Object.keys(sourceMaps.ext).length) + }); + } + + return {}; +} catch (e) { + return {}; +} +'''; + +SELECT + client, + COUNTIF(sourcemaps.hasSourceMaps = true) AS has_sourcemaps, + COUNTIF(sourcemaps.isPublic = true) AS has_public_sourcemaps, + COUNT(0) AS total_pages, + COUNTIF(sourcemaps.hasSourceMaps = true) / COUNT(0) AS pct_has_sourcemaps, + COUNTIF(sourcemaps.isPublic = true) / COUNT(0) AS pct_has_public_sourcemaps +FROM ( + SELECT + _TABLE_SUFFIX AS client, + url AS page, + getSourceMaps(payload) AS sourcemaps + FROM + `httparchive.pages.2022_06_01_*` +) +GROUP BY + client diff --git a/sql/2024/javascript/sync_requests.sql b/sql/2024/javascript/sync_requests.sql new file mode 100644 index 00000000000..c8daf5c2b2f --- /dev/null +++ b/sql/2024/javascript/sync_requests.sql @@ -0,0 +1,16 @@ +# standardSQL +# usage of synchronous XMLHttpRequest using blink features from usage counters +SELECT + client, + pct_urls +FROM + `httparchive.blink_features.usage` +WHERE + yyyymmdd = '20220601' AND + feature = 'XMLHttpRequestSynchronous' +GROUP BY + pct_urls, + client +ORDER BY + pct_urls, + client diff --git a/sql/2024/javascript/tbt.sql b/sql/2024/javascript/tbt.sql new file mode 100644 index 00000000000..4226d4d3cd6 --- /dev/null +++ b/sql/2024/javascript/tbt.sql @@ -0,0 +1,25 @@ +#standardSQL +# TBT + +WITH tbt_stats AS ( + SELECT + _TABLE_SUFFIX AS client, + url, + CAST(JSON_EXTRACT_SCALAR(report, '$.audits.total-blocking-time.numericValue') AS FLOAT64) AS tbtValue + FROM + `httparchive.lighthouse.2022_06_01_*` +) + +SELECT + client, + percentile, + APPROX_QUANTILES(tbtValue, 1000)[OFFSET(percentile * 10)] AS tbt +FROM + tbt_stats, + UNNEST([10, 25, 50, 75, 90, 100]) AS percentile +GROUP BY + client, + percentile +ORDER BY + client, + percentile diff --git a/sql/2024/javascript/time_to_interactive.sql b/sql/2024/javascript/time_to_interactive.sql new file mode 100644 index 00000000000..3f58208f7f0 --- /dev/null +++ b/sql/2024/javascript/time_to_interactive.sql @@ -0,0 +1,19 @@ +#standardSQL +# Percentiles of time to interactive +SELECT + client, + percentile, + ROUND(APPROX_QUANTILES(tti, 1000)[OFFSET(percentile * 10)] / 1000, 2) AS tti +FROM ( + SELECT + _TABLE_SUFFIX AS client, + CAST(IFNULL(JSON_EXTRACT(report, '$.audits.consistently-interactive.numericValue'), JSON_EXTRACT(report, '$.audits.interactive.numericValue')) AS FLOAT64) AS tti + FROM + `httparchive.lighthouse.2022_06_01_*`), + UNNEST([10, 25, 50, 75, 90]) AS percentile +GROUP BY + client, + percentile +ORDER BY + client, + percentile diff --git a/sql/2024/javascript/unused_js_bytes_distribution.sql b/sql/2024/javascript/unused_js_bytes_distribution.sql new file mode 100644 index 00000000000..2d5aeefbb30 --- /dev/null +++ b/sql/2024/javascript/unused_js_bytes_distribution.sql @@ -0,0 +1,15 @@ +#standardSQL +# Distribution of unused JS request bytes per page +SELECT + _TABLE_SUFFIX AS client, + percentile, + APPROX_QUANTILES(CAST(JSON_EXTRACT_SCALAR(report, '$.audits.unused-javascript.details.overallSavingsBytes') AS INT64) / 1024, 1000)[OFFSET(percentile * 10)] AS js_kilobytes +FROM + `httparchive.lighthouse.2022_06_01_*`, + UNNEST([10, 25, 50, 75, 90, 100]) AS percentile +GROUP BY + client, + percentile +ORDER BY + client, + percentile diff --git a/sql/2024/javascript/usage_of_document_write.sql b/sql/2024/javascript/usage_of_document_write.sql new file mode 100644 index 00000000000..7d3fa1fa056 --- /dev/null +++ b/sql/2024/javascript/usage_of_document_write.sql @@ -0,0 +1,56 @@ +#standardSQL +# Third-parties that use legacy JavaScript + +CREATE TEMPORARY FUNCTION +getUrls(audit STRING) +RETURNS ARRAY> LANGUAGE js AS ''' +try { + var $ = JSON.parse(audit); + return $.details.items.map(i => ({url: i.url})); +} catch(e) { + return []; +} +'''; + +WITH base AS ( + SELECT + client, + page + FROM + ( + SELECT + _TABLE_SUFFIX AS client, + lighthouse.url AS page + FROM + `httparchive.lighthouse.2022_06_01_*` AS lighthouse, + UNNEST(getUrls(JSON_EXTRACT(report, "$.audits['no-document-write']"))) AS data + ) + GROUP BY + client, + page +) + +SELECT + base.client AS client, + COUNT(0) AS freq, + total, + COUNT(0) / total AS pct +FROM + base +JOIN ( + SELECT + _TABLE_SUFFIX AS client, + COUNT(DISTINCT url) AS total + FROM + `httparchive.lighthouse.2022_06_01_*` + GROUP BY + client +) +USING + (client) +GROUP BY + client, + total +ORDER BY + client, + freq DESC diff --git a/sql/2024/javascript/usage_of_legacy_javascript.sql b/sql/2024/javascript/usage_of_legacy_javascript.sql new file mode 100644 index 00000000000..0b608280a12 --- /dev/null +++ b/sql/2024/javascript/usage_of_legacy_javascript.sql @@ -0,0 +1,56 @@ +#standardSQL +# Third-parties that use legacy JavaScript + +CREATE TEMPORARY FUNCTION +getUrls(audit STRING) +RETURNS ARRAY> LANGUAGE js AS ''' +try { + var $ = JSON.parse(audit); + return $.details.items.map(i => ({url: i.url})); +} catch(e) { + return []; +} +'''; + +WITH base AS ( + SELECT + client, + page + FROM + ( + SELECT + _TABLE_SUFFIX AS client, + lighthouse.url AS page + FROM + `httparchive.lighthouse.2022_06_01_*` AS lighthouse, + UNNEST(getUrls(JSON_EXTRACT(report, "$.audits['legacy-javascript']"))) AS data + ) + GROUP BY + client, + page +) + +SELECT + base.client AS client, + COUNT(0) AS freq, + total, + COUNT(0) / total AS pct +FROM + base +JOIN ( + SELECT + _TABLE_SUFFIX AS client, + COUNT(DISTINCT url) AS total + FROM + `httparchive.lighthouse.2022_06_01_*` + GROUP BY + client +) +USING + (client) +GROUP BY + client, + total +ORDER BY + client, + freq DESC diff --git a/sql/2024/javascript/usage_of_parcel.sql b/sql/2024/javascript/usage_of_parcel.sql new file mode 100644 index 00000000000..798cdd057b1 --- /dev/null +++ b/sql/2024/javascript/usage_of_parcel.sql @@ -0,0 +1,37 @@ +#standardSQL +# Percent of pages using parcel + +WITH totals AS ( + SELECT + _TABLE_SUFFIX AS client, + COUNT(DISTINCT url) AS total_pages + FROM + `httparchive.summary_pages.2022_06_01_*` + GROUP BY + client +), + +parcel AS ( + SELECT + _TABLE_SUFFIX AS client, + COUNT(DISTINCT url) AS parcel_pages + FROM + `httparchive.technologies.2022_06_01_*` + WHERE + app = 'parcel' + GROUP BY + client +) + +SELECT + client, + parcel_pages, + total_pages, + parcel_pages / total_pages AS pct_parcel_pages +FROM + totals +JOIN + parcel +USING (client) +ORDER BY + client diff --git a/sql/2024/javascript/usage_of_parcel_by_rank.sql b/sql/2024/javascript/usage_of_parcel_by_rank.sql new file mode 100644 index 00000000000..f216a9883a0 --- /dev/null +++ b/sql/2024/javascript/usage_of_parcel_by_rank.sql @@ -0,0 +1,61 @@ +#standardSQL +# Percent of pages using parcel grouped by rank + +WITH parcel_pages AS ( + SELECT + _TABLE_SUFFIX AS client, + url AS page + FROM + `httparchive.technologies.2022_06_01_*` + WHERE + app = 'parcel' +), + +rank_totals AS ( + SELECT + _TABLE_SUFFIX AS client, + rank_grouping, + COUNT(0) AS total + FROM + `httparchive.summary_pages.2022_06_01_*`, + UNNEST([1000, 10000, 100000, 1000000, 10000000]) AS rank_grouping + WHERE + rank <= rank_grouping + GROUP BY + client, + rank_grouping +), + +pages AS ( + SELECT + _TABLE_SUFFIX AS client, + rank_grouping, + url AS page + FROM + `httparchive.summary_pages.2022_06_01_*`, + UNNEST([1000, 10000, 100000, 1000000, 10000000]) AS rank_grouping + WHERE + rank <= rank_grouping +) + +SELECT + client, + rank_grouping AS rank, + COUNT(DISTINCT parcel_pages.page) AS count_parcel_pages, + total, + COUNT(DISTINCT parcel_pages.page) / total AS pct_parcel_pages +FROM + parcel_pages +LEFT OUTER JOIN + pages +USING (client, page) +JOIN + rank_totals +USING (client, rank_grouping) +GROUP BY + client, + total, + rank_grouping +ORDER BY + client, + rank_grouping diff --git a/sql/2024/javascript/usage_of_typescript_and_babel.sql b/sql/2024/javascript/usage_of_typescript_and_babel.sql new file mode 100644 index 00000000000..70cc9e3723b --- /dev/null +++ b/sql/2024/javascript/usage_of_typescript_and_babel.sql @@ -0,0 +1,47 @@ +#standardSQL +# Number of pages using TypeScript or Babel + +# returns boolean whether the page uses Babel or TypeScript +CREATE TEMPORARY FUNCTION getSourceMaps(payload STRING) +RETURNS STRUCT +LANGUAGE js AS ''' +try { + const $ = JSON.parse(payload); + const javascript = JSON.parse($._javascript); + + if (javascript && javascript.sourceMaps) { + const { sourceMaps } = javascript; + + return ({ + hasSourceMaps: Boolean(sourceMaps.count), + isPublic: Boolean(Object.keys(sourceMaps.ext).length), + isTypeScript: Boolean(sourceMaps.ext && (sourceMaps.ext.ts || sourceMaps.ext.tsx)), + isBabel: Boolean(sourceMaps.babel) + }); + } + + return {}; +} catch (e) { + return {}; +} +'''; + +SELECT + client, + COUNTIF(sourcemaps.isBabel = true) AS use_babel, + COUNTIF(sourcemaps.isTypeScript = true) AS use_typescript, + COUNT(0) AS total_pages_with_sourcemaps, + COUNTIF(sourcemaps.isBabel = true) / COUNT(0) AS pct_use_babel, + COUNTIF(sourcemaps.isTypeScript = true) / COUNT(0) AS pct_use_typescript +FROM ( + SELECT + _TABLE_SUFFIX AS client, + url AS page, + getSourceMaps(payload) AS sourcemaps + FROM + `httparchive.pages.2022_06_01_*` +) +WHERE + sourcemaps.isPublic = true +GROUP BY + client diff --git a/sql/2024/javascript/usage_of_typescript_and_babel_by_rank.sql b/sql/2024/javascript/usage_of_typescript_and_babel_by_rank.sql new file mode 100644 index 00000000000..0f1566e569a --- /dev/null +++ b/sql/2024/javascript/usage_of_typescript_and_babel_by_rank.sql @@ -0,0 +1,71 @@ +#standardSQL +# Number of pages using TypeScript or Babel grouped by rank + +# returns boolean whether the page uses Babel or TypeScript +CREATE TEMPORARY FUNCTION getSourceMaps(payload STRING) +RETURNS STRUCT +LANGUAGE js AS ''' +try { + const $ = JSON.parse(payload); + const javascript = JSON.parse($._javascript); + + if (javascript && javascript.sourceMaps) { + const { sourceMaps } = javascript; + + return ({ + hasSourceMaps: Boolean(sourceMaps.count), + isPublic: Boolean(Object.keys(sourceMaps.ext).length), + isTypeScript: Boolean(sourceMaps.ext && (sourceMaps.ext.ts || sourceMaps.ext.tsx)), + isBabel: Boolean(sourceMaps.babel) + }); + } + + return {}; +} catch (e) { + return {}; +} +'''; + +WITH pages AS ( + SELECT + _TABLE_SUFFIX AS client, + rank_grouping, + url AS page + FROM + `httparchive.summary_pages.2022_06_01_*`, + UNNEST([1000, 10000, 100000, 1000000, 10000000]) AS rank_grouping + WHERE + rank <= rank_grouping +), + +pages_sourcemaps AS ( + SELECT + _TABLE_SUFFIX AS client, + url AS page, + getSourceMaps(payload) AS sourcemaps + FROM + `httparchive.pages.2022_06_01_*` +) + +SELECT + client, + rank_grouping AS rank, + COUNTIF(sourcemaps.isBabel = true) AS use_babel, + COUNTIF(sourcemaps.isTypeScript = true) AS use_typescript, + COUNT(0) AS total_pages_with_sourcemaps, + COUNTIF(sourcemaps.isBabel = true) / COUNT(0) AS pct_use_babel, + COUNTIF(sourcemaps.isTypeScript = true) / COUNT(0) AS pct_use_typescript +FROM + pages_sourcemaps +JOIN + pages +USING + (client, page) +WHERE + sourcemaps.isPublic = true +GROUP BY + client, + rank_grouping +ORDER BY + client, + rank_grouping diff --git a/sql/2024/javascript/usage_of_webpack.sql b/sql/2024/javascript/usage_of_webpack.sql new file mode 100644 index 00000000000..4f2e08e8e3a --- /dev/null +++ b/sql/2024/javascript/usage_of_webpack.sql @@ -0,0 +1,37 @@ +#standardSQL +# Percent of pages using webpack + +WITH totals AS ( + SELECT + _TABLE_SUFFIX AS client, + COUNT(DISTINCT url) AS total_pages + FROM + `httparchive.summary_pages.2022_06_01_*` + GROUP BY + client +), + +webpack AS ( + SELECT + _TABLE_SUFFIX AS client, + COUNT(DISTINCT url) AS webpack_pages + FROM + `httparchive.technologies.2022_06_01_*` + WHERE + app = 'webpack' + GROUP BY + client +) + +SELECT + client, + webpack_pages, + total_pages, + webpack_pages / total_pages AS pct_webpack_pages +FROM + totals +JOIN + webpack +USING (client) +ORDER BY + client diff --git a/sql/2024/javascript/usage_of_webpack_by_rank.sql b/sql/2024/javascript/usage_of_webpack_by_rank.sql new file mode 100644 index 00000000000..84e55bce15a --- /dev/null +++ b/sql/2024/javascript/usage_of_webpack_by_rank.sql @@ -0,0 +1,61 @@ +#standardSQL +# Percent of pages using webpack grouped by rank + +WITH webpack_pages AS ( + SELECT + _TABLE_SUFFIX AS client, + url AS page + FROM + `httparchive.technologies.2022_06_01_*` + WHERE + app = 'webpack' +), + +rank_totals AS ( + SELECT + _TABLE_SUFFIX AS client, + rank_grouping, + COUNT(0) AS total + FROM + `httparchive.summary_pages.2022_06_01_*`, + UNNEST([1000, 10000, 100000, 1000000, 10000000]) AS rank_grouping + WHERE + rank <= rank_grouping + GROUP BY + client, + rank_grouping +), + +pages AS ( + SELECT + _TABLE_SUFFIX AS client, + rank_grouping, + url AS page + FROM + `httparchive.summary_pages.2022_06_01_*`, + UNNEST([1000, 10000, 100000, 1000000, 10000000]) AS rank_grouping + WHERE + rank <= rank_grouping +) + +SELECT + client, + rank_grouping AS rank, + COUNT(DISTINCT webpack_pages.page) AS count_webpack_pages, + total, + COUNT(DISTINCT webpack_pages.page) / total AS pct_webpack_pages +FROM + webpack_pages +LEFT OUTER JOIN + pages +USING (client, page) +JOIN + rank_totals +USING (client, rank_grouping) +GROUP BY + client, + total, + rank_grouping +ORDER BY + client, + rank_grouping diff --git a/sql/2024/javascript/web_components_is_attribute.sql b/sql/2024/javascript/web_components_is_attribute.sql new file mode 100644 index 00000000000..878652c4186 --- /dev/null +++ b/sql/2024/javascript/web_components_is_attribute.sql @@ -0,0 +1,33 @@ +#standardSQL +# pages which use `is` attribute + +CREATE TEMPORARY FUNCTION get_almanac_attribute_info(almanac_string STRING) +RETURNS ARRAY> LANGUAGE js AS ''' +try { + var almanac = JSON.parse(almanac_string); + + if (Array.isArray(almanac) || typeof almanac != 'object') return []; + + if (almanac.attributes_used_on_elements) { + return Object.entries(almanac.attributes_used_on_elements).map(([name, freq]) => ({name, freq})); + } + +} catch (e) { + +} +return []; +'''; + +SELECT + _TABLE_SUFFIX AS client, + COUNTIF(almanac_attribute_info.name = 'is' AND almanac_attribute_info.freq > 0) AS freq, + SUM(COUNT(DISTINCT url)) OVER (PARTITION BY _TABLE_SUFFIX) AS total, + COUNTIF(almanac_attribute_info.name = 'is' AND almanac_attribute_info.freq > 0) / SUM(COUNT(DISTINCT url)) OVER (PARTITION BY _TABLE_SUFFIX) AS pct_pages +FROM + `httparchive.pages.2022_06_01_*`, + UNNEST(get_almanac_attribute_info(JSON_EXTRACT_SCALAR(payload, '$._almanac'))) AS almanac_attribute_info +GROUP BY + client +ORDER BY + client, + pct_pages DESC diff --git a/sql/2024/javascript/web_components_pct.sql b/sql/2024/javascript/web_components_pct.sql new file mode 100644 index 00000000000..3ef720a7889 --- /dev/null +++ b/sql/2024/javascript/web_components_pct.sql @@ -0,0 +1,24 @@ +SELECT + client, + custom_elements, + shadow_roots, + templates, + total, + custom_elements / total AS pct_custom_elements, + shadow_roots / total AS pct_shadow_roots, + templates / total AS pct_templates +FROM ( + SELECT + client, + COUNT(0) AS total, + COUNTIF(ARRAY_LENGTH(JSON_EXTRACT_ARRAY(js, '$.web_component_specs.custom_elements')) > 0) AS custom_elements, + COUNTIF(ARRAY_LENGTH(JSON_EXTRACT_ARRAY(js, '$.web_component_specs.shadow_roots')) > 0) AS shadow_roots, + COUNTIF(ARRAY_LENGTH(JSON_EXTRACT_ARRAY(js, '$.web_component_specs.template')) > 0) AS templates + FROM ( + SELECT + _TABLE_SUFFIX AS client, + JSON_EXTRACT_SCALAR(payload, '$._javascript') AS js + FROM + `httparchive.pages.2022_06_01_*`) + GROUP BY + client) diff --git a/sql/2024/javascript/web_components_specs.sql b/sql/2024/javascript/web_components_specs.sql new file mode 100644 index 00000000000..ac77b374165 --- /dev/null +++ b/sql/2024/javascript/web_components_specs.sql @@ -0,0 +1,23 @@ +SELECT + client, + percentile, + APPROX_QUANTILES(custom_elements, 1000)[OFFSET(percentile * 10)] AS custom_elements, + APPROX_QUANTILES(shadow_roots, 1000)[OFFSET(percentile * 10)] AS shadow_roots, + APPROX_QUANTILES(template, 1000)[OFFSET(percentile * 10)] AS template +FROM ( + SELECT + _TABLE_SUFFIX AS client, + ARRAY_LENGTH(JSON_EXTRACT_ARRAY(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.web_component_specs.custom_elements')) AS custom_elements, + ARRAY_LENGTH(JSON_EXTRACT_ARRAY(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.web_component_specs.shadow_roots')) AS shadow_roots, + ARRAY_LENGTH(JSON_EXTRACT_ARRAY(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.web_component_specs.template')) AS template + FROM + # Note: We're intentionally querying the September dataset here because of a bug in the custom metric. + # See https://github.com/HTTPArchive/legacy.httparchive.org/pull/231. + `httparchive.pages.2022_06_01_*`), + UNNEST([10, 25, 50, 75, 90, 100]) AS percentile +GROUP BY + percentile, + client +ORDER BY + client, + percentile diff --git a/sql/2024/javascript/web_workers.sql b/sql/2024/javascript/web_workers.sql new file mode 100644 index 00000000000..7b80d9927df --- /dev/null +++ b/sql/2024/javascript/web_workers.sql @@ -0,0 +1,17 @@ +# standardSQL +# usage of web worker using blink features from usage counters + +SELECT + client, + pct_urls +FROM + `httparchive.blink_features.usage` +WHERE + yyyymmdd = '20220601' AND + feature = 'WorkerStart' +GROUP BY + pct_urls, + client +ORDER BY + pct_urls, + client diff --git a/sql/2024/javascript/worklets.sql b/sql/2024/javascript/worklets.sql new file mode 100644 index 00000000000..4982c6935f4 --- /dev/null +++ b/sql/2024/javascript/worklets.sql @@ -0,0 +1,22 @@ +# standardSQL +# usage of AudioWorklet.addModule and PaintWorklet using blink features from usage counters +SELECT + feature, + client, + pct_urls +FROM + `httparchive.blink_features.usage` +WHERE + yyyymmdd = '20220601' AND + ( + feature = 'PaintWorklet' OR + feature = 'AudioWorkletAddModule' + ) +GROUP BY + feature, + client, + pct_urls +ORDER BY + feature, + client, + pct_urls From 6947541a3d941e702cf67f4cb21c43960ef02518 Mon Sep 17 00:00:00 2001 From: Nurullah Demir Date: Fri, 16 Aug 2024 16:03:58 +0200 Subject: [PATCH 02/53] JavaScript Queries for 2024 --- .../breakdown_of_pages_using_async_defer.sql | 5 ++- ...ipts_using_async_defer_module_nomodule.sql | 29 +++++++++----- sql/2024/javascript/bytes_2023.sql | 15 +++++++ sql/2024/javascript/bytes_2024.sql | 15 +++++++ sql/2024/javascript/bytes_by_3p.sql | 9 +++-- sql/2024/javascript/compression_method.sql | 6 +-- .../javascript/compression_method_by_3p.sql | 8 ++-- .../javascript/compression_none_by_bytes.sql | 8 ++-- .../distribution_of_injected_scripts.sql | 9 +++-- .../distribution_of_long_tasks_time.sql | 7 ++-- .../distribution_of_number_of_long_tasks.sql | 7 ++-- sql/2024/javascript/dynamic_import.sql | 2 +- sql/2024/javascript/fid.sql | 2 +- .../frameworks_bytes_by_framework.sql | 4 +- sql/2024/javascript/frameworks_libraries.sql | 4 +- .../frameworks_libraries_by_version.sql | 6 +-- .../frameworks_libraries_combos.sql | 4 +- sql/2024/javascript/injected_scripts.sql | 8 ++-- sql/2024/javascript/inp.sql | 2 +- .../javascript/lighthouse_unminified_js.sql | 2 +- .../lighthouse_unminified_js_by_3p.sql | 4 +- .../lighthouse_unminified_js_bytes.sql | 2 +- .../javascript/lighthouse_vulnerabilities.sql | 22 ----------- .../lighthouse_vulnerable_libraries.sql | 39 ------------------- sql/2024/javascript/module_and_nomodule.sql | 9 +++-- .../javascript/percent_of_dns_prefetch.sql | 11 +++--- .../percent_of_dns_prefetch_by_host.sql | 12 +++--- sql/2024/javascript/percent_of_preconnect.sql | 6 +-- .../percent_of_preconnect_by_host.sql | 11 +++--- sql/2024/javascript/posttask_scheduler.sql | 2 +- .../javascript/render_blocking_javascript.sql | 7 ++-- .../render_blocking_javascript_by_rank.sql | 10 ++--- sql/2024/javascript/requests_2023.sql | 15 +++++++ sql/2024/javascript/requests_2024.sql | 15 +++++++ sql/2024/javascript/requests_by_3p.sql | 6 +-- ...fetch-preload-modulepreload-percentage.sql | 10 +++-- ...ad-prefetch-modulepreload-distribution.sql | 8 ++-- sql/2024/javascript/resource_hints.sql | 12 +++--- .../javascript/resource_hints_per_page.sql | 9 +++-- sql/2024/javascript/sourcemap_header.sql | 6 +-- .../javascript/sourcemap_header_by_3p.sql | 8 ++-- sql/2024/javascript/sourcemaps.sql | 8 ++-- sql/2024/javascript/sync_requests.sql | 2 +- sql/2024/javascript/tbt.sql | 2 +- sql/2024/javascript/time_to_interactive.sql | 2 +- .../unused_js_bytes_distribution.sql | 2 +- .../javascript/usage_of_document_write.sql | 4 +- .../javascript/usage_of_legacy_javascript.sql | 4 +- sql/2024/javascript/usage_of_parcel.sql | 4 +- .../javascript/usage_of_parcel_by_rank.sql | 7 ++-- .../usage_of_typescript_and_babel.sql | 7 ++-- .../usage_of_typescript_and_babel_by_rank.sql | 4 +- sql/2024/javascript/usage_of_webpack.sql | 6 +-- .../javascript/usage_of_webpack_by_rank.sql | 8 ++-- .../web_components_is_attribute.sql | 10 ++--- sql/2024/javascript/web_components_pct.sql | 6 ++- sql/2024/javascript/web_components_specs.sql | 6 +-- sql/2024/javascript/web_workers.sql | 4 +- sql/2024/javascript/worklets.sql | 2 +- 59 files changed, 248 insertions(+), 216 deletions(-) create mode 100644 sql/2024/javascript/bytes_2023.sql create mode 100644 sql/2024/javascript/bytes_2024.sql delete mode 100644 sql/2024/javascript/lighthouse_vulnerabilities.sql delete mode 100644 sql/2024/javascript/lighthouse_vulnerable_libraries.sql create mode 100644 sql/2024/javascript/requests_2023.sql create mode 100644 sql/2024/javascript/requests_2024.sql diff --git a/sql/2024/javascript/breakdown_of_pages_using_async_defer.sql b/sql/2024/javascript/breakdown_of_pages_using_async_defer.sql index ae9176f3e33..502c7ff6595 100644 --- a/sql/2024/javascript/breakdown_of_pages_using_async_defer.sql +++ b/sql/2024/javascript/breakdown_of_pages_using_async_defer.sql @@ -15,6 +15,9 @@ SELECT COUNTIF(CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.script_tags.async') AS INT64) = 0 AND CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.script_tags.defer') AS INT64) = 0) AS neither, COUNTIF(CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.script_tags.async') AS INT64) = 0 AND CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.script_tags.defer') AS INT64) = 0) / COUNT(0) AS neither_pct FROM - `httparchive.pages.2022_06_01_*` + `httparchive.all.pages` +WHERE +WHERE date = '2024-06-01' + GROUP BY client diff --git a/sql/2024/javascript/breakdown_of_scripts_using_async_defer_module_nomodule.sql b/sql/2024/javascript/breakdown_of_scripts_using_async_defer_module_nomodule.sql index afa405f6311..be5a3b22edb 100644 --- a/sql/2024/javascript/breakdown_of_scripts_using_async_defer_module_nomodule.sql +++ b/sql/2024/javascript/breakdown_of_scripts_using_async_defer_module_nomodule.sql @@ -1,8 +1,13 @@ #standardSQL # Breakdown of scripts using Async, Defer, Module or NoModule attributes. Also breakdown of inline vs external scripts -CREATE TEMPORARY FUNCTION getScripts(payload STRING) -RETURNS STRUCT -LANGUAGE js AS ''' +CREATE TEMPORARY FUNCTION getScripts(payload STRING) RETURNS STRUCT < total INT64, +inline INT64, +src INT64, +async INT64, +defer INT64, +async_and_defer INT64, +type_module INT64, +nomodule INT64 > LANGUAGE js AS ''' try { var $ = JSON.parse(payload); var javascript = JSON.parse($._javascript); @@ -29,11 +34,15 @@ SELECT SUM(script.async_and_defer) / SUM(script.src) AS pct_external_async_defer, SUM(script.type_module) / SUM(script.src) AS pct_external_module, SUM(script.nomodule) / SUM(script.src) AS pct_external_nomodule -FROM ( - SELECT - _TABLE_SUFFIX AS client, - getScripts(payload) AS script - FROM - `httparchive.pages.2022_06_01_*`) +FROM + ( + SELECT + client, + getScripts(payload) AS script + FROM + `httparchive.all.pages` + WHERE + date = '2024-06-01' + ) GROUP BY - client + client \ No newline at end of file diff --git a/sql/2024/javascript/bytes_2023.sql b/sql/2024/javascript/bytes_2023.sql new file mode 100644 index 00000000000..5d8e6a57e28 --- /dev/null +++ b/sql/2024/javascript/bytes_2023.sql @@ -0,0 +1,15 @@ +#standardSQL +# Sum of JS request bytes per page (2023) +SELECT + percentile, + _TABLE_SUFFIX AS client, + APPROX_QUANTILES(bytesJs / 1024, 1000)[OFFSET(percentile * 10)] AS js_kilobytes +FROM + `httparchive.summary_pages.2023_06_01_*`, + UNNEST([10, 25, 50, 75, 90, 100]) AS percentile +GROUP BY + percentile, + client +ORDER BY + client, + percentile diff --git a/sql/2024/javascript/bytes_2024.sql b/sql/2024/javascript/bytes_2024.sql new file mode 100644 index 00000000000..7f028be4138 --- /dev/null +++ b/sql/2024/javascript/bytes_2024.sql @@ -0,0 +1,15 @@ +#standardSQL +# Sum of JS request bytes per page (2024) +SELECT + percentile, + _TABLE_SUFFIX AS client, + APPROX_QUANTILES(bytesJs / 1024, 1000)[OFFSET(percentile * 10)] AS js_kilobytes +FROM + `httparchive.summary_pages.2024_06_01_*`, + UNNEST([10, 25, 50, 75, 90, 100]) AS percentile +GROUP BY + percentile, + client +ORDER BY + client, + percentile diff --git a/sql/2024/javascript/bytes_by_3p.sql b/sql/2024/javascript/bytes_by_3p.sql index c2eb6329230..dadb14b177d 100644 --- a/sql/2024/javascript/bytes_by_3p.sql +++ b/sql/2024/javascript/bytes_by_3p.sql @@ -10,13 +10,14 @@ FROM ( client, page, IF(NET.HOST(url) IN ( - SELECT domain FROM `httparchive.almanac.third_parties` WHERE date = '2022-06-01' AND category != 'hosting' + SELECT domain FROM `httparchive.almanac.third_parties` WHERE date = '2024-06-01' AND category != 'hosting' ), 'third party', 'first party') AS host, - SUM(respSize) / 1024 AS kbytes + SUM(cast(json_value(payload,"$.response.bodySize") as int64)) / 1024 AS kbytes FROM - `httparchive.almanac.requests` + `httparchive.all.requests` TABLESAMPLE SYSTEM (0.01 PERCENT) + WHERE - date = '2022-06-01' AND + date = '2024-06-01' AND type = 'script' GROUP BY client, diff --git a/sql/2024/javascript/compression_method.sql b/sql/2024/javascript/compression_method.sql index 09da63c4e53..ec9adfe9d95 100644 --- a/sql/2024/javascript/compression_method.sql +++ b/sql/2024/javascript/compression_method.sql @@ -26,16 +26,16 @@ FROM ( page, getHeader(JSON_EXTRACT(payload, '$.response.headers'), 'Content-Encoding') AS compression FROM - `httparchive.almanac.requests` + `httparchive.all.requests` WHERE - date = '2022-06-01' AND + date = '2024-06-01' AND type = 'script') JOIN ( SELECT _TABLE_SUFFIX AS client, COUNT(0) AS total_pages FROM - `httparchive.summary_pages.2022_06_01_*` + `httparchive.summary_pages.2024_06_01_*` GROUP BY client) USING diff --git a/sql/2024/javascript/compression_method_by_3p.sql b/sql/2024/javascript/compression_method_by_3p.sql index 2ad5fe79c61..a0c4ac78187 100644 --- a/sql/2024/javascript/compression_method_by_3p.sql +++ b/sql/2024/javascript/compression_method_by_3p.sql @@ -26,20 +26,20 @@ FROM ( client, page, IF(NET.HOST(url) IN ( - SELECT domain FROM `httparchive.almanac.third_parties` WHERE date = '2022-06-01' AND category != 'hosting' + SELECT domain FROM `httparchive.almanac.third_parties` WHERE date = '2024-06-01' AND category != 'hosting' ), 'third party', 'first party') AS host, getHeader(JSON_EXTRACT(payload, '$.response.headers'), 'Content-Encoding') AS compression FROM - `httparchive.almanac.requests` + `httparchive.all.requests` WHERE - date = '2022-06-01' AND + date = '2024-06-01' AND type = 'script') JOIN ( SELECT _TABLE_SUFFIX AS client, COUNT(0) AS total_pages FROM - `httparchive.summary_pages.2022_06_01_*` + `httparchive.summary_pages.2024_06_01_*` GROUP BY client) USING diff --git a/sql/2024/javascript/compression_none_by_bytes.sql b/sql/2024/javascript/compression_none_by_bytes.sql index 1a3081cd2ff..1adb5805544 100644 --- a/sql/2024/javascript/compression_none_by_bytes.sql +++ b/sql/2024/javascript/compression_none_by_bytes.sql @@ -26,13 +26,13 @@ FROM ( client, page, IF(NET.HOST(url) IN ( - SELECT domain FROM `httparchive.almanac.third_parties` WHERE date = '2022-06-01' AND category != 'hosting' + SELECT domain FROM `httparchive.almanac.third_parties` WHERE date = '2024-06-01' AND category != 'hosting' ), 'third party', 'first party') AS host, respSize / 1024 AS kbytes FROM - `httparchive.almanac.requests` + `httparchive.all.requests` WHERE - date = '2022-06-01' AND + date = '2024-06-01' AND type = 'script' AND getHeader(JSON_EXTRACT(payload, '$.response.headers'), 'Content-Encoding') IS NULL) JOIN ( @@ -40,7 +40,7 @@ JOIN ( _TABLE_SUFFIX AS client, COUNT(0) AS total_pages FROM - `httparchive.summary_pages.2022_06_01_*` + `httparchive.summary_pages.2024_06_01_*` GROUP BY client) USING diff --git a/sql/2024/javascript/distribution_of_injected_scripts.sql b/sql/2024/javascript/distribution_of_injected_scripts.sql index ac938f6361a..7e8b91eb9ac 100644 --- a/sql/2024/javascript/distribution_of_injected_scripts.sql +++ b/sql/2024/javascript/distribution_of_injected_scripts.sql @@ -1,7 +1,6 @@ #standardSQL # Number of injected scripts per page. # Pages without any scripts are excluded - # returns the number of injected scripts and inline CREATE TEMPORARY FUNCTION getScripts(payload STRING) RETURNS STRUCT @@ -39,11 +38,13 @@ SELECT APPROX_QUANTILES(IF(scripts.scripts > 0, scripts.injectedScripts / scripts.scripts, 0), 1000)[OFFSET(percentile * 10)] AS pct_injected_scripts_per_page FROM ( SELECT - _TABLE_SUFFIX AS client, - url AS page, + client, + page, getScripts(payload) AS scripts FROM - `httparchive.pages.2022_06_01_*` + `httparchive.all.pages` + where + date = "2024-06-01" ), UNNEST([10, 25, 50, 75, 90, 100]) AS percentile WHERE diff --git a/sql/2024/javascript/distribution_of_long_tasks_time.sql b/sql/2024/javascript/distribution_of_long_tasks_time.sql index 9234775164c..9507c8e55ce 100644 --- a/sql/2024/javascript/distribution_of_long_tasks_time.sql +++ b/sql/2024/javascript/distribution_of_long_tasks_time.sql @@ -21,12 +21,13 @@ try { WITH long_tasks_pages AS ( SELECT - _TABLE_SUFFIX AS client, - url, + client, + page as url, _longTasks AS long_tasks FROM - `httparchive.pages.2022_06_01_*`, + `httparchive.all.pages`, UNNEST(getLongTasks(payload)) AS _longTasks + where date = "2024-06-01" ), long_tasks_by_page AS ( diff --git a/sql/2024/javascript/distribution_of_number_of_long_tasks.sql b/sql/2024/javascript/distribution_of_number_of_long_tasks.sql index 16bb14763ce..c4e9e32ccef 100644 --- a/sql/2024/javascript/distribution_of_number_of_long_tasks.sql +++ b/sql/2024/javascript/distribution_of_number_of_long_tasks.sql @@ -21,12 +21,13 @@ try { WITH long_tasks_pages AS ( SELECT - _TABLE_SUFFIX AS client, - url, + client, + page as url, _longTasks AS long_tasks FROM - `httparchive.pages.2022_06_01_*`, + `httparchive.all.pages`, UNNEST(getLongTasks(payload)) AS _longTasks + where date = "2024-06-01" ), long_tasks_by_page AS ( diff --git a/sql/2024/javascript/dynamic_import.sql b/sql/2024/javascript/dynamic_import.sql index 7a865a0aa9a..df7e1d434fb 100644 --- a/sql/2024/javascript/dynamic_import.sql +++ b/sql/2024/javascript/dynamic_import.sql @@ -6,7 +6,7 @@ SELECT FROM `httparchive.blink_features.usage` WHERE - yyyymmdd = '20220601' AND + yyyymmdd = '20240601' AND feature = 'DynamicImportModuleScript' GROUP BY pct_urls, diff --git a/sql/2024/javascript/fid.sql b/sql/2024/javascript/fid.sql index 92f2aae3dd0..77fef4c5369 100644 --- a/sql/2024/javascript/fid.sql +++ b/sql/2024/javascript/fid.sql @@ -11,7 +11,7 @@ base AS ( `chrome-ux-report.materialized.device_summary` WHERE device IN ('desktop', 'phone') AND - date IN ('2022-06-01') + date IN ('2024-06-01') ) SELECT diff --git a/sql/2024/javascript/frameworks_bytes_by_framework.sql b/sql/2024/javascript/frameworks_bytes_by_framework.sql index a15cd9398da..7ee9c414fd6 100644 --- a/sql/2024/javascript/frameworks_bytes_by_framework.sql +++ b/sql/2024/javascript/frameworks_bytes_by_framework.sql @@ -12,14 +12,14 @@ FROM ( url AS page, bytesJs FROM - `httparchive.summary_pages.2022_06_01_*`) + `httparchive.summary_pages.2024_06_01_*`) JOIN ( SELECT DISTINCT _TABLE_SUFFIX AS client, url AS page, app FROM - `httparchive.technologies.2022_06_01_*` + `httparchive.technologies.2024_06_01_*` WHERE category = 'JavaScript frameworks') USING diff --git a/sql/2024/javascript/frameworks_libraries.sql b/sql/2024/javascript/frameworks_libraries.sql index 288d9972772..9db34f4abe9 100644 --- a/sql/2024/javascript/frameworks_libraries.sql +++ b/sql/2024/javascript/frameworks_libraries.sql @@ -8,13 +8,13 @@ SELECT total, COUNT(DISTINCT url) / total AS pct FROM - `httparchive.technologies.2022_06_01_*` + `httparchive.technologies.2024_06_01_*` JOIN ( SELECT _TABLE_SUFFIX, COUNT(0) AS total FROM - `httparchive.summary_pages.2022_06_01_*` + `httparchive.summary_pages.2024_06_01_*` GROUP BY _TABLE_SUFFIX) USING diff --git a/sql/2024/javascript/frameworks_libraries_by_version.sql b/sql/2024/javascript/frameworks_libraries_by_version.sql index 8735ddce2d7..b5360d4382b 100644 --- a/sql/2024/javascript/frameworks_libraries_by_version.sql +++ b/sql/2024/javascript/frameworks_libraries_by_version.sql @@ -9,19 +9,19 @@ SELECT total, COUNT(DISTINCT url) / total AS pct FROM - `httparchive.technologies.2022_06_01_*` + `httparchive.technologies.2024_06_01_*` JOIN ( SELECT _TABLE_SUFFIX, COUNT(0) AS total FROM - `httparchive.summary_pages.2022_06_01_*` + `httparchive.summary_pages.2024_06_01_*` GROUP BY _TABLE_SUFFIX) USING (_TABLE_SUFFIX) WHERE - app IN ('jQuery', 'jQuery Migrate', 'jQuery UI', 'Modernizr', 'FancyBox', 'Slick', 'Lightbox', 'Moment.js', 'Underscore.js', 'Lodash', 'React') +app IN ('jQuery', 'jQuery Migrate', 'jQuery UI', 'Modernizr', 'FancyBox', 'Slick', 'Lightbox', 'Moment.js', 'Underscore.js', 'Lodash', 'React', 'GSAP', 'Vue.js', 'styled-components', 'Emotion', 'Backbone.js', 'RequireJS', 'AngularJS', 'AMP', 'Redux', 'Next.js', 'Stimulus', 'Angular', 'Handlebars', 'Zone.js', 'Marionette.js', 'Mustache', 'Prototype', 'toastr', 'JSS', 'React Router', 'Nuxt.js', 'Alpine.js', 'Svelte', 'MooTools', 'Knockout.js', 'Socket.io', 'Gatsby', 'Adobe Client Data Layer') GROUP BY client, category, diff --git a/sql/2024/javascript/frameworks_libraries_combos.sql b/sql/2024/javascript/frameworks_libraries_combos.sql index 7ce666da60f..9885628519f 100644 --- a/sql/2024/javascript/frameworks_libraries_combos.sql +++ b/sql/2024/javascript/frameworks_libraries_combos.sql @@ -16,13 +16,13 @@ FROM ( total, ARRAY_TO_STRING(ARRAY_AGG(app ORDER BY app), ', ') AS apps FROM - `httparchive.technologies.2022_06_01_*` + `httparchive.technologies.2024_06_01_*` JOIN ( SELECT _TABLE_SUFFIX, COUNT(0) AS total FROM - `httparchive.summary_pages.2022_06_01_*` + `httparchive.summary_pages.2024_06_01_*` GROUP BY _TABLE_SUFFIX) USING diff --git a/sql/2024/javascript/injected_scripts.sql b/sql/2024/javascript/injected_scripts.sql index 8696e065458..ab86eca902d 100644 --- a/sql/2024/javascript/injected_scripts.sql +++ b/sql/2024/javascript/injected_scripts.sql @@ -39,11 +39,13 @@ SELECT COUNTIF(scripts.injectedScripts > 0) / COUNT(0) AS pct_injected_scripts FROM ( SELECT - _TABLE_SUFFIX AS client, - url AS page, + client, + page, getScripts(payload) AS scripts FROM - `httparchive.pages.2022_06_01_*` + `httparchive.all.pages` + WHERE + date = '2024-06-01' ) GROUP BY client diff --git a/sql/2024/javascript/inp.sql b/sql/2024/javascript/inp.sql index 61f8be138d3..e32b4eb50d1 100644 --- a/sql/2024/javascript/inp.sql +++ b/sql/2024/javascript/inp.sql @@ -11,7 +11,7 @@ base AS ( `chrome-ux-report.materialized.device_summary` WHERE device IN ('desktop', 'phone') AND - date IN ('2022-06-01') + date IN ('2024-06-01') ) SELECT diff --git a/sql/2024/javascript/lighthouse_unminified_js.sql b/sql/2024/javascript/lighthouse_unminified_js.sql index 3af42232b97..7af0f0b3031 100644 --- a/sql/2024/javascript/lighthouse_unminified_js.sql +++ b/sql/2024/javascript/lighthouse_unminified_js.sql @@ -11,7 +11,7 @@ FROM ( _TABLE_SUFFIX AS client, JSON_EXTRACT_SCALAR(report, "$.audits['unminified-javascript'].score") AS score FROM - `httparchive.lighthouse.2022_06_01_*` + `httparchive.lighthouse.2024_06_01_*` ) WHERE score IS NOT NULL diff --git a/sql/2024/javascript/lighthouse_unminified_js_by_3p.sql b/sql/2024/javascript/lighthouse_unminified_js_by_3p.sql index a9094a92ffd..0ec88a14d39 100644 --- a/sql/2024/javascript/lighthouse_unminified_js_by_3p.sql +++ b/sql/2024/javascript/lighthouse_unminified_js_by_3p.sql @@ -27,11 +27,11 @@ FROM ( _TABLE_SUFFIX AS client, lighthouse.url AS page, NET.HOST(unminified.url) IS NOT NULL AND NET.HOST(unminified.url) IN ( - SELECT domain FROM `httparchive.almanac.third_parties` WHERE date = '2022-06-01' AND category != 'hosting' + SELECT domain FROM `httparchive.almanac.third_parties` WHERE date = '2024-06-01' AND category != 'hosting' ) AS is_3p, unminified.wastedBytes AS wasted_bytes FROM - `httparchive.lighthouse.2022_06_01_*` AS lighthouse, + `httparchive.lighthouse.2024_06_01_*` AS lighthouse, UNNEST(getUnminifiedJsUrls(JSON_EXTRACT(report, "$.audits['unminified-javascript']"))) AS unminified ) GROUP BY diff --git a/sql/2024/javascript/lighthouse_unminified_js_bytes.sql b/sql/2024/javascript/lighthouse_unminified_js_bytes.sql index 1555efc80e2..40f3c8e5c0f 100644 --- a/sql/2024/javascript/lighthouse_unminified_js_bytes.sql +++ b/sql/2024/javascript/lighthouse_unminified_js_bytes.sql @@ -22,7 +22,7 @@ FROM ( lighthouse.url AS page, SUM(IFNULL(unminified_js_bytes, 0)) / 1024 AS unminified_js_kbytes FROM - `httparchive.lighthouse.2022_06_01_*` AS lighthouse + `httparchive.lighthouse.2024_06_01_*` AS lighthouse LEFT JOIN UNNEST(getUnminifiedJsBytes(JSON_EXTRACT(report, "$.audits['unminified-javascript']"))) AS unminified_js_bytes GROUP BY diff --git a/sql/2024/javascript/lighthouse_vulnerabilities.sql b/sql/2024/javascript/lighthouse_vulnerabilities.sql deleted file mode 100644 index 37992a5ae3b..00000000000 --- a/sql/2024/javascript/lighthouse_vulnerabilities.sql +++ /dev/null @@ -1,22 +0,0 @@ -#standardSQL -# Pages with vulnerable libraries -SELECT - client, - score, - COUNT(0) AS pages, - SUM(COUNT(0)) OVER (PARTITION BY client) AS total, - COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY client) AS pct -FROM ( - SELECT - _TABLE_SUFFIX AS client, - JSON_EXTRACT_SCALAR(report, "$.audits['no-vulnerable-libraries'].score") AS score - FROM - `httparchive.lighthouse.2022_06_01_*`) -WHERE - score IS NOT NULL -GROUP BY - client, - score -ORDER BY - client, - score diff --git a/sql/2024/javascript/lighthouse_vulnerable_libraries.sql b/sql/2024/javascript/lighthouse_vulnerable_libraries.sql deleted file mode 100644 index 28f6f14d403..00000000000 --- a/sql/2024/javascript/lighthouse_vulnerable_libraries.sql +++ /dev/null @@ -1,39 +0,0 @@ -#standardSQL -# Most frequent vulnerable libraries -CREATE TEMPORARY FUNCTION getVulnerabilities(audit STRING) -RETURNS ARRAY LANGUAGE js AS ''' -try { - var $ = JSON.parse(audit); - return $.details.items.map(i => i.detectedLib.text.split('@')[0]); -} catch(e) { - return []; -} -'''; - -SELECT - _TABLE_SUFFIX AS client, - lib, - COUNT(0) AS freq, - total, - COUNT(0) / total AS pct -FROM - `httparchive.lighthouse.2022_06_01_*` AS lighthouse, - UNNEST(getVulnerabilities(JSON_EXTRACT(report, "$.audits['no-vulnerable-libraries']"))) AS lib -JOIN ( - SELECT - _TABLE_SUFFIX AS client, - COUNT(DISTINCT url) AS total - FROM - `httparchive.lighthouse.2022_06_01_*` - GROUP BY - client -) -ON - lighthouse._TABLE_SUFFIX = client -GROUP BY - client, - lib, - total -ORDER BY - client, - freq DESC diff --git a/sql/2024/javascript/module_and_nomodule.sql b/sql/2024/javascript/module_and_nomodule.sql index 76a5df03a1b..b580cd4e6b5 100644 --- a/sql/2024/javascript/module_and_nomodule.sql +++ b/sql/2024/javascript/module_and_nomodule.sql @@ -1,3 +1,4 @@ +# module_and_nomodule.sql SELECT client, COUNT(DISTINCT IF(module, page, NULL)) AS module, @@ -15,12 +16,12 @@ FROM ( REGEXP_CONTAINS(script, r'(?i)\bmodule\b') AS module, REGEXP_CONTAINS(script, r'(?i)\bnomodule\b') AS nomodule FROM - `httparchive.almanac.summary_response_bodies` + `httparchive.all.requests` LEFT JOIN - UNNEST(REGEXP_EXTRACT_ALL(body, r'(?i)(]*>)')) AS script + UNNEST(REGEXP_EXTRACT_ALL(response_body, r'(?i)(]*>)')) AS script WHERE - date = '2022-06-01' AND - firstHtml + date = '2024-06-01' AND + cast(json_value(summary,"$.firstHtml") as bool) ) GROUP BY client diff --git a/sql/2024/javascript/percent_of_dns_prefetch.sql b/sql/2024/javascript/percent_of_dns_prefetch.sql index 7e5b22829e2..e181255e22e 100644 --- a/sql/2024/javascript/percent_of_dns_prefetch.sql +++ b/sql/2024/javascript/percent_of_dns_prefetch.sql @@ -20,12 +20,13 @@ WITH resource_hints AS ( host FROM ( SELECT - _TABLE_SUFFIX AS client, - url AS page, + client, + page, NET.HOST(href) AS host FROM - `httparchive.pages.2022_06_01_*`, - UNNEST(getResourceHintsHrefs(payload, 'dns-prefetch')) AS href + `httparchive.all.pages`, + UNNEST(getResourceHintsHrefs(payload, 'dns-prefetch')) AS href + where date = "2024-06-01" ) GROUP BY client, @@ -52,7 +53,7 @@ requests AS ( ON (third_party_domains.host = NET.HOST(url)) WHERE - date = '2022-06-01' AND + date = '2024-06-01' AND type = 'script' GROUP BY client, diff --git a/sql/2024/javascript/percent_of_dns_prefetch_by_host.sql b/sql/2024/javascript/percent_of_dns_prefetch_by_host.sql index fecc2ee4052..7ab1e176366 100644 --- a/sql/2024/javascript/percent_of_dns_prefetch_by_host.sql +++ b/sql/2024/javascript/percent_of_dns_prefetch_by_host.sql @@ -1,6 +1,5 @@ #standardSQL # Percentage of requests for script files which are preceded by a dns-prefetch - CREATE TEMPORARY FUNCTION getResourceHintsHrefs(payload STRING, hint STRING) RETURNS ARRAY LANGUAGE js AS ''' @@ -20,12 +19,13 @@ WITH resource_hints AS ( host FROM ( SELECT - _TABLE_SUFFIX AS client, - url AS page, + client, + page, NET.HOST(href) AS host FROM - `httparchive.pages.2022_06_01_*`, + `httparchive.all.pages`, UNNEST(getResourceHintsHrefs(payload, 'dns-prefetch')) AS href + where date='2024-06-01' ) GROUP BY client, @@ -46,13 +46,13 @@ requests AS ( page, NET.HOST(url) AS host FROM - `httparchive.almanac.requests` + `httparchive.all.requests` INNER JOIN third_party_domains ON (third_party_domains.host = NET.HOST(url)) WHERE - date = '2022-06-01' AND + date = '2024-06-01' AND type = 'script' GROUP BY client, diff --git a/sql/2024/javascript/percent_of_preconnect.sql b/sql/2024/javascript/percent_of_preconnect.sql index 74686f1778e..9b0f248988d 100644 --- a/sql/2024/javascript/percent_of_preconnect.sql +++ b/sql/2024/javascript/percent_of_preconnect.sql @@ -24,7 +24,7 @@ WITH resource_hints AS ( url AS page, NET.HOST(href) AS host FROM - `httparchive.pages.2022_06_01_*`, + `httparchive.pages.2024_06_01_*`, UNNEST(getResourceHintsHrefs(payload, 'preconnect')) AS href ) GROUP BY @@ -46,13 +46,13 @@ requests AS ( page, NET.HOST(url) AS host FROM - `httparchive.almanac.requests` + `httparchive.all.requests` INNER JOIN third_party_domains ON (third_party_domains.host = NET.HOST(url)) WHERE - date = '2022-06-01' AND + date = '2024-06-01' AND type = 'script' GROUP BY client, diff --git a/sql/2024/javascript/percent_of_preconnect_by_host.sql b/sql/2024/javascript/percent_of_preconnect_by_host.sql index 5a85ec875b2..8d2c9920912 100644 --- a/sql/2024/javascript/percent_of_preconnect_by_host.sql +++ b/sql/2024/javascript/percent_of_preconnect_by_host.sql @@ -20,12 +20,13 @@ WITH resource_hints AS ( host FROM ( SELECT - _TABLE_SUFFIX AS client, - url AS page, + client, + page, NET.HOST(href) AS host FROM - `httparchive.pages.2022_06_01_*`, + `httparchive.all.pages`, UNNEST(getResourceHintsHrefs(payload, 'preconnect')) AS href + where date = "2024-06-01" ) GROUP BY client, @@ -46,13 +47,13 @@ requests AS ( page, NET.HOST(url) AS host FROM - `httparchive.almanac.requests` + `httparchive.all.requests` INNER JOIN third_party_domains ON (third_party_domains.host = NET.HOST(url)) WHERE - date = '2022-06-01' AND + date = '2024-06-01' AND type = 'script' GROUP BY client, diff --git a/sql/2024/javascript/posttask_scheduler.sql b/sql/2024/javascript/posttask_scheduler.sql index 42ef918c323..d524e52a883 100644 --- a/sql/2024/javascript/posttask_scheduler.sql +++ b/sql/2024/javascript/posttask_scheduler.sql @@ -6,7 +6,7 @@ SELECT FROM `httparchive.blink_features.usage` WHERE - yyyymmdd = '20220601' AND + yyyymmdd = '20240601' AND feature = 'SchedulerPostTask' GROUP BY pct_urls, diff --git a/sql/2024/javascript/render_blocking_javascript.sql b/sql/2024/javascript/render_blocking_javascript.sql index 8b469b62d3d..0bd73cfeb5b 100644 --- a/sql/2024/javascript/render_blocking_javascript.sql +++ b/sql/2024/javascript/render_blocking_javascript.sql @@ -15,11 +15,12 @@ try { WITH render_blocking_scripts AS ( SELECT - _TABLE_SUFFIX AS client, - url, + client, + page as url, getRenderBlockingScripts(payload) AS number_of_render_blocking_scripts FROM - `httparchive.pages.2022_06_01_*` + `httparchive.all.pages` + where date="2024-06-01" ) SELECT diff --git a/sql/2024/javascript/render_blocking_javascript_by_rank.sql b/sql/2024/javascript/render_blocking_javascript_by_rank.sql index 312fd09bb6c..483d3c4794a 100644 --- a/sql/2024/javascript/render_blocking_javascript_by_rank.sql +++ b/sql/2024/javascript/render_blocking_javascript_by_rank.sql @@ -1,6 +1,5 @@ #standardSQL # Percent of pages using render-blocking JavaScript. - CREATE TEMPORARY FUNCTION getRenderBlockingScripts(payload STRING) RETURNS INT64 LANGUAGE js AS ''' @@ -21,11 +20,12 @@ WITH render_blocking_scripts AS ( number_of_render_blocking_scripts FROM ( SELECT - _TABLE_SUFFIX AS client, - url, + client, + page as url, getRenderBlockingScripts(payload) AS number_of_render_blocking_scripts FROM - `httparchive.pages.2022_06_01_*` + `httparchive.all.pages` + where date="2024-06-01" ) JOIN ( SELECT @@ -33,7 +33,7 @@ WITH render_blocking_scripts AS ( url, rank FROM - `httparchive.summary_pages.2022_06_01_*` + `httparchive.summary_pages.2024_06_01_*` ) USING (client, url) diff --git a/sql/2024/javascript/requests_2023.sql b/sql/2024/javascript/requests_2023.sql new file mode 100644 index 00000000000..8a5048fc8fc --- /dev/null +++ b/sql/2024/javascript/requests_2023.sql @@ -0,0 +1,15 @@ +#standardSQL +# Sum of JS requests per page (2022) +SELECT + percentile, + _TABLE_SUFFIX AS client, + APPROX_QUANTILES(reqJS, 1000)[OFFSET(percentile * 10)] AS js_requests +FROM + `httparchive.summary_pages.2023_06_01_*`, + UNNEST([10, 25, 50, 75, 90, 100]) AS percentile +GROUP BY + percentile, + client +ORDER BY + client, + percentile diff --git a/sql/2024/javascript/requests_2024.sql b/sql/2024/javascript/requests_2024.sql new file mode 100644 index 00000000000..9d1117959dc --- /dev/null +++ b/sql/2024/javascript/requests_2024.sql @@ -0,0 +1,15 @@ +#standardSQL +# Sum of JS requests per page (2022) +SELECT + percentile, + _TABLE_SUFFIX AS client, + APPROX_QUANTILES(reqJS, 1000)[OFFSET(percentile * 10)] AS js_requests +FROM + `httparchive.summary_pages.2024_06_01_*`, + UNNEST([10, 25, 50, 75, 90, 100]) AS percentile +GROUP BY + percentile, + client +ORDER BY + client, + percentile diff --git a/sql/2024/javascript/requests_by_3p.sql b/sql/2024/javascript/requests_by_3p.sql index b4f04717aa1..c9ea6125012 100644 --- a/sql/2024/javascript/requests_by_3p.sql +++ b/sql/2024/javascript/requests_by_3p.sql @@ -10,13 +10,13 @@ FROM ( client, page, IF(NET.HOST(url) IN ( - SELECT domain FROM `httparchive.almanac.third_parties` WHERE date = '2022-06-01' AND category != 'hosting' + SELECT domain FROM `httparchive.almanac.third_parties` WHERE date = '2024-06-01' AND category != 'hosting' ), 'third party', 'first party') AS host, COUNT(0) AS requests FROM - `httparchive.almanac.requests` + `httparchive.all.requests` WHERE - date = '2022-06-01' AND + date = '2024-06-01' AND type = 'script' GROUP BY client, diff --git a/sql/2024/javascript/resource-hints-prefetch-preload-modulepreload-percentage.sql b/sql/2024/javascript/resource-hints-prefetch-preload-modulepreload-percentage.sql index 6ed7cb9765e..bf98f2f395c 100644 --- a/sql/2024/javascript/resource-hints-prefetch-preload-modulepreload-percentage.sql +++ b/sql/2024/javascript/resource-hints-prefetch-preload-modulepreload-percentage.sql @@ -39,14 +39,16 @@ SELECT COUNT(DISTINCT IF(modulepreload_hint, page, NULL)) / COUNT(DISTINCT page) AS modulepreload_pct FROM ( SELECT - _TABLE_SUFFIX AS client, - url AS page, + client, + page, hint.name = 'prefetch' AND hint.value = 'script' AS prefetch_hint, hint.name = 'preload' AND hint.value = 'script' AS preload_hint, hint.name = 'modulepreload' AND hint.value = 'script' AS modulepreload_hint FROM - `httparchive.pages.2022_06_01_*` + `httparchive.all.pages` LEFT JOIN - UNNEST(getResourceHintAttrs(payload)) AS hint) + UNNEST(getResourceHintAttrs(payload)) AS hint + where date ="2024-06-01" + ) GROUP BY client diff --git a/sql/2024/javascript/resource-hints-preload-prefetch-modulepreload-distribution.sql b/sql/2024/javascript/resource-hints-preload-prefetch-modulepreload-distribution.sql index e806dd0e191..86698ff9932 100644 --- a/sql/2024/javascript/resource-hints-preload-prefetch-modulepreload-distribution.sql +++ b/sql/2024/javascript/resource-hints-preload-prefetch-modulepreload-distribution.sql @@ -39,15 +39,17 @@ SELECT APPROX_QUANTILES(IF(modulepreload_hint = 0, NULL, modulepreload_hint), 1000 IGNORE NULLS)[OFFSET(percentile * 10)] AS modulepreload_hints_per_page_with_hints FROM ( SELECT - _TABLE_SUFFIX AS client, - url AS page, + client, + page, COUNTIF(hint.name = 'prefetch' AND hint.value = 'script') AS prefetch_hint, COUNTIF(hint.name = 'preload' AND hint.value = 'script') AS preload_hint, COUNTIF(hint.name = 'modulepreload' AND hint.value = 'script') AS modulepreload_hint FROM - `httparchive.pages.2022_06_01_*` + `httparchive.all.pages` LEFT JOIN UNNEST(getResourceHintAttrs(payload)) AS hint + where date ="2024-06-01" + GROUP BY client, page), diff --git a/sql/2024/javascript/resource_hints.sql b/sql/2024/javascript/resource_hints.sql index 73b236c045c..125066f9abb 100644 --- a/sql/2024/javascript/resource_hints.sql +++ b/sql/2024/javascript/resource_hints.sql @@ -1,6 +1,6 @@ #standardSQL # Returns the number of pages which have preload, prefetch or modulepreload for scripts - +# resource_hints.sql CREATE TEMPORARY FUNCTION getResourceHintAttrs(payload STRING) RETURNS ARRAY> LANGUAGE js AS ''' @@ -37,12 +37,14 @@ SELECT COUNT(DISTINCT IF(script_hint, page, NULL)) / COUNT(DISTINCT page) AS pct FROM ( SELECT - _TABLE_SUFFIX AS client, - url AS page, + client, + page, hint.name IN ('prefetch', 'preload', 'modulepreload') AND hint.value = 'script' AS script_hint FROM - `httparchive.pages.2022_06_01_*` + `httparchive.all.pages` LEFT JOIN - UNNEST(getResourceHintAttrs(payload)) AS hint) + UNNEST(getResourceHintAttrs(payload)) AS hint + where date = "2024-06-01" + ) GROUP BY client diff --git a/sql/2024/javascript/resource_hints_per_page.sql b/sql/2024/javascript/resource_hints_per_page.sql index 14686eac410..25facd1dba7 100644 --- a/sql/2024/javascript/resource_hints_per_page.sql +++ b/sql/2024/javascript/resource_hints_per_page.sql @@ -1,6 +1,5 @@ #standardSQL # Returns the number of resource hints per page which are preload, prefetch or modulepreload - CREATE TEMPORARY FUNCTION getResourceHintAttrs(payload STRING) RETURNS ARRAY> LANGUAGE js AS ''' @@ -37,13 +36,15 @@ SELECT APPROX_QUANTILES(IF(script_hint = 0, NULL, script_hint), 1000 IGNORE NULLS)[OFFSET(percentile * 10)] AS hints_per_page_with_hints FROM ( SELECT - _TABLE_SUFFIX AS client, - url AS page, + client, + page, COUNTIF(hint.name IN ('prefetch', 'preload', 'modulepreload') AND hint.value = 'script') AS script_hint FROM - `httparchive.pages.2022_06_01_*` + `httparchive.all.pages` LEFT JOIN UNNEST(getResourceHintAttrs(payload)) AS hint + + where date="2024-06-01" GROUP BY client, page), diff --git a/sql/2024/javascript/sourcemap_header.sql b/sql/2024/javascript/sourcemap_header.sql index 93b59b5b4f6..99f73e3851a 100644 --- a/sql/2024/javascript/sourcemap_header.sql +++ b/sql/2024/javascript/sourcemap_header.sql @@ -26,16 +26,16 @@ FROM ( page, getHeader(JSON_EXTRACT(payload, '$.response.headers'), 'SourceMap') IS NOT NULL AS has_sourcemap_header FROM - `httparchive.almanac.requests` + `httparchive.all.requests` WHERE - date = '2022-06-01' AND + date = '2024-06-01' AND type = 'script') JOIN ( SELECT _TABLE_SUFFIX AS client, COUNT(0) AS total_pages FROM - `httparchive.summary_pages.2022_06_01_*` + `httparchive.summary_pages.2024_06_01_*` GROUP BY client) USING diff --git a/sql/2024/javascript/sourcemap_header_by_3p.sql b/sql/2024/javascript/sourcemap_header_by_3p.sql index 7937b123806..1e91724d380 100644 --- a/sql/2024/javascript/sourcemap_header_by_3p.sql +++ b/sql/2024/javascript/sourcemap_header_by_3p.sql @@ -26,20 +26,20 @@ FROM ( client, page, IF(NET.HOST(url) IN ( - SELECT domain FROM `httparchive.almanac.third_parties` WHERE date = '2022-06-01' AND category != 'hosting' + SELECT domain FROM `httparchive.almanac.third_parties` WHERE date = '2024-06-01' AND category != 'hosting' ), 'third party', 'first party') AS host, getHeader(JSON_EXTRACT(payload, '$.response.headers'), 'SourceMap') IS NOT NULL AS has_sourcemap_header FROM - `httparchive.almanac.requests` + `httparchive.all.requests` WHERE - date = '2022-06-01' AND + date = '2024-06-01' AND type = 'script') JOIN ( SELECT _TABLE_SUFFIX AS client, COUNT(0) AS total_pages FROM - `httparchive.summary_pages.2022_06_01_*` + `httparchive.summary_pages.2024_06_01_*` GROUP BY client) USING diff --git a/sql/2024/javascript/sourcemaps.sql b/sql/2024/javascript/sourcemaps.sql index e9182b37199..e314b987826 100644 --- a/sql/2024/javascript/sourcemaps.sql +++ b/sql/2024/javascript/sourcemaps.sql @@ -1,7 +1,6 @@ #standardSQL # Number of pages with publicly available sourcemaps -# returns boolean whether the page has sourcemaps or not CREATE TEMPORARY FUNCTION getSourceMaps(payload STRING) RETURNS STRUCT LANGUAGE js AS ''' @@ -33,11 +32,12 @@ SELECT COUNTIF(sourcemaps.isPublic = true) / COUNT(0) AS pct_has_public_sourcemaps FROM ( SELECT - _TABLE_SUFFIX AS client, - url AS page, + client, + page, getSourceMaps(payload) AS sourcemaps FROM - `httparchive.pages.2022_06_01_*` + `httparchive.all.pages` + where date="2024-06-01" ) GROUP BY client diff --git a/sql/2024/javascript/sync_requests.sql b/sql/2024/javascript/sync_requests.sql index c8daf5c2b2f..84b3d82ec0a 100644 --- a/sql/2024/javascript/sync_requests.sql +++ b/sql/2024/javascript/sync_requests.sql @@ -6,7 +6,7 @@ SELECT FROM `httparchive.blink_features.usage` WHERE - yyyymmdd = '20220601' AND + yyyymmdd = '20240601' AND feature = 'XMLHttpRequestSynchronous' GROUP BY pct_urls, diff --git a/sql/2024/javascript/tbt.sql b/sql/2024/javascript/tbt.sql index 4226d4d3cd6..cf645b7bc85 100644 --- a/sql/2024/javascript/tbt.sql +++ b/sql/2024/javascript/tbt.sql @@ -7,7 +7,7 @@ WITH tbt_stats AS ( url, CAST(JSON_EXTRACT_SCALAR(report, '$.audits.total-blocking-time.numericValue') AS FLOAT64) AS tbtValue FROM - `httparchive.lighthouse.2022_06_01_*` + `httparchive.lighthouse.2024_06_01_*` ) SELECT diff --git a/sql/2024/javascript/time_to_interactive.sql b/sql/2024/javascript/time_to_interactive.sql index 3f58208f7f0..cacda04d7ef 100644 --- a/sql/2024/javascript/time_to_interactive.sql +++ b/sql/2024/javascript/time_to_interactive.sql @@ -9,7 +9,7 @@ FROM ( _TABLE_SUFFIX AS client, CAST(IFNULL(JSON_EXTRACT(report, '$.audits.consistently-interactive.numericValue'), JSON_EXTRACT(report, '$.audits.interactive.numericValue')) AS FLOAT64) AS tti FROM - `httparchive.lighthouse.2022_06_01_*`), + `httparchive.lighthouse.2024_06_01_*`), UNNEST([10, 25, 50, 75, 90]) AS percentile GROUP BY client, diff --git a/sql/2024/javascript/unused_js_bytes_distribution.sql b/sql/2024/javascript/unused_js_bytes_distribution.sql index 2d5aeefbb30..b2e2ea7ecfa 100644 --- a/sql/2024/javascript/unused_js_bytes_distribution.sql +++ b/sql/2024/javascript/unused_js_bytes_distribution.sql @@ -5,7 +5,7 @@ SELECT percentile, APPROX_QUANTILES(CAST(JSON_EXTRACT_SCALAR(report, '$.audits.unused-javascript.details.overallSavingsBytes') AS INT64) / 1024, 1000)[OFFSET(percentile * 10)] AS js_kilobytes FROM - `httparchive.lighthouse.2022_06_01_*`, + `httparchive.lighthouse.2024_06_01_*`, UNNEST([10, 25, 50, 75, 90, 100]) AS percentile GROUP BY client, diff --git a/sql/2024/javascript/usage_of_document_write.sql b/sql/2024/javascript/usage_of_document_write.sql index 7d3fa1fa056..172cd63a42f 100644 --- a/sql/2024/javascript/usage_of_document_write.sql +++ b/sql/2024/javascript/usage_of_document_write.sql @@ -22,7 +22,7 @@ WITH base AS ( _TABLE_SUFFIX AS client, lighthouse.url AS page FROM - `httparchive.lighthouse.2022_06_01_*` AS lighthouse, + `httparchive.lighthouse.2024_06_01_*` AS lighthouse, UNNEST(getUrls(JSON_EXTRACT(report, "$.audits['no-document-write']"))) AS data ) GROUP BY @@ -42,7 +42,7 @@ JOIN ( _TABLE_SUFFIX AS client, COUNT(DISTINCT url) AS total FROM - `httparchive.lighthouse.2022_06_01_*` + `httparchive.lighthouse.2024_06_01_*` GROUP BY client ) diff --git a/sql/2024/javascript/usage_of_legacy_javascript.sql b/sql/2024/javascript/usage_of_legacy_javascript.sql index 0b608280a12..9e5a99baf52 100644 --- a/sql/2024/javascript/usage_of_legacy_javascript.sql +++ b/sql/2024/javascript/usage_of_legacy_javascript.sql @@ -22,7 +22,7 @@ WITH base AS ( _TABLE_SUFFIX AS client, lighthouse.url AS page FROM - `httparchive.lighthouse.2022_06_01_*` AS lighthouse, + `httparchive.lighthouse.2024_06_01_*` AS lighthouse, UNNEST(getUrls(JSON_EXTRACT(report, "$.audits['legacy-javascript']"))) AS data ) GROUP BY @@ -42,7 +42,7 @@ JOIN ( _TABLE_SUFFIX AS client, COUNT(DISTINCT url) AS total FROM - `httparchive.lighthouse.2022_06_01_*` + `httparchive.lighthouse.2024_06_01_*` GROUP BY client ) diff --git a/sql/2024/javascript/usage_of_parcel.sql b/sql/2024/javascript/usage_of_parcel.sql index 798cdd057b1..2f9a3fc5ab7 100644 --- a/sql/2024/javascript/usage_of_parcel.sql +++ b/sql/2024/javascript/usage_of_parcel.sql @@ -6,7 +6,7 @@ WITH totals AS ( _TABLE_SUFFIX AS client, COUNT(DISTINCT url) AS total_pages FROM - `httparchive.summary_pages.2022_06_01_*` + `httparchive.summary_pages.2024_06_01_*` GROUP BY client ), @@ -16,7 +16,7 @@ parcel AS ( _TABLE_SUFFIX AS client, COUNT(DISTINCT url) AS parcel_pages FROM - `httparchive.technologies.2022_06_01_*` + `httparchive.technologies.2024_06_01_*` WHERE app = 'parcel' GROUP BY diff --git a/sql/2024/javascript/usage_of_parcel_by_rank.sql b/sql/2024/javascript/usage_of_parcel_by_rank.sql index f216a9883a0..d60f5ed09e4 100644 --- a/sql/2024/javascript/usage_of_parcel_by_rank.sql +++ b/sql/2024/javascript/usage_of_parcel_by_rank.sql @@ -1,12 +1,13 @@ #standardSQL # Percent of pages using parcel grouped by rank +# usage_of_parcel_by_rank.sql WITH parcel_pages AS ( SELECT _TABLE_SUFFIX AS client, url AS page FROM - `httparchive.technologies.2022_06_01_*` + `httparchive.technologies.2024_06_01_*` WHERE app = 'parcel' ), @@ -17,7 +18,7 @@ rank_totals AS ( rank_grouping, COUNT(0) AS total FROM - `httparchive.summary_pages.2022_06_01_*`, + `httparchive.summary_pages.2024_06_01_*`, UNNEST([1000, 10000, 100000, 1000000, 10000000]) AS rank_grouping WHERE rank <= rank_grouping @@ -32,7 +33,7 @@ pages AS ( rank_grouping, url AS page FROM - `httparchive.summary_pages.2022_06_01_*`, + `httparchive.summary_pages.2024_06_01_*`, UNNEST([1000, 10000, 100000, 1000000, 10000000]) AS rank_grouping WHERE rank <= rank_grouping diff --git a/sql/2024/javascript/usage_of_typescript_and_babel.sql b/sql/2024/javascript/usage_of_typescript_and_babel.sql index 70cc9e3723b..cde638f6eea 100644 --- a/sql/2024/javascript/usage_of_typescript_and_babel.sql +++ b/sql/2024/javascript/usage_of_typescript_and_babel.sql @@ -35,11 +35,12 @@ SELECT COUNTIF(sourcemaps.isTypeScript = true) / COUNT(0) AS pct_use_typescript FROM ( SELECT - _TABLE_SUFFIX AS client, - url AS page, + client, + page, getSourceMaps(payload) AS sourcemaps FROM - `httparchive.pages.2022_06_01_*` + `httparchive.all.pages` + where date='2024-06-01' ) WHERE sourcemaps.isPublic = true diff --git a/sql/2024/javascript/usage_of_typescript_and_babel_by_rank.sql b/sql/2024/javascript/usage_of_typescript_and_babel_by_rank.sql index 0f1566e569a..46faf137921 100644 --- a/sql/2024/javascript/usage_of_typescript_and_babel_by_rank.sql +++ b/sql/2024/javascript/usage_of_typescript_and_babel_by_rank.sql @@ -32,7 +32,7 @@ WITH pages AS ( rank_grouping, url AS page FROM - `httparchive.summary_pages.2022_06_01_*`, + `httparchive.summary_pages.2024_06_01_*`, UNNEST([1000, 10000, 100000, 1000000, 10000000]) AS rank_grouping WHERE rank <= rank_grouping @@ -44,7 +44,7 @@ pages_sourcemaps AS ( url AS page, getSourceMaps(payload) AS sourcemaps FROM - `httparchive.pages.2022_06_01_*` + `httparchive.pages.2024_06_01_*` ) SELECT diff --git a/sql/2024/javascript/usage_of_webpack.sql b/sql/2024/javascript/usage_of_webpack.sql index 4f2e08e8e3a..d1c358fbb69 100644 --- a/sql/2024/javascript/usage_of_webpack.sql +++ b/sql/2024/javascript/usage_of_webpack.sql @@ -6,7 +6,7 @@ WITH totals AS ( _TABLE_SUFFIX AS client, COUNT(DISTINCT url) AS total_pages FROM - `httparchive.summary_pages.2022_06_01_*` + `httparchive.summary_pages.2024_06_01_*` GROUP BY client ), @@ -16,9 +16,9 @@ webpack AS ( _TABLE_SUFFIX AS client, COUNT(DISTINCT url) AS webpack_pages FROM - `httparchive.technologies.2022_06_01_*` + `httparchive.technologies.2024_06_01_*` WHERE - app = 'webpack' + lower(app) = 'webpack' GROUP BY client ) diff --git a/sql/2024/javascript/usage_of_webpack_by_rank.sql b/sql/2024/javascript/usage_of_webpack_by_rank.sql index 84e55bce15a..80316905a8b 100644 --- a/sql/2024/javascript/usage_of_webpack_by_rank.sql +++ b/sql/2024/javascript/usage_of_webpack_by_rank.sql @@ -6,9 +6,9 @@ WITH webpack_pages AS ( _TABLE_SUFFIX AS client, url AS page FROM - `httparchive.technologies.2022_06_01_*` + `httparchive.technologies.2024_06_01_*` WHERE - app = 'webpack' + lower(app) = 'webpack' ), rank_totals AS ( @@ -17,7 +17,7 @@ rank_totals AS ( rank_grouping, COUNT(0) AS total FROM - `httparchive.summary_pages.2022_06_01_*`, + `httparchive.summary_pages.2024_06_01_*`, UNNEST([1000, 10000, 100000, 1000000, 10000000]) AS rank_grouping WHERE rank <= rank_grouping @@ -32,7 +32,7 @@ pages AS ( rank_grouping, url AS page FROM - `httparchive.summary_pages.2022_06_01_*`, + `httparchive.summary_pages.2024_06_01_*`, UNNEST([1000, 10000, 100000, 1000000, 10000000]) AS rank_grouping WHERE rank <= rank_grouping diff --git a/sql/2024/javascript/web_components_is_attribute.sql b/sql/2024/javascript/web_components_is_attribute.sql index 878652c4186..8fa5fdfd4dc 100644 --- a/sql/2024/javascript/web_components_is_attribute.sql +++ b/sql/2024/javascript/web_components_is_attribute.sql @@ -1,6 +1,5 @@ #standardSQL # pages which use `is` attribute - CREATE TEMPORARY FUNCTION get_almanac_attribute_info(almanac_string STRING) RETURNS ARRAY> LANGUAGE js AS ''' try { @@ -19,13 +18,14 @@ return []; '''; SELECT - _TABLE_SUFFIX AS client, + client, COUNTIF(almanac_attribute_info.name = 'is' AND almanac_attribute_info.freq > 0) AS freq, - SUM(COUNT(DISTINCT url)) OVER (PARTITION BY _TABLE_SUFFIX) AS total, - COUNTIF(almanac_attribute_info.name = 'is' AND almanac_attribute_info.freq > 0) / SUM(COUNT(DISTINCT url)) OVER (PARTITION BY _TABLE_SUFFIX) AS pct_pages + SUM(COUNT(DISTINCT page)) OVER (PARTITION BY client) AS total, + COUNTIF(almanac_attribute_info.name = 'is' AND almanac_attribute_info.freq > 0) / SUM(COUNT(DISTINCT page)) OVER (PARTITION BY client) AS pct_pages FROM - `httparchive.pages.2022_06_01_*`, + `httparchive.all.pages`, UNNEST(get_almanac_attribute_info(JSON_EXTRACT_SCALAR(payload, '$._almanac'))) AS almanac_attribute_info + where date="2024-06-01" GROUP BY client ORDER BY diff --git a/sql/2024/javascript/web_components_pct.sql b/sql/2024/javascript/web_components_pct.sql index 3ef720a7889..1f7b29b98ac 100644 --- a/sql/2024/javascript/web_components_pct.sql +++ b/sql/2024/javascript/web_components_pct.sql @@ -1,3 +1,4 @@ +# web_components_pct.sql SELECT client, custom_elements, @@ -16,9 +17,10 @@ FROM ( COUNTIF(ARRAY_LENGTH(JSON_EXTRACT_ARRAY(js, '$.web_component_specs.template')) > 0) AS templates FROM ( SELECT - _TABLE_SUFFIX AS client, + client, JSON_EXTRACT_SCALAR(payload, '$._javascript') AS js FROM - `httparchive.pages.2022_06_01_*`) + `httparchive.all.pages` + where date='2024-06-01') GROUP BY client) diff --git a/sql/2024/javascript/web_components_specs.sql b/sql/2024/javascript/web_components_specs.sql index ac77b374165..3c09424cdd5 100644 --- a/sql/2024/javascript/web_components_specs.sql +++ b/sql/2024/javascript/web_components_specs.sql @@ -1,3 +1,4 @@ +#web_components_specs.sql SELECT client, percentile, @@ -11,9 +12,8 @@ FROM ( ARRAY_LENGTH(JSON_EXTRACT_ARRAY(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.web_component_specs.shadow_roots')) AS shadow_roots, ARRAY_LENGTH(JSON_EXTRACT_ARRAY(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.web_component_specs.template')) AS template FROM - # Note: We're intentionally querying the September dataset here because of a bug in the custom metric. - # See https://github.com/HTTPArchive/legacy.httparchive.org/pull/231. - `httparchive.pages.2022_06_01_*`), + `httparchive.all.pages` where + date='2024-06-01'), UNNEST([10, 25, 50, 75, 90, 100]) AS percentile GROUP BY percentile, diff --git a/sql/2024/javascript/web_workers.sql b/sql/2024/javascript/web_workers.sql index 7b80d9927df..daf9ad612e6 100644 --- a/sql/2024/javascript/web_workers.sql +++ b/sql/2024/javascript/web_workers.sql @@ -1,13 +1,13 @@ # standardSQL # usage of web worker using blink features from usage counters - +# web_workers.sql SELECT client, pct_urls FROM `httparchive.blink_features.usage` WHERE - yyyymmdd = '20220601' AND + yyyymmdd = '20240601' AND feature = 'WorkerStart' GROUP BY pct_urls, diff --git a/sql/2024/javascript/worklets.sql b/sql/2024/javascript/worklets.sql index 4982c6935f4..c37ba52cad2 100644 --- a/sql/2024/javascript/worklets.sql +++ b/sql/2024/javascript/worklets.sql @@ -7,7 +7,7 @@ SELECT FROM `httparchive.blink_features.usage` WHERE - yyyymmdd = '20220601' AND + yyyymmdd = '20240601' AND ( feature = 'PaintWorklet' OR feature = 'AudioWorkletAddModule' From f468716c8564b9ff4f5b4bbba1efc2555a5f172f Mon Sep 17 00:00:00 2001 From: Nurullah Demir Date: Fri, 16 Aug 2024 17:55:38 +0200 Subject: [PATCH 03/53] bytes_by_3p.sql fixed --- sql/2024/javascript/bytes_by_3p.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/2024/javascript/bytes_by_3p.sql b/sql/2024/javascript/bytes_by_3p.sql index dadb14b177d..5f4940ba2fa 100644 --- a/sql/2024/javascript/bytes_by_3p.sql +++ b/sql/2024/javascript/bytes_by_3p.sql @@ -14,7 +14,7 @@ FROM ( ), 'third party', 'first party') AS host, SUM(cast(json_value(payload,"$.response.bodySize") as int64)) / 1024 AS kbytes FROM - `httparchive.all.requests` TABLESAMPLE SYSTEM (0.01 PERCENT) + `httparchive.all.requests` WHERE date = '2024-06-01' AND From 27a0bb64ff38e1bf6fb1548e4a7699de60a58827 Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 17:10:50 +0200 Subject: [PATCH 04/53] Update breakdown_of_pages_using_async_defer.sql fixing easy linter issue. --- sql/2024/javascript/breakdown_of_pages_using_async_defer.sql | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sql/2024/javascript/breakdown_of_pages_using_async_defer.sql b/sql/2024/javascript/breakdown_of_pages_using_async_defer.sql index 502c7ff6595..387957de8cf 100644 --- a/sql/2024/javascript/breakdown_of_pages_using_async_defer.sql +++ b/sql/2024/javascript/breakdown_of_pages_using_async_defer.sql @@ -17,7 +17,8 @@ SELECT FROM `httparchive.all.pages` WHERE -WHERE date = '2024-06-01' - +WHERE + date = '2024-06-01' + GROUP BY client From 46b3ccf2d1e5a12557a6960cc8769061862a90c2 Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 17:11:38 +0200 Subject: [PATCH 05/53] Update breakdown_of_scripts_using_async_defer_module_nomodule.sql fixing linter error --- .../breakdown_of_scripts_using_async_defer_module_nomodule.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/2024/javascript/breakdown_of_scripts_using_async_defer_module_nomodule.sql b/sql/2024/javascript/breakdown_of_scripts_using_async_defer_module_nomodule.sql index be5a3b22edb..2cbe8f9723b 100644 --- a/sql/2024/javascript/breakdown_of_scripts_using_async_defer_module_nomodule.sql +++ b/sql/2024/javascript/breakdown_of_scripts_using_async_defer_module_nomodule.sql @@ -45,4 +45,4 @@ FROM date = '2024-06-01' ) GROUP BY - client \ No newline at end of file + client From 044b2fcccbe89fc16ffc3b9e194d24a0763eaf16 Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 17:13:03 +0200 Subject: [PATCH 06/53] Update distribution_of_injected_scripts.sql Fixing linter errors --- sql/2024/javascript/distribution_of_injected_scripts.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sql/2024/javascript/distribution_of_injected_scripts.sql b/sql/2024/javascript/distribution_of_injected_scripts.sql index 7e8b91eb9ac..cd9ab3714a5 100644 --- a/sql/2024/javascript/distribution_of_injected_scripts.sql +++ b/sql/2024/javascript/distribution_of_injected_scripts.sql @@ -38,12 +38,12 @@ SELECT APPROX_QUANTILES(IF(scripts.scripts > 0, scripts.injectedScripts / scripts.scripts, 0), 1000)[OFFSET(percentile * 10)] AS pct_injected_scripts_per_page FROM ( SELECT - client, - page, + client, + page, getScripts(payload) AS scripts FROM `httparchive.all.pages` - where + WHERE date = "2024-06-01" ), UNNEST([10, 25, 50, 75, 90, 100]) AS percentile From bf995896c76a6b8ae1293d8dc13174a077d0e7b2 Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 17:13:58 +0200 Subject: [PATCH 07/53] Update distribution_of_long_tasks_time.sql Fixing linter --- sql/2024/javascript/distribution_of_long_tasks_time.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/2024/javascript/distribution_of_long_tasks_time.sql b/sql/2024/javascript/distribution_of_long_tasks_time.sql index 9507c8e55ce..54489d6c75b 100644 --- a/sql/2024/javascript/distribution_of_long_tasks_time.sql +++ b/sql/2024/javascript/distribution_of_long_tasks_time.sql @@ -21,7 +21,7 @@ try { WITH long_tasks_pages AS ( SELECT - client, + client, page as url, _longTasks AS long_tasks FROM From 1a22470e09726499c7fbe6347a129e137945ce94 Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 17:14:57 +0200 Subject: [PATCH 08/53] Update injected_scripts.sql fixing linter errors --- sql/2024/javascript/injected_scripts.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/2024/javascript/injected_scripts.sql b/sql/2024/javascript/injected_scripts.sql index ab86eca902d..f26239456fa 100644 --- a/sql/2024/javascript/injected_scripts.sql +++ b/sql/2024/javascript/injected_scripts.sql @@ -39,12 +39,12 @@ SELECT COUNTIF(scripts.injectedScripts > 0) / COUNT(0) AS pct_injected_scripts FROM ( SELECT - client, + client, page, getScripts(payload) AS scripts FROM `httparchive.all.pages` - WHERE + WHERE date = '2024-06-01' ) GROUP BY From 416aa4f2fa61321273b41598acc18539a901df85 Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 17:16:30 +0200 Subject: [PATCH 09/53] Update percent_of_dns_prefetch.sql Fixing Linter --- sql/2024/javascript/percent_of_dns_prefetch.sql | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/sql/2024/javascript/percent_of_dns_prefetch.sql b/sql/2024/javascript/percent_of_dns_prefetch.sql index e181255e22e..36b24338b26 100644 --- a/sql/2024/javascript/percent_of_dns_prefetch.sql +++ b/sql/2024/javascript/percent_of_dns_prefetch.sql @@ -20,13 +20,14 @@ WITH resource_hints AS ( host FROM ( SELECT - client, - page, + client, + page, NET.HOST(href) AS host FROM `httparchive.all.pages`, - UNNEST(getResourceHintsHrefs(payload, 'dns-prefetch')) AS href - where date = "2024-06-01" + UNNEST(getResourceHintsHrefs(payload, 'dns-prefetch')) AS href + WHERE + date = "2024-06-01" ) GROUP BY client, From 12c9dd391c7e097c21b28e8a37c2ad7aa85e3bac Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 17:17:45 +0200 Subject: [PATCH 10/53] Update percent_of_preconnect_by_host.sql Fixing linter errors --- sql/2024/javascript/percent_of_preconnect_by_host.sql | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/sql/2024/javascript/percent_of_preconnect_by_host.sql b/sql/2024/javascript/percent_of_preconnect_by_host.sql index 8d2c9920912..2d46c5eecb6 100644 --- a/sql/2024/javascript/percent_of_preconnect_by_host.sql +++ b/sql/2024/javascript/percent_of_preconnect_by_host.sql @@ -20,13 +20,14 @@ WITH resource_hints AS ( host FROM ( SELECT - client, - page, + client, + page, NET.HOST(href) AS host FROM `httparchive.all.pages`, UNNEST(getResourceHintsHrefs(payload, 'preconnect')) AS href - where date = "2024-06-01" + WHERE + date = "2024-06-01" ) GROUP BY client, From 7fbce855a381456714a9b337dd23a2dd105ad652 Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 17:19:04 +0200 Subject: [PATCH 11/53] Update render_blocking_javascript.sql Fixing linter errors --- sql/2024/javascript/render_blocking_javascript.sql | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sql/2024/javascript/render_blocking_javascript.sql b/sql/2024/javascript/render_blocking_javascript.sql index 0bd73cfeb5b..3224d5cb2ea 100644 --- a/sql/2024/javascript/render_blocking_javascript.sql +++ b/sql/2024/javascript/render_blocking_javascript.sql @@ -15,12 +15,13 @@ try { WITH render_blocking_scripts AS ( SELECT - client, + client, page as url, getRenderBlockingScripts(payload) AS number_of_render_blocking_scripts FROM `httparchive.all.pages` - where date="2024-06-01" + WHERE + date="2024-06-01" ) SELECT From d3da1b3934d712f1eb99fee72303856c102ebc3f Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 17:19:51 +0200 Subject: [PATCH 12/53] Update render_blocking_javascript_by_rank.sql Fixing Linter errors --- sql/2024/javascript/render_blocking_javascript_by_rank.sql | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sql/2024/javascript/render_blocking_javascript_by_rank.sql b/sql/2024/javascript/render_blocking_javascript_by_rank.sql index 483d3c4794a..769d8ecc8a0 100644 --- a/sql/2024/javascript/render_blocking_javascript_by_rank.sql +++ b/sql/2024/javascript/render_blocking_javascript_by_rank.sql @@ -20,12 +20,13 @@ WITH render_blocking_scripts AS ( number_of_render_blocking_scripts FROM ( SELECT - client, + client, page as url, getRenderBlockingScripts(payload) AS number_of_render_blocking_scripts FROM `httparchive.all.pages` - where date="2024-06-01" + WHERE + date="2024-06-01" ) JOIN ( SELECT From 75e1a9d26c86097db4b779c7ac58f8756b74fe13 Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 17:21:59 +0200 Subject: [PATCH 13/53] Update resource-hints-prefetch-preload-modulepreload-percentage.sql Fixing Linter issues --- ...rce-hints-prefetch-preload-modulepreload-percentage.sql | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/sql/2024/javascript/resource-hints-prefetch-preload-modulepreload-percentage.sql b/sql/2024/javascript/resource-hints-prefetch-preload-modulepreload-percentage.sql index bf98f2f395c..21280302d87 100644 --- a/sql/2024/javascript/resource-hints-prefetch-preload-modulepreload-percentage.sql +++ b/sql/2024/javascript/resource-hints-prefetch-preload-modulepreload-percentage.sql @@ -39,8 +39,8 @@ SELECT COUNT(DISTINCT IF(modulepreload_hint, page, NULL)) / COUNT(DISTINCT page) AS modulepreload_pct FROM ( SELECT - client, - page, + client, + page, hint.name = 'prefetch' AND hint.value = 'script' AS prefetch_hint, hint.name = 'preload' AND hint.value = 'script' AS preload_hint, hint.name = 'modulepreload' AND hint.value = 'script' AS modulepreload_hint @@ -48,7 +48,8 @@ FROM ( `httparchive.all.pages` LEFT JOIN UNNEST(getResourceHintAttrs(payload)) AS hint - where date ="2024-06-01" + WHERE + date ="2024-06-01" ) GROUP BY client From ac1fb96b779d24580baf16fdebc4d135dd04e534 Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 17:22:54 +0200 Subject: [PATCH 14/53] Update resource-hints-preload-prefetch-modulepreload-distribution.sql Linter errors --- ...e-hints-preload-prefetch-modulepreload-distribution.sql | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/sql/2024/javascript/resource-hints-preload-prefetch-modulepreload-distribution.sql b/sql/2024/javascript/resource-hints-preload-prefetch-modulepreload-distribution.sql index 86698ff9932..90811942892 100644 --- a/sql/2024/javascript/resource-hints-preload-prefetch-modulepreload-distribution.sql +++ b/sql/2024/javascript/resource-hints-preload-prefetch-modulepreload-distribution.sql @@ -39,8 +39,8 @@ SELECT APPROX_QUANTILES(IF(modulepreload_hint = 0, NULL, modulepreload_hint), 1000 IGNORE NULLS)[OFFSET(percentile * 10)] AS modulepreload_hints_per_page_with_hints FROM ( SELECT - client, - page, + client, + page, COUNTIF(hint.name = 'prefetch' AND hint.value = 'script') AS prefetch_hint, COUNTIF(hint.name = 'preload' AND hint.value = 'script') AS preload_hint, COUNTIF(hint.name = 'modulepreload' AND hint.value = 'script') AS modulepreload_hint @@ -48,7 +48,8 @@ FROM ( `httparchive.all.pages` LEFT JOIN UNNEST(getResourceHintAttrs(payload)) AS hint - where date ="2024-06-01" + WHERE + date ="2024-06-01" GROUP BY client, From 87fa25d5ffde998ebc65e909c3888d89f80dd1c7 Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 17:23:50 +0200 Subject: [PATCH 15/53] Update resource_hints.sql Fixing Linter errors. --- sql/2024/javascript/resource_hints.sql | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/sql/2024/javascript/resource_hints.sql b/sql/2024/javascript/resource_hints.sql index 125066f9abb..f8c4accb79f 100644 --- a/sql/2024/javascript/resource_hints.sql +++ b/sql/2024/javascript/resource_hints.sql @@ -37,14 +37,15 @@ SELECT COUNT(DISTINCT IF(script_hint, page, NULL)) / COUNT(DISTINCT page) AS pct FROM ( SELECT - client, - page, + client, + page, hint.name IN ('prefetch', 'preload', 'modulepreload') AND hint.value = 'script' AS script_hint FROM `httparchive.all.pages` LEFT JOIN UNNEST(getResourceHintAttrs(payload)) AS hint - where date = "2024-06-01" + WHERE + date = "2024-06-01" ) GROUP BY client From c61e7465899d4ee88ef32508eef1135741dc0caa Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 17:25:03 +0200 Subject: [PATCH 16/53] Update resource_hints_per_page.sql Fixing Linter errors --- sql/2024/javascript/resource_hints_per_page.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sql/2024/javascript/resource_hints_per_page.sql b/sql/2024/javascript/resource_hints_per_page.sql index 25facd1dba7..e3669dc8147 100644 --- a/sql/2024/javascript/resource_hints_per_page.sql +++ b/sql/2024/javascript/resource_hints_per_page.sql @@ -36,15 +36,15 @@ SELECT APPROX_QUANTILES(IF(script_hint = 0, NULL, script_hint), 1000 IGNORE NULLS)[OFFSET(percentile * 10)] AS hints_per_page_with_hints FROM ( SELECT - client, - page, + client, + page, COUNTIF(hint.name IN ('prefetch', 'preload', 'modulepreload') AND hint.value = 'script') AS script_hint FROM `httparchive.all.pages` LEFT JOIN UNNEST(getResourceHintAttrs(payload)) AS hint - - where date="2024-06-01" + WHERE + date="2024-06-01" GROUP BY client, page), From 03e11d3e0307a1e68e1901358e3050c41ad3b039 Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 17:26:11 +0200 Subject: [PATCH 17/53] Update sourcemaps.sql Fixing Linter Errors --- sql/2024/javascript/sourcemaps.sql | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/sql/2024/javascript/sourcemaps.sql b/sql/2024/javascript/sourcemaps.sql index e314b987826..9904e7d5615 100644 --- a/sql/2024/javascript/sourcemaps.sql +++ b/sql/2024/javascript/sourcemaps.sql @@ -32,12 +32,13 @@ SELECT COUNTIF(sourcemaps.isPublic = true) / COUNT(0) AS pct_has_public_sourcemaps FROM ( SELECT - client, - page, + client, + page, getSourceMaps(payload) AS sourcemaps FROM `httparchive.all.pages` - where date="2024-06-01" + WHERE + date="2024-06-01" ) GROUP BY client From a79bf12ede1b788f200d59f875eb8f789f28bb94 Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 17:27:09 +0200 Subject: [PATCH 18/53] Update usage_of_typescript_and_babel.sql Fixing linter errors --- sql/2024/javascript/usage_of_typescript_and_babel.sql | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/sql/2024/javascript/usage_of_typescript_and_babel.sql b/sql/2024/javascript/usage_of_typescript_and_babel.sql index cde638f6eea..66c7bb812c4 100644 --- a/sql/2024/javascript/usage_of_typescript_and_babel.sql +++ b/sql/2024/javascript/usage_of_typescript_and_babel.sql @@ -35,12 +35,13 @@ SELECT COUNTIF(sourcemaps.isTypeScript = true) / COUNT(0) AS pct_use_typescript FROM ( SELECT - client, - page, + client, + page, getSourceMaps(payload) AS sourcemaps FROM `httparchive.all.pages` - where date='2024-06-01' + WHERE + date='2024-06-01' ) WHERE sourcemaps.isPublic = true From e7a585df736faa3e7646214ffce74ceab2d60ff8 Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 17:28:35 +0200 Subject: [PATCH 19/53] Update web_components_is_attribute.sql Linter error... --- sql/2024/javascript/web_components_is_attribute.sql | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sql/2024/javascript/web_components_is_attribute.sql b/sql/2024/javascript/web_components_is_attribute.sql index 8fa5fdfd4dc..6bddf611361 100644 --- a/sql/2024/javascript/web_components_is_attribute.sql +++ b/sql/2024/javascript/web_components_is_attribute.sql @@ -18,14 +18,15 @@ return []; '''; SELECT - client, + client, COUNTIF(almanac_attribute_info.name = 'is' AND almanac_attribute_info.freq > 0) AS freq, SUM(COUNT(DISTINCT page)) OVER (PARTITION BY client) AS total, COUNTIF(almanac_attribute_info.name = 'is' AND almanac_attribute_info.freq > 0) / SUM(COUNT(DISTINCT page)) OVER (PARTITION BY client) AS pct_pages FROM `httparchive.all.pages`, UNNEST(get_almanac_attribute_info(JSON_EXTRACT_SCALAR(payload, '$._almanac'))) AS almanac_attribute_info - where date="2024-06-01" + WHERE + date="2024-06-01" GROUP BY client ORDER BY From 42907b1a1fe6f8375b315b4d131a124e3bd02a7e Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 17:29:22 +0200 Subject: [PATCH 20/53] Update web_components_pct.sql Fixing linter --- sql/2024/javascript/web_components_pct.sql | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sql/2024/javascript/web_components_pct.sql b/sql/2024/javascript/web_components_pct.sql index 1f7b29b98ac..14bc7fff28a 100644 --- a/sql/2024/javascript/web_components_pct.sql +++ b/sql/2024/javascript/web_components_pct.sql @@ -17,10 +17,11 @@ FROM ( COUNTIF(ARRAY_LENGTH(JSON_EXTRACT_ARRAY(js, '$.web_component_specs.template')) > 0) AS templates FROM ( SELECT - client, + client, JSON_EXTRACT_SCALAR(payload, '$._javascript') AS js FROM `httparchive.all.pages` - where date='2024-06-01') + WHERE + date='2024-06-01') GROUP BY client) From 85de9f7841c4214a396f24d5cdba49dd3a590efb Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 17:30:40 +0200 Subject: [PATCH 21/53] Update breakdown_of_pages_using_async_defer.sql This just had to be a typo. Where should just be there once. Linter error. --- sql/2024/javascript/breakdown_of_pages_using_async_defer.sql | 1 - 1 file changed, 1 deletion(-) diff --git a/sql/2024/javascript/breakdown_of_pages_using_async_defer.sql b/sql/2024/javascript/breakdown_of_pages_using_async_defer.sql index 387957de8cf..0c52685ee8a 100644 --- a/sql/2024/javascript/breakdown_of_pages_using_async_defer.sql +++ b/sql/2024/javascript/breakdown_of_pages_using_async_defer.sql @@ -16,7 +16,6 @@ SELECT COUNTIF(CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.script_tags.async') AS INT64) = 0 AND CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.script_tags.defer') AS INT64) = 0) / COUNT(0) AS neither_pct FROM `httparchive.all.pages` -WHERE WHERE date = '2024-06-01' From 6718ceadfe8889928f31c56633074e219d67c926 Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 17:32:17 +0200 Subject: [PATCH 22/53] Update breakdown_of_scripts_using_async_defer_module_nomodule.sql Spacing thanks to linter. --- ...f_scripts_using_async_defer_module_nomodule.sql | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/sql/2024/javascript/breakdown_of_scripts_using_async_defer_module_nomodule.sql b/sql/2024/javascript/breakdown_of_scripts_using_async_defer_module_nomodule.sql index 2cbe8f9723b..fd1f76c8d9c 100644 --- a/sql/2024/javascript/breakdown_of_scripts_using_async_defer_module_nomodule.sql +++ b/sql/2024/javascript/breakdown_of_scripts_using_async_defer_module_nomodule.sql @@ -1,13 +1,13 @@ #standardSQL # Breakdown of scripts using Async, Defer, Module or NoModule attributes. Also breakdown of inline vs external scripts CREATE TEMPORARY FUNCTION getScripts(payload STRING) RETURNS STRUCT < total INT64, -inline INT64, -src INT64, -async INT64, -defer INT64, -async_and_defer INT64, -type_module INT64, -nomodule INT64 > LANGUAGE js AS ''' + inline INT64, + src INT64, + async INT64, + defer INT64, + async_and_defer INT64, + type_module INT64, + nomodule INT64 > LANGUAGE js AS ''' try { var $ = JSON.parse(payload); var javascript = JSON.parse($._javascript); From b1441adbbf64dffcf2025ac62b4d45883c077296 Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 17:38:42 +0200 Subject: [PATCH 23/53] Update resource-hints-prefetch-preload-modulepreload-percentage.sql trailing white space --- ...resource-hints-prefetch-preload-modulepreload-percentage.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/2024/javascript/resource-hints-prefetch-preload-modulepreload-percentage.sql b/sql/2024/javascript/resource-hints-prefetch-preload-modulepreload-percentage.sql index 21280302d87..65a5c03080d 100644 --- a/sql/2024/javascript/resource-hints-prefetch-preload-modulepreload-percentage.sql +++ b/sql/2024/javascript/resource-hints-prefetch-preload-modulepreload-percentage.sql @@ -48,7 +48,7 @@ FROM ( `httparchive.all.pages` LEFT JOIN UNNEST(getResourceHintAttrs(payload)) AS hint - WHERE + WHERE date ="2024-06-01" ) GROUP BY From 111892364055e5b4e617e51280d2c1982a2a8603 Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 17:39:23 +0200 Subject: [PATCH 24/53] Update resource-hints-preload-prefetch-modulepreload-distribution.sql trailing spaces --- ...urce-hints-preload-prefetch-modulepreload-distribution.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/2024/javascript/resource-hints-preload-prefetch-modulepreload-distribution.sql b/sql/2024/javascript/resource-hints-preload-prefetch-modulepreload-distribution.sql index 90811942892..05a2dd23909 100644 --- a/sql/2024/javascript/resource-hints-preload-prefetch-modulepreload-distribution.sql +++ b/sql/2024/javascript/resource-hints-preload-prefetch-modulepreload-distribution.sql @@ -48,9 +48,9 @@ FROM ( `httparchive.all.pages` LEFT JOIN UNNEST(getResourceHintAttrs(payload)) AS hint - WHERE + WHERE date ="2024-06-01" - + GROUP BY client, page), From 17f86f31d40a04e8c747445a389ba69f53871df7 Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 17:39:59 +0200 Subject: [PATCH 25/53] Update resource_hints.sql trailing space --- sql/2024/javascript/resource_hints.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/2024/javascript/resource_hints.sql b/sql/2024/javascript/resource_hints.sql index f8c4accb79f..ac96b82ae36 100644 --- a/sql/2024/javascript/resource_hints.sql +++ b/sql/2024/javascript/resource_hints.sql @@ -43,7 +43,7 @@ FROM ( FROM `httparchive.all.pages` LEFT JOIN - UNNEST(getResourceHintAttrs(payload)) AS hint + UNNEST(getResourceHintAttrs(payload)) AS hint WHERE date = "2024-06-01" ) From 27a7ce232f48e5abbf948c8ed9a9da7a27cd64f2 Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 17:40:45 +0200 Subject: [PATCH 26/53] Update breakdown_of_pages_using_async_defer.sql trailing white space --- sql/2024/javascript/breakdown_of_pages_using_async_defer.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/2024/javascript/breakdown_of_pages_using_async_defer.sql b/sql/2024/javascript/breakdown_of_pages_using_async_defer.sql index 0c52685ee8a..6fc6e0549f2 100644 --- a/sql/2024/javascript/breakdown_of_pages_using_async_defer.sql +++ b/sql/2024/javascript/breakdown_of_pages_using_async_defer.sql @@ -16,7 +16,7 @@ SELECT COUNTIF(CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.script_tags.async') AS INT64) = 0 AND CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.script_tags.defer') AS INT64) = 0) / COUNT(0) AS neither_pct FROM `httparchive.all.pages` -WHERE +WHERE date = '2024-06-01' GROUP BY From 2cc80b45a63dc03aa796868b6ffad0cc1c92374c Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 17:42:29 +0200 Subject: [PATCH 27/53] Update bytes_by_3p.sql fixing linter errors --- sql/2024/javascript/bytes_by_3p.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/2024/javascript/bytes_by_3p.sql b/sql/2024/javascript/bytes_by_3p.sql index 5f4940ba2fa..39d76ef6b49 100644 --- a/sql/2024/javascript/bytes_by_3p.sql +++ b/sql/2024/javascript/bytes_by_3p.sql @@ -12,7 +12,7 @@ FROM ( IF(NET.HOST(url) IN ( SELECT domain FROM `httparchive.almanac.third_parties` WHERE date = '2024-06-01' AND category != 'hosting' ), 'third party', 'first party') AS host, - SUM(cast(json_value(payload,"$.response.bodySize") as int64)) / 1024 AS kbytes + SUM(cast(json_value(payload, '$.response.bodySize') AS int64)) / 1024 AS kbytes FROM `httparchive.all.requests` From 0c40db8c478ef7d3116074ace3438ebf0351eeb7 Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 17:43:18 +0200 Subject: [PATCH 28/53] Update distribution_of_injected_scripts.sql linter prefers ' --- sql/2024/javascript/distribution_of_injected_scripts.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/2024/javascript/distribution_of_injected_scripts.sql b/sql/2024/javascript/distribution_of_injected_scripts.sql index cd9ab3714a5..132925ec67c 100644 --- a/sql/2024/javascript/distribution_of_injected_scripts.sql +++ b/sql/2024/javascript/distribution_of_injected_scripts.sql @@ -44,7 +44,7 @@ FROM ( FROM `httparchive.all.pages` WHERE - date = "2024-06-01" + date = '2024-06-01' ), UNNEST([10, 25, 50, 75, 90, 100]) AS percentile WHERE From 90db4ad1d076d374cecbe38ee514ec62cffb2eab Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 17:44:17 +0200 Subject: [PATCH 29/53] Update distribution_of_long_tasks_time.sql adjusting for linter --- sql/2024/javascript/distribution_of_long_tasks_time.sql | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sql/2024/javascript/distribution_of_long_tasks_time.sql b/sql/2024/javascript/distribution_of_long_tasks_time.sql index 54489d6c75b..4b4fa845ab4 100644 --- a/sql/2024/javascript/distribution_of_long_tasks_time.sql +++ b/sql/2024/javascript/distribution_of_long_tasks_time.sql @@ -22,12 +22,13 @@ try { WITH long_tasks_pages AS ( SELECT client, - page as url, + page AS url, _longTasks AS long_tasks FROM `httparchive.all.pages`, UNNEST(getLongTasks(payload)) AS _longTasks - where date = "2024-06-01" + WHERE + date = '2024-06-01' ), long_tasks_by_page AS ( From de4fef75427d7f177b9eefd0db5f9002819466aa Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 17:45:25 +0200 Subject: [PATCH 30/53] Update distribution_of_number_of_long_tasks.sql Linter --- sql/2024/javascript/distribution_of_number_of_long_tasks.sql | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sql/2024/javascript/distribution_of_number_of_long_tasks.sql b/sql/2024/javascript/distribution_of_number_of_long_tasks.sql index c4e9e32ccef..11409a5431d 100644 --- a/sql/2024/javascript/distribution_of_number_of_long_tasks.sql +++ b/sql/2024/javascript/distribution_of_number_of_long_tasks.sql @@ -22,12 +22,13 @@ try { WITH long_tasks_pages AS ( SELECT client, - page as url, + page AS url, _longTasks AS long_tasks FROM `httparchive.all.pages`, UNNEST(getLongTasks(payload)) AS _longTasks - where date = "2024-06-01" + WHERE + date = '2024-06-01' ), long_tasks_by_page AS ( From 6c3fce854ad8dc794a71ad60cda69c2b8276bc3c Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 17:48:08 +0200 Subject: [PATCH 31/53] Update frameworks_libraries_by_version.sql Spac ing --- sql/2024/javascript/frameworks_libraries_by_version.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/2024/javascript/frameworks_libraries_by_version.sql b/sql/2024/javascript/frameworks_libraries_by_version.sql index b5360d4382b..6bf886f8724 100644 --- a/sql/2024/javascript/frameworks_libraries_by_version.sql +++ b/sql/2024/javascript/frameworks_libraries_by_version.sql @@ -21,7 +21,7 @@ JOIN ( USING (_TABLE_SUFFIX) WHERE -app IN ('jQuery', 'jQuery Migrate', 'jQuery UI', 'Modernizr', 'FancyBox', 'Slick', 'Lightbox', 'Moment.js', 'Underscore.js', 'Lodash', 'React', 'GSAP', 'Vue.js', 'styled-components', 'Emotion', 'Backbone.js', 'RequireJS', 'AngularJS', 'AMP', 'Redux', 'Next.js', 'Stimulus', 'Angular', 'Handlebars', 'Zone.js', 'Marionette.js', 'Mustache', 'Prototype', 'toastr', 'JSS', 'React Router', 'Nuxt.js', 'Alpine.js', 'Svelte', 'MooTools', 'Knockout.js', 'Socket.io', 'Gatsby', 'Adobe Client Data Layer') + app IN ('jQuery', 'jQuery Migrate', 'jQuery UI', 'Modernizr', 'FancyBox', 'Slick', 'Lightbox', 'Moment.js', 'Underscore.js', 'Lodash', 'React', 'GSAP', 'Vue.js', 'styled-components', 'Emotion', 'Backbone.js', 'RequireJS', 'AngularJS', 'AMP', 'Redux', 'Next.js', 'Stimulus', 'Angular', 'Handlebars', 'Zone.js', 'Marionette.js', 'Mustache', 'Prototype', 'toastr', 'JSS', 'React Router', 'Nuxt.js', 'Alpine.js', 'Svelte', 'MooTools', 'Knockout.js', 'Socket.io', 'Gatsby', 'Adobe Client Data Layer') GROUP BY client, category, From e52da82cbfeaccd7948df647f9bdb4254da68469 Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 17:48:48 +0200 Subject: [PATCH 32/53] Update injected_scripts.sql Trailing spaces --- sql/2024/javascript/injected_scripts.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/2024/javascript/injected_scripts.sql b/sql/2024/javascript/injected_scripts.sql index f26239456fa..28da47efc7d 100644 --- a/sql/2024/javascript/injected_scripts.sql +++ b/sql/2024/javascript/injected_scripts.sql @@ -44,7 +44,7 @@ FROM ( getScripts(payload) AS scripts FROM `httparchive.all.pages` - WHERE + WHERE date = '2024-06-01' ) GROUP BY From 39885e62c5dde4a61304f2b7cac77dd4467ad1f6 Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 17:49:42 +0200 Subject: [PATCH 33/53] Update module_and_nomodule.sql adjusting for linter --- sql/2024/javascript/module_and_nomodule.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/2024/javascript/module_and_nomodule.sql b/sql/2024/javascript/module_and_nomodule.sql index b580cd4e6b5..29c129a6486 100644 --- a/sql/2024/javascript/module_and_nomodule.sql +++ b/sql/2024/javascript/module_and_nomodule.sql @@ -21,7 +21,7 @@ FROM ( UNNEST(REGEXP_EXTRACT_ALL(response_body, r'(?i)(]*>)')) AS script WHERE date = '2024-06-01' AND - cast(json_value(summary,"$.firstHtml") as bool) + cast(json_value(summary,'$.firstHtml') AS bool) ) GROUP BY client From 2cb6ddd1ab36d0253d396fca7cc5cf3583d0eba2 Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 17:50:35 +0200 Subject: [PATCH 34/53] Update percent_of_dns_prefetch.sql Adjusting for linter --- sql/2024/javascript/percent_of_dns_prefetch.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/2024/javascript/percent_of_dns_prefetch.sql b/sql/2024/javascript/percent_of_dns_prefetch.sql index 36b24338b26..2fead821b06 100644 --- a/sql/2024/javascript/percent_of_dns_prefetch.sql +++ b/sql/2024/javascript/percent_of_dns_prefetch.sql @@ -27,7 +27,7 @@ WITH resource_hints AS ( `httparchive.all.pages`, UNNEST(getResourceHintsHrefs(payload, 'dns-prefetch')) AS href WHERE - date = "2024-06-01" + date = '2024-06-01' ) GROUP BY client, From bdecf7f5a22aba409174a70b0095f7a987f684db Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 17:51:54 +0200 Subject: [PATCH 35/53] Update percent_of_dns_prefetch_by_host.sql Linter --- sql/2024/javascript/percent_of_dns_prefetch_by_host.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sql/2024/javascript/percent_of_dns_prefetch_by_host.sql b/sql/2024/javascript/percent_of_dns_prefetch_by_host.sql index 7ab1e176366..2aa2d6e1e0a 100644 --- a/sql/2024/javascript/percent_of_dns_prefetch_by_host.sql +++ b/sql/2024/javascript/percent_of_dns_prefetch_by_host.sql @@ -25,7 +25,8 @@ WITH resource_hints AS ( FROM `httparchive.all.pages`, UNNEST(getResourceHintsHrefs(payload, 'dns-prefetch')) AS href - where date='2024-06-01' + WHERE + date = '2024-06-01' ) GROUP BY client, From dff6ab3e0fd0afede504c0f4a1022a0a04f9918c Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 17:52:36 +0200 Subject: [PATCH 36/53] Update percent_of_preconnect_by_host.sql Linter --- sql/2024/javascript/percent_of_preconnect_by_host.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/2024/javascript/percent_of_preconnect_by_host.sql b/sql/2024/javascript/percent_of_preconnect_by_host.sql index 2d46c5eecb6..44a03d9e676 100644 --- a/sql/2024/javascript/percent_of_preconnect_by_host.sql +++ b/sql/2024/javascript/percent_of_preconnect_by_host.sql @@ -27,7 +27,7 @@ WITH resource_hints AS ( `httparchive.all.pages`, UNNEST(getResourceHintsHrefs(payload, 'preconnect')) AS href WHERE - date = "2024-06-01" + date = '2024-06-01' ) GROUP BY client, From 5438910c97b944adaa00193d1da234db32e7aec9 Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 17:53:27 +0200 Subject: [PATCH 37/53] Update render_blocking_javascript.sql Linter --- sql/2024/javascript/render_blocking_javascript.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/2024/javascript/render_blocking_javascript.sql b/sql/2024/javascript/render_blocking_javascript.sql index 3224d5cb2ea..530d9b617dc 100644 --- a/sql/2024/javascript/render_blocking_javascript.sql +++ b/sql/2024/javascript/render_blocking_javascript.sql @@ -16,12 +16,12 @@ try { WITH render_blocking_scripts AS ( SELECT client, - page as url, + page AS url, getRenderBlockingScripts(payload) AS number_of_render_blocking_scripts FROM `httparchive.all.pages` WHERE - date="2024-06-01" + date = '2024-06-01' ) SELECT From edb7e4dccec22d4278d8a289e8604d815384f002 Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 17:54:11 +0200 Subject: [PATCH 38/53] Update render_blocking_javascript_by_rank.sql Linter --- sql/2024/javascript/render_blocking_javascript_by_rank.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/2024/javascript/render_blocking_javascript_by_rank.sql b/sql/2024/javascript/render_blocking_javascript_by_rank.sql index 769d8ecc8a0..ac01734ec98 100644 --- a/sql/2024/javascript/render_blocking_javascript_by_rank.sql +++ b/sql/2024/javascript/render_blocking_javascript_by_rank.sql @@ -21,12 +21,12 @@ WITH render_blocking_scripts AS ( FROM ( SELECT client, - page as url, + page AS url, getRenderBlockingScripts(payload) AS number_of_render_blocking_scripts FROM `httparchive.all.pages` WHERE - date="2024-06-01" + date = '2024-06-01' ) JOIN ( SELECT From 70c651b63dca68932c7d14a9ee46655830729137 Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 17:56:11 +0200 Subject: [PATCH 39/53] Update resource-hints-prefetch-preload-modulepreload-percentage.sql Linter --- ...urce-hints-prefetch-preload-modulepreload-percentage.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sql/2024/javascript/resource-hints-prefetch-preload-modulepreload-percentage.sql b/sql/2024/javascript/resource-hints-prefetch-preload-modulepreload-percentage.sql index 65a5c03080d..bdd4532316f 100644 --- a/sql/2024/javascript/resource-hints-prefetch-preload-modulepreload-percentage.sql +++ b/sql/2024/javascript/resource-hints-prefetch-preload-modulepreload-percentage.sql @@ -48,8 +48,8 @@ FROM ( `httparchive.all.pages` LEFT JOIN UNNEST(getResourceHintAttrs(payload)) AS hint - WHERE - date ="2024-06-01" - ) + WHERE + date = '2024-06-01' +) GROUP BY client From c9626e3194154600aa99ec2264cbddcef540380c Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 17:57:15 +0200 Subject: [PATCH 40/53] Update resource-hints-preload-prefetch-modulepreload-distribution.sql Linter errors --- ...rce-hints-preload-prefetch-modulepreload-distribution.sql | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/sql/2024/javascript/resource-hints-preload-prefetch-modulepreload-distribution.sql b/sql/2024/javascript/resource-hints-preload-prefetch-modulepreload-distribution.sql index 05a2dd23909..6706e897b5a 100644 --- a/sql/2024/javascript/resource-hints-preload-prefetch-modulepreload-distribution.sql +++ b/sql/2024/javascript/resource-hints-preload-prefetch-modulepreload-distribution.sql @@ -48,9 +48,8 @@ FROM ( `httparchive.all.pages` LEFT JOIN UNNEST(getResourceHintAttrs(payload)) AS hint - WHERE - date ="2024-06-01" - + WHERE + date = '2024-06-01' GROUP BY client, page), From ea46dcd1e3d4761ef34f9cebef2133c03a00ba96 Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 17:58:14 +0200 Subject: [PATCH 41/53] Update resource_hints.sql Linter --- sql/2024/javascript/resource_hints.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sql/2024/javascript/resource_hints.sql b/sql/2024/javascript/resource_hints.sql index ac96b82ae36..6a61da8b9a9 100644 --- a/sql/2024/javascript/resource_hints.sql +++ b/sql/2024/javascript/resource_hints.sql @@ -44,8 +44,8 @@ FROM ( `httparchive.all.pages` LEFT JOIN UNNEST(getResourceHintAttrs(payload)) AS hint - WHERE - date = "2024-06-01" - ) + WHERE + date = '2024-06-01' +) GROUP BY client From 2bf3a75920137cffd10a3c02ae2b05071ce0c0ae Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 17:59:15 +0200 Subject: [PATCH 42/53] Update resource_hints_per_page.sql Linter --- sql/2024/javascript/resource_hints_per_page.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/2024/javascript/resource_hints_per_page.sql b/sql/2024/javascript/resource_hints_per_page.sql index e3669dc8147..5d0278bcc61 100644 --- a/sql/2024/javascript/resource_hints_per_page.sql +++ b/sql/2024/javascript/resource_hints_per_page.sql @@ -43,8 +43,8 @@ FROM ( `httparchive.all.pages` LEFT JOIN UNNEST(getResourceHintAttrs(payload)) AS hint - WHERE - date="2024-06-01" + WHERE + date = '2024-06-01' GROUP BY client, page), From 263643e3559ca34f57118192bcf4399426b7925d Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 18:00:12 +0200 Subject: [PATCH 43/53] Update sourcemaps.sql Linter --- sql/2024/javascript/sourcemaps.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/2024/javascript/sourcemaps.sql b/sql/2024/javascript/sourcemaps.sql index 9904e7d5615..5b943070263 100644 --- a/sql/2024/javascript/sourcemaps.sql +++ b/sql/2024/javascript/sourcemaps.sql @@ -37,8 +37,8 @@ FROM ( getSourceMaps(payload) AS sourcemaps FROM `httparchive.all.pages` - WHERE - date="2024-06-01" + WHERE + date = '2024-06-01' ) GROUP BY client From 194d1ea655789dc50ff0ca9cd930ba3630cafbf1 Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 18:00:59 +0200 Subject: [PATCH 44/53] Update usage_of_typescript_and_babel.sql Linter --- sql/2024/javascript/usage_of_typescript_and_babel.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/2024/javascript/usage_of_typescript_and_babel.sql b/sql/2024/javascript/usage_of_typescript_and_babel.sql index 66c7bb812c4..b6ed4dde067 100644 --- a/sql/2024/javascript/usage_of_typescript_and_babel.sql +++ b/sql/2024/javascript/usage_of_typescript_and_babel.sql @@ -40,8 +40,8 @@ FROM ( getSourceMaps(payload) AS sourcemaps FROM `httparchive.all.pages` - WHERE - date='2024-06-01' + WHERE + date = '2024-06-01' ) WHERE sourcemaps.isPublic = true From ac0d77fd96916f00209aa03c61ac6549d4e3ce5c Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 18:01:45 +0200 Subject: [PATCH 45/53] Update web_components_is_attribute.sql Linter --- sql/2024/javascript/web_components_is_attribute.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/2024/javascript/web_components_is_attribute.sql b/sql/2024/javascript/web_components_is_attribute.sql index 6bddf611361..ff1c1a20ddf 100644 --- a/sql/2024/javascript/web_components_is_attribute.sql +++ b/sql/2024/javascript/web_components_is_attribute.sql @@ -25,8 +25,8 @@ SELECT FROM `httparchive.all.pages`, UNNEST(get_almanac_attribute_info(JSON_EXTRACT_SCALAR(payload, '$._almanac'))) AS almanac_attribute_info - WHERE - date="2024-06-01" +WHERE + date = '2024-06-01' GROUP BY client ORDER BY From 09cbeac2725e6180d6849faeba771935d2b3ca8c Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 18:02:45 +0200 Subject: [PATCH 46/53] Update web_components_pct.sql Linter --- sql/2024/javascript/web_components_pct.sql | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/sql/2024/javascript/web_components_pct.sql b/sql/2024/javascript/web_components_pct.sql index 14bc7fff28a..130312ee444 100644 --- a/sql/2024/javascript/web_components_pct.sql +++ b/sql/2024/javascript/web_components_pct.sql @@ -21,7 +21,9 @@ FROM ( JSON_EXTRACT_SCALAR(payload, '$._javascript') AS js FROM `httparchive.all.pages` - WHERE - date='2024-06-01') + WHERE + date = '2024-06-01' + ) GROUP BY - client) + client +) From 42e4e44f8df0f32dca9bce9416be7b2f660c510c Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 18:04:03 +0200 Subject: [PATCH 47/53] Update web_components_specs.sql Linter --- sql/2024/javascript/web_components_specs.sql | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sql/2024/javascript/web_components_specs.sql b/sql/2024/javascript/web_components_specs.sql index 3c09424cdd5..b1ba22d5c59 100644 --- a/sql/2024/javascript/web_components_specs.sql +++ b/sql/2024/javascript/web_components_specs.sql @@ -12,8 +12,10 @@ FROM ( ARRAY_LENGTH(JSON_EXTRACT_ARRAY(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.web_component_specs.shadow_roots')) AS shadow_roots, ARRAY_LENGTH(JSON_EXTRACT_ARRAY(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.web_component_specs.template')) AS template FROM - `httparchive.all.pages` where - date='2024-06-01'), + `httparchive.all.pages` + WHERE + date = '2024-06-01' + ), UNNEST([10, 25, 50, 75, 90, 100]) AS percentile GROUP BY percentile, From 6be0c5ddcb5b38f63c194f00f332323559ec0cc5 Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 18:09:10 +0200 Subject: [PATCH 48/53] Update module_and_nomodule.sql Adding a space --- sql/2024/javascript/module_and_nomodule.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/2024/javascript/module_and_nomodule.sql b/sql/2024/javascript/module_and_nomodule.sql index 29c129a6486..d52ec8bef5d 100644 --- a/sql/2024/javascript/module_and_nomodule.sql +++ b/sql/2024/javascript/module_and_nomodule.sql @@ -21,7 +21,7 @@ FROM ( UNNEST(REGEXP_EXTRACT_ALL(response_body, r'(?i)(]*>)')) AS script WHERE date = '2024-06-01' AND - cast(json_value(summary,'$.firstHtml') AS bool) + cast(json_value(summary, '$.firstHtml') AS bool) ) GROUP BY client From 8917eb06819149e5a57f36dfd85fd2693434d54d Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 18:10:46 +0200 Subject: [PATCH 49/53] Update distribution_of_number_of_long_tasks.sql Spacing --- sql/2024/javascript/distribution_of_number_of_long_tasks.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/2024/javascript/distribution_of_number_of_long_tasks.sql b/sql/2024/javascript/distribution_of_number_of_long_tasks.sql index 11409a5431d..0e3228b34ee 100644 --- a/sql/2024/javascript/distribution_of_number_of_long_tasks.sql +++ b/sql/2024/javascript/distribution_of_number_of_long_tasks.sql @@ -27,8 +27,8 @@ WITH long_tasks_pages AS ( FROM `httparchive.all.pages`, UNNEST(getLongTasks(payload)) AS _longTasks - WHERE - date = '2024-06-01' + WHERE + date = '2024-06-01' ), long_tasks_by_page AS ( From e286b802f6531d0ab5434ee7b6dc2222e7d7b1ac Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 18:11:32 +0200 Subject: [PATCH 50/53] Update distribution_of_long_tasks_time.sql Spaces --- sql/2024/javascript/distribution_of_long_tasks_time.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/2024/javascript/distribution_of_long_tasks_time.sql b/sql/2024/javascript/distribution_of_long_tasks_time.sql index 4b4fa845ab4..11096dc3b26 100644 --- a/sql/2024/javascript/distribution_of_long_tasks_time.sql +++ b/sql/2024/javascript/distribution_of_long_tasks_time.sql @@ -27,8 +27,8 @@ WITH long_tasks_pages AS ( FROM `httparchive.all.pages`, UNNEST(getLongTasks(payload)) AS _longTasks - WHERE - date = '2024-06-01' + WHERE + date = '2024-06-01' ), long_tasks_by_page AS ( From 5c0883f73babd467a6a2e7d7bc80cff44a44ebda Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 20:20:03 +0200 Subject: [PATCH 51/53] Update frameworks_libraries_by_version.sql Fixing double spaces. --- sql/2024/javascript/frameworks_libraries_by_version.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/2024/javascript/frameworks_libraries_by_version.sql b/sql/2024/javascript/frameworks_libraries_by_version.sql index 6bf886f8724..f0738eb564a 100644 --- a/sql/2024/javascript/frameworks_libraries_by_version.sql +++ b/sql/2024/javascript/frameworks_libraries_by_version.sql @@ -21,7 +21,7 @@ JOIN ( USING (_TABLE_SUFFIX) WHERE - app IN ('jQuery', 'jQuery Migrate', 'jQuery UI', 'Modernizr', 'FancyBox', 'Slick', 'Lightbox', 'Moment.js', 'Underscore.js', 'Lodash', 'React', 'GSAP', 'Vue.js', 'styled-components', 'Emotion', 'Backbone.js', 'RequireJS', 'AngularJS', 'AMP', 'Redux', 'Next.js', 'Stimulus', 'Angular', 'Handlebars', 'Zone.js', 'Marionette.js', 'Mustache', 'Prototype', 'toastr', 'JSS', 'React Router', 'Nuxt.js', 'Alpine.js', 'Svelte', 'MooTools', 'Knockout.js', 'Socket.io', 'Gatsby', 'Adobe Client Data Layer') + app IN ('jQuery', 'jQuery Migrate', 'jQuery UI', 'Modernizr', 'FancyBox', 'Slick', 'Lightbox', 'Moment.js', 'Underscore.js', 'Lodash', 'React', 'GSAP', 'Vue.js', 'styled-components', 'Emotion', 'Backbone.js', 'RequireJS', 'AngularJS', 'AMP', 'Redux', 'Next.js', 'Stimulus', 'Angular', 'Handlebars', 'Zone.js', 'Marionette.js', 'Mustache', 'Prototype', 'toastr', 'JSS', 'React Router', 'Nuxt.js', 'Alpine.js', 'Svelte', 'MooTools', 'Knockout.js', 'Socket.io', 'Gatsby', 'Adobe Client Data Layer') GROUP BY client, category, From d5825fef5c02e27b30f54f999d7325cc24d41a4d Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 20:23:28 +0200 Subject: [PATCH 52/53] Update module_and_nomodule.sql Datatypes must be upper case. --- sql/2024/javascript/module_and_nomodule.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/2024/javascript/module_and_nomodule.sql b/sql/2024/javascript/module_and_nomodule.sql index d52ec8bef5d..60ccb4d1199 100644 --- a/sql/2024/javascript/module_and_nomodule.sql +++ b/sql/2024/javascript/module_and_nomodule.sql @@ -21,7 +21,7 @@ FROM ( UNNEST(REGEXP_EXTRACT_ALL(response_body, r'(?i)(]*>)')) AS script WHERE date = '2024-06-01' AND - cast(json_value(summary, '$.firstHtml') AS bool) + cast(json_value(summary, '$.firstHtml') AS BOOL) ) GROUP BY client From e44448fa1781b69ce35b8660b1923678c924b119 Mon Sep 17 00:00:00 2001 From: Mike Gifford Date: Mon, 26 Aug 2024 20:25:32 +0200 Subject: [PATCH 53/53] Update bytes_by_3p.sql Datatypes must be upper case. --- sql/2024/javascript/bytes_by_3p.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/2024/javascript/bytes_by_3p.sql b/sql/2024/javascript/bytes_by_3p.sql index 39d76ef6b49..311da77abd7 100644 --- a/sql/2024/javascript/bytes_by_3p.sql +++ b/sql/2024/javascript/bytes_by_3p.sql @@ -12,7 +12,7 @@ FROM ( IF(NET.HOST(url) IN ( SELECT domain FROM `httparchive.almanac.third_parties` WHERE date = '2024-06-01' AND category != 'hosting' ), 'third party', 'first party') AS host, - SUM(cast(json_value(payload, '$.response.bodySize') AS int64)) / 1024 AS kbytes + SUM(cast(json_value(payload, '$.response.bodySize') AS INT64)) / 1024 AS kbytes FROM `httparchive.all.requests`