JavaScript Queries for 2024

sql/2024/javascript/breakdown_of_pages_using_async_defer.sql

@@ -0,0 +1,23 @@
+#standardSQL
+# Breakdown of scripts using Async, Defer, Module or NoModule attributes.  Also breakdown of inline vs external scripts
+SELECT
+  _TABLE_SUFFIX AS client,
+  COUNTIF(CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.script_tags.async') AS INT64) > 0) AS async,
+  COUNTIF(CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.script_tags.defer') AS INT64) > 0) AS defer,
+  COUNTIF(CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.script_tags.async_and_defer') AS INT64) > 0) AS async_and_defer,
+  COUNTIF(CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.script_tags.type_module') AS INT64) > 0) AS module,
+  COUNTIF(CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.script_tags.nomodule') AS INT64) > 0) AS nomodule,
+  COUNTIF(CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.script_tags.async') AS INT64) > 0) / COUNT(0) AS async_pct,
+  COUNTIF(CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.script_tags.defer') AS INT64) > 0) / COUNT(0) AS defer_pct,
+  COUNTIF(CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.script_tags.async_and_defer') AS INT64) > 0) / COUNT(0) AS async_and_defer_pct,
+  COUNTIF(CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.script_tags.type_module') AS INT64) > 0) / COUNT(0) AS module_pct,
+  COUNTIF(CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.script_tags.nomodule') AS INT64) > 0) / COUNT(0) AS nomodule_pct,
+  COUNTIF(CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.script_tags.async') AS INT64) = 0 AND CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.script_tags.defer') AS INT64) = 0) AS neither,
+  COUNTIF(CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.script_tags.async') AS INT64) = 0 AND CAST(JSON_EXTRACT(JSON_EXTRACT_SCALAR(payload, '$._javascript'), '$.script_tags.defer') AS INT64) = 0) / COUNT(0) AS neither_pct
+FROM
+  `httparchive.all.pages`
+WHERE
+  date = '2024-06-01'
+
+GROUP BY
+  client

sql/2024/javascript/breakdown_of_scripts_using_async_defer_module_nomodule.sql

@@ -0,0 +1,48 @@
+#standardSQL
+# Breakdown of scripts using Async, Defer, Module or NoModule attributes.  Also breakdown of inline vs external scripts
+CREATE TEMPORARY FUNCTION getScripts(payload STRING) RETURNS STRUCT < total INT64,
+  inline INT64,
+  src INT64,
+  async INT64,
+  defer INT64,
+  async_and_defer INT64,
+  type_module INT64,
+  nomodule INT64 > LANGUAGE js AS '''
+try {
+  var $ = JSON.parse(payload);
+  var javascript = JSON.parse($._javascript);
+  return javascript.script_tags;
+} catch (e) {
+  return {};
+}
+''';
+
+SELECT
+  client,
+  SUM(script.total) AS total_scripts,
+  SUM(script.inline) AS inline_script,
+  SUM(script.src) AS external_script,
+  SUM(script.src) / SUM(script.total) AS pct_external_script,
+  SUM(script.inline) / SUM(script.total) AS pct_inline_script,
+  SUM(script.async) AS async,
+  SUM(script.defer) AS defer,
+  SUM(script.async_and_defer) AS async_and_defer,
+  SUM(script.type_module) AS module,
+  SUM(script.nomodule) AS nomodule,
+  SUM(script.async) / SUM(script.src) AS pct_external_async,
+  SUM(script.defer) / SUM(script.src) AS pct_external_defer,
+  SUM(script.async_and_defer) / SUM(script.src) AS pct_external_async_defer,
+  SUM(script.type_module) / SUM(script.src) AS pct_external_module,
+  SUM(script.nomodule) / SUM(script.src) AS pct_external_nomodule
+FROM
+  (
+    SELECT
+      client,
+      getScripts(payload) AS script
+    FROM
+      `httparchive.all.pages`
+    WHERE
+      date = '2024-06-01'
+  )
+GROUP BY
+  client

sql/2024/javascript/bytes_2019.sql

@@ -0,0 +1,15 @@
+#standardSQL
+# Sum of JS request bytes per page (2019)
+SELECT
+  percentile,
+  _TABLE_SUFFIX AS client,
+  APPROX_QUANTILES(bytesJs / 1024, 1000)[OFFSET(percentile * 10)] AS js_kilobytes
+FROM
+  `httparchive.summary_pages.2019_07_01_*`,
+  UNNEST([10, 25, 50, 75, 90, 100]) AS percentile
+GROUP BY
+  percentile,
+  client
+ORDER BY
+  client,
+  percentile

sql/2024/javascript/bytes_2020.sql

@@ -0,0 +1,15 @@
+#standardSQL
+# Sum of JS request bytes per page (2020)
+SELECT
+  percentile,
+  _TABLE_SUFFIX AS client,
+  APPROX_QUANTILES(bytesJs / 1024, 1000)[OFFSET(percentile * 10)] AS js_kilobytes
+FROM
+  `httparchive.summary_pages.2020_08_01_*`,
+  UNNEST([10, 25, 50, 75, 90, 100]) AS percentile
+GROUP BY
+  percentile,
+  client
+ORDER BY
+  client,
+  percentile

sql/2024/javascript/bytes_2021.sql

@@ -0,0 +1,15 @@
+#standardSQL
+# Sum of JS request bytes per page (2021)
+SELECT
+  percentile,
+  _TABLE_SUFFIX AS client,
+  APPROX_QUANTILES(bytesJs / 1024, 1000)[OFFSET(percentile * 10)] AS js_kilobytes
+FROM
+  `httparchive.summary_pages.2021_07_01_*`,
+  UNNEST([10, 25, 50, 75, 90, 100]) AS percentile
+GROUP BY
+  percentile,
+  client
+ORDER BY
+  client,
+  percentile

sql/2024/javascript/bytes_2022.sql

@@ -0,0 +1,15 @@
+#standardSQL
+# Sum of JS request bytes per page (2022)
+SELECT
+  percentile,
+  _TABLE_SUFFIX AS client,
+  APPROX_QUANTILES(bytesJs / 1024, 1000)[OFFSET(percentile * 10)] AS js_kilobytes
+FROM
+  `httparchive.summary_pages.2022_06_01_*`,
+  UNNEST([10, 25, 50, 75, 90, 100]) AS percentile
+GROUP BY
+  percentile,
+  client
+ORDER BY
+  client,
+  percentile

sql/2024/javascript/bytes_2023.sql

@@ -0,0 +1,15 @@
+#standardSQL
+# Sum of JS request bytes per page (2023)
+SELECT
+  percentile,
+  _TABLE_SUFFIX AS client,
+  APPROX_QUANTILES(bytesJs / 1024, 1000)[OFFSET(percentile * 10)] AS js_kilobytes
+FROM
+  `httparchive.summary_pages.2023_06_01_*`,
+  UNNEST([10, 25, 50, 75, 90, 100]) AS percentile
+GROUP BY
+  percentile,
+  client
+ORDER BY
+  client,
+  percentile

sql/2024/javascript/bytes_2024.sql

@@ -0,0 +1,15 @@
+#standardSQL
+# Sum of JS request bytes per page (2024)
+SELECT
+  percentile,
+  _TABLE_SUFFIX AS client,
+  APPROX_QUANTILES(bytesJs / 1024, 1000)[OFFSET(percentile * 10)] AS js_kilobytes
+FROM
+  `httparchive.summary_pages.2024_06_01_*`,
+  UNNEST([10, 25, 50, 75, 90, 100]) AS percentile
+GROUP BY
+  percentile,
+  client
+ORDER BY
+  client,
+  percentile

sql/2024/javascript/bytes_by_3p.sql

@@ -0,0 +1,34 @@
+#standardSQL
+# Distribution of 1P/3P JS bytes
+SELECT
+  percentile,
+  client,
+  host,
+  APPROX_QUANTILES(kbytes, 1000)[OFFSET(percentile * 10)] AS kbytes
+FROM (
+  SELECT
+    client,
+    page,
+    IF(NET.HOST(url) IN (
+      SELECT domain FROM `httparchive.almanac.third_parties` WHERE date = '2024-06-01' AND category != 'hosting'
+    ), 'third party', 'first party') AS host,
+    SUM(cast(json_value(payload, '$.response.bodySize') AS INT64)) / 1024 AS kbytes
+  FROM
+    `httparchive.all.requests`
+
+  WHERE
+    date = '2024-06-01' AND
+    type = 'script'
+  GROUP BY
+    client,
+    page,
+    host),
+  UNNEST([10, 25, 50, 75, 90, 100]) AS percentile
+GROUP BY
+  percentile,
+  client,
+  host
+ORDER BY
+  client,
+  percentile,
+  host

sql/2024/javascript/compression_method.sql

@@ -0,0 +1,47 @@
+#standardSQL
+CREATE TEMPORARY FUNCTION getHeader(headers STRING, headername STRING)
+RETURNS STRING
+DETERMINISTIC
+LANGUAGE js AS '''
+  const parsed_headers = JSON.parse(headers);
+  const matching_headers = parsed_headers.filter(h => h.name.toLowerCase() == headername.toLowerCase());
+  if (matching_headers.length > 0) {
+    return matching_headers[0].value;
+  }
+  return null;
+''';
+
+SELECT
+  client,
+  compression,
+  COUNT(DISTINCT page) AS pages,
+  ANY_VALUE(total_pages) AS total_pages,
+  COUNT(DISTINCT page) / ANY_VALUE(total_pages) AS pct_pages,
+  COUNT(0) AS js_requests,
+  SUM(COUNT(0)) OVER (PARTITION BY client) AS total_js_requests,
+  COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY client) AS pct_js_requests
+FROM (
+  SELECT
+    client,
+    page,
+    getHeader(JSON_EXTRACT(payload, '$.response.headers'), 'Content-Encoding') AS compression
+  FROM
+    `httparchive.all.requests`
+  WHERE
+    date = '2024-06-01' AND
+    type = 'script')
+JOIN (
+  SELECT
+    _TABLE_SUFFIX AS client,
+    COUNT(0) AS total_pages
+  FROM
+    `httparchive.summary_pages.2024_06_01_*`
+  GROUP BY
+    client)
+USING
+  (client)
+GROUP BY
+  client,
+  compression
+ORDER BY
+  pct_js_requests DESC

sql/2024/javascript/compression_method_by_3p.sql

@@ -0,0 +1,52 @@
+#standardSQL
+CREATE TEMPORARY FUNCTION getHeader(headers STRING, headername STRING)
+RETURNS STRING
+DETERMINISTIC
+LANGUAGE js AS '''
+  const parsed_headers = JSON.parse(headers);
+  const matching_headers = parsed_headers.filter(h => h.name.toLowerCase() == headername.toLowerCase());
+  if (matching_headers.length > 0) {
+    return matching_headers[0].value;
+  }
+  return null;
+''';
+
+SELECT
+  client,
+  host,
+  compression,
+  COUNT(DISTINCT page) AS pages,
+  ANY_VALUE(total_pages) AS total_pages,
+  COUNT(DISTINCT page) / ANY_VALUE(total_pages) AS pct_pages,
+  COUNT(0) AS js_requests,
+  SUM(COUNT(0)) OVER (PARTITION BY client, host) AS total,
+  COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY client, host) AS pct
+FROM (
+  SELECT
+    client,
+    page,
+    IF(NET.HOST(url) IN (
+      SELECT domain FROM `httparchive.almanac.third_parties` WHERE date = '2024-06-01' AND category != 'hosting'
+    ), 'third party', 'first party') AS host,
+    getHeader(JSON_EXTRACT(payload, '$.response.headers'), 'Content-Encoding') AS compression
+  FROM
+    `httparchive.all.requests`
+  WHERE
+    date = '2024-06-01' AND
+    type = 'script')
+JOIN (
+  SELECT
+    _TABLE_SUFFIX AS client,
+    COUNT(0) AS total_pages
+  FROM
+    `httparchive.summary_pages.2024_06_01_*`
+  GROUP BY
+    client)
+USING
+  (client)
+GROUP BY
+  client,
+  host,
+  compression
+ORDER BY
+  pct DESC

sql/2024/javascript/compression_none_by_bytes.sql

@@ -0,0 +1,55 @@
+#standardSQL
+CREATE TEMPORARY FUNCTION getHeader(headers STRING, headername STRING)
+RETURNS STRING
+DETERMINISTIC
+LANGUAGE js AS '''
+  const parsed_headers = JSON.parse(headers);
+  const matching_headers = parsed_headers.filter(h => h.name.toLowerCase() == headername.toLowerCase());
+  if (matching_headers.length > 0) {
+    return matching_headers[0].value;
+  }
+  return null;
+''';
+
+SELECT
+  client,
+  host,
+  IF(kbytes < 100, FLOOR(kbytes / 5) * 5, 100) AS kbytes,
+  COUNT(DISTINCT page) AS pages,
+  ANY_VALUE(total_pages) AS total_pages,
+  COUNT(DISTINCT page) / ANY_VALUE(total_pages) AS pct_pages,
+  COUNT(0) AS js_requests,
+  SUM(COUNT(0)) OVER (PARTITION BY client, host) AS total,
+  COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY client, host) AS pct
+FROM (
+  SELECT
+    client,
+    page,
+    IF(NET.HOST(url) IN (
+      SELECT domain FROM `httparchive.almanac.third_parties` WHERE date = '2024-06-01' AND category != 'hosting'
+    ), 'third party', 'first party') AS host,
+    respSize / 1024 AS kbytes
+  FROM
+    `httparchive.all.requests`
+  WHERE
+    date = '2024-06-01' AND
+    type = 'script' AND
+    getHeader(JSON_EXTRACT(payload, '$.response.headers'), 'Content-Encoding') IS NULL)
+JOIN (
+  SELECT
+    _TABLE_SUFFIX AS client,
+    COUNT(0) AS total_pages
+  FROM
+    `httparchive.summary_pages.2024_06_01_*`
+  GROUP BY
+    client)
+USING
+  (client)
+GROUP BY
+  client,
+  host,
+  kbytes
+ORDER BY
+  client,
+  host,
+  kbytes