Skip to content

feat: add epoch, daily, weekly, and monthly agg tables for block_first_seen_by_node #119

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
samcm wants to merge 6 commits into
base: master
Choose a base branch
from
Open

feat: add epoch, daily, weekly, and monthly agg tables for block_first_seen_by_node #119

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
bf78cbf
feat(transformations): add epoch, daily, weekly, and monthly aggregat…
samcm Nov 6, 2025
96c650f
yeet
samcm Nov 6, 2025
1a1f478
yeet
samcm Nov 7, 2025
7e80dc3
Merge branch 'master' into block-seen-by-node-aggs
samcm Nov 13, 2025
364734a
proto
samcm Nov 14, 2025
c7cdd66
Merge branch 'block-seen-by-node-aggs' of github.com:ethpandaops/xatu…
samcm Nov 14, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions migrations/033_fct_block_first_seen_by_node_daily.down.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
DROP TABLE IF EXISTS `${NETWORK_NAME}`.fct_block_first_seen_by_node_daily ON CLUSTER '{cluster}';
DROP TABLE IF EXISTS `${NETWORK_NAME}`.fct_block_first_seen_by_node_daily_local ON CLUSTER '{cluster}';
55 changes: 55 additions & 0 deletions migrations/033_fct_block_first_seen_by_node_daily.up.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
CREATE TABLE `${NETWORK_NAME}`.fct_block_first_seen_by_node_daily_local on cluster '{cluster}' (
`updated_date_time` DateTime COMMENT 'Timestamp when the record was last updated' CODEC(DoubleDelta, ZSTD(1)),
`day` Date COMMENT 'Calendar day (UTC)' CODEC(DoubleDelta, ZSTD(1)),
`day_start_date_time` DateTime COMMENT 'Start of day at midnight UTC' CODEC(DoubleDelta, ZSTD(1)),
`username` LowCardinality(String) COMMENT 'Username of the node' CODEC(ZSTD(1)),
`node_id` String COMMENT 'ID of the node' CODEC(ZSTD(1)),
`classification` LowCardinality(String) COMMENT 'Classification of the node, e.g. "individual", "corporate", "internal" (aka ethPandaOps) or "unclassified"' CODEC(ZSTD(1)),
`meta_client_name` LowCardinality(String) COMMENT 'Name of the client',
`meta_client_implementation` LowCardinality(String) COMMENT 'Implementation of the client',
`meta_client_geo_city` LowCardinality(String) COMMENT 'City of the client' CODEC(ZSTD(1)),
`meta_client_geo_country` LowCardinality(String) COMMENT 'Country of the client' CODEC(ZSTD(1)),
`meta_client_geo_country_code` LowCardinality(String) COMMENT 'Country code of the client' CODEC(ZSTD(1)),
`meta_client_geo_continent_code` LowCardinality(String) COMMENT 'Continent code of the client' CODEC(ZSTD(1)),
`meta_client_geo_longitude` Nullable(Float64) COMMENT 'Longitude of the client' CODEC(ZSTD(1)),
`meta_client_geo_latitude` Nullable(Float64) COMMENT 'Latitude of the client' CODEC(ZSTD(1)),
`meta_client_geo_autonomous_system_number` Nullable(UInt32) COMMENT 'Autonomous system number of the client' CODEC(ZSTD(1)),
`meta_client_geo_autonomous_system_organization` Nullable(String) COMMENT 'Autonomous system organization of the client' CODEC(ZSTD(1)),
`min_slot` UInt64 COMMENT 'Minimum slot number in this day' CODEC(ZSTD(1)),
`max_slot` UInt64 COMMENT 'Maximum slot number in this day' CODEC(ZSTD(1)),
`epoch_count` UInt32 COMMENT 'Number of epochs in this day' CODEC(ZSTD(1)),
`slot_count` UInt32 COMMENT 'Number of slots with blocks seen in this day' CODEC(ZSTD(1)),
`min_seen_slot_start_diff_ms` UInt32 COMMENT 'Minimum time from slot start for the node to see a block (milliseconds)' CODEC(ZSTD(1)),
`p05_seen_slot_start_diff_ms` UInt32 COMMENT '5th percentile time from slot start (milliseconds)' CODEC(ZSTD(1)),
`p50_seen_slot_start_diff_ms` UInt32 COMMENT 'Median (p50) time from slot start (milliseconds)' CODEC(ZSTD(1)),
`avg_seen_slot_start_diff_ms` UInt32 COMMENT 'Average time from slot start (milliseconds)' CODEC(ZSTD(1)),
`p90_seen_slot_start_diff_ms` UInt32 COMMENT '90th percentile time from slot start (milliseconds)' CODEC(ZSTD(1)),
`p95_seen_slot_start_diff_ms` UInt32 COMMENT '95th percentile time from slot start (milliseconds)' CODEC(ZSTD(1)),
`p99_seen_slot_start_diff_ms` UInt32 COMMENT '99th percentile time from slot start (milliseconds)' CODEC(ZSTD(1)),
`max_seen_slot_start_diff_ms` UInt32 COMMENT 'Maximum time from slot start (milliseconds)' CODEC(ZSTD(1))
) ENGINE = ReplicatedReplacingMergeTree(
'/clickhouse/{installation}/{cluster}/tables/{shard}/{database}/{table}',
'{replica}',
`updated_date_time`
) PARTITION BY toYYYYMM(day)
ORDER BY
(`day`, `username`, `node_id`, `classification`, `meta_client_name`)
SETTINGS
deduplicate_merge_projection_mode = 'rebuild',
min_age_to_force_merge_seconds = 384,
min_age_to_force_merge_on_partition_only=false
COMMENT 'Block first seen statistics by node aggregated by day';

CREATE TABLE `${NETWORK_NAME}`.fct_block_first_seen_by_node_daily ON CLUSTER '{cluster}' AS `${NETWORK_NAME}`.fct_block_first_seen_by_node_daily_local ENGINE = Distributed(
'{cluster}',
'${NETWORK_NAME}',
fct_block_first_seen_by_node_daily_local,
cityHash64(`day`, `username`, `node_id`, `classification`, `meta_client_name`)
);

ALTER TABLE `${NETWORK_NAME}`.fct_block_first_seen_by_node_daily_local ON CLUSTER '{cluster}'
ADD PROJECTION p_by_day_node
(
SELECT *
ORDER BY (`day`, `username`, `node_id`, `classification`, `meta_client_name`)
);
2 changes: 2 additions & 0 deletions migrations/034_fct_block_first_seen_by_node_weekly.down.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
DROP TABLE IF EXISTS `${NETWORK_NAME}`.fct_block_first_seen_by_node_weekly ON CLUSTER '{cluster}';
DROP TABLE IF EXISTS `${NETWORK_NAME}`.fct_block_first_seen_by_node_weekly_local ON CLUSTER '{cluster}';
56 changes: 56 additions & 0 deletions migrations/034_fct_block_first_seen_by_node_weekly.up.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
CREATE TABLE `${NETWORK_NAME}`.fct_block_first_seen_by_node_weekly_local on cluster '{cluster}' (
`updated_date_time` DateTime COMMENT 'Timestamp when the record was last updated' CODEC(DoubleDelta, ZSTD(1)),
`week` Date COMMENT 'Start of week (Monday)' CODEC(DoubleDelta, ZSTD(1)),
`week_start_date_time` DateTime COMMENT 'Start of week at midnight UTC (Monday)' CODEC(DoubleDelta, ZSTD(1)),
`username` LowCardinality(String) COMMENT 'Username of the node' CODEC(ZSTD(1)),
`node_id` String COMMENT 'ID of the node' CODEC(ZSTD(1)),
`classification` LowCardinality(String) COMMENT 'Classification of the node, e.g. "individual", "corporate", "internal" (aka ethPandaOps) or "unclassified"' CODEC(ZSTD(1)),
`meta_client_name` LowCardinality(String) COMMENT 'Name of the client',
`meta_client_implementation` LowCardinality(String) COMMENT 'Implementation of the client',
`meta_client_geo_city` LowCardinality(String) COMMENT 'City of the client' CODEC(ZSTD(1)),
`meta_client_geo_country` LowCardinality(String) COMMENT 'Country of the client' CODEC(ZSTD(1)),
`meta_client_geo_country_code` LowCardinality(String) COMMENT 'Country code of the client' CODEC(ZSTD(1)),
`meta_client_geo_continent_code` LowCardinality(String) COMMENT 'Continent code of the client' CODEC(ZSTD(1)),
`meta_client_geo_longitude` Nullable(Float64) COMMENT 'Longitude of the client' CODEC(ZSTD(1)),
`meta_client_geo_latitude` Nullable(Float64) COMMENT 'Latitude of the client' CODEC(ZSTD(1)),
`meta_client_geo_autonomous_system_number` Nullable(UInt32) COMMENT 'Autonomous system number of the client' CODEC(ZSTD(1)),
`meta_client_geo_autonomous_system_organization` Nullable(String) COMMENT 'Autonomous system organization of the client' CODEC(ZSTD(1)),
`min_slot` UInt64 COMMENT 'Minimum slot number in this week' CODEC(ZSTD(1)),
`max_slot` UInt64 COMMENT 'Maximum slot number in this week' CODEC(ZSTD(1)),
`day_count` UInt32 COMMENT 'Number of days in this week' CODEC(ZSTD(1)),
`epoch_count` UInt32 COMMENT 'Number of epochs in this week' CODEC(ZSTD(1)),
`slot_count` UInt32 COMMENT 'Number of slots with blocks seen in this week' CODEC(ZSTD(1)),
`min_seen_slot_start_diff_ms` UInt32 COMMENT 'Minimum time from slot start for the node to see a block (milliseconds)' CODEC(ZSTD(1)),
`p05_seen_slot_start_diff_ms` UInt32 COMMENT '5th percentile time from slot start (milliseconds)' CODEC(ZSTD(1)),
`p50_seen_slot_start_diff_ms` UInt32 COMMENT 'Median (p50) time from slot start (milliseconds)' CODEC(ZSTD(1)),
`avg_seen_slot_start_diff_ms` UInt32 COMMENT 'Average time from slot start (milliseconds)' CODEC(ZSTD(1)),
`p90_seen_slot_start_diff_ms` UInt32 COMMENT '90th percentile time from slot start (milliseconds)' CODEC(ZSTD(1)),
`p95_seen_slot_start_diff_ms` UInt32 COMMENT '95th percentile time from slot start (milliseconds)' CODEC(ZSTD(1)),
`p99_seen_slot_start_diff_ms` UInt32 COMMENT '99th percentile time from slot start (milliseconds)' CODEC(ZSTD(1)),
`max_seen_slot_start_diff_ms` UInt32 COMMENT 'Maximum time from slot start (milliseconds)' CODEC(ZSTD(1))
) ENGINE = ReplicatedReplacingMergeTree(
'/clickhouse/{installation}/{cluster}/tables/{shard}/{database}/{table}',
'{replica}',
`updated_date_time`
) PARTITION BY toYYYYMM(week)
ORDER BY
(`week`, `username`, `node_id`, `classification`, `meta_client_name`)
SETTINGS
deduplicate_merge_projection_mode = 'rebuild',
min_age_to_force_merge_seconds = 384,
min_age_to_force_merge_on_partition_only=false
COMMENT 'Block first seen statistics by node aggregated by week';

CREATE TABLE `${NETWORK_NAME}`.fct_block_first_seen_by_node_weekly ON CLUSTER '{cluster}' AS `${NETWORK_NAME}`.fct_block_first_seen_by_node_weekly_local ENGINE = Distributed(
'{cluster}',
'${NETWORK_NAME}',
fct_block_first_seen_by_node_weekly_local,
cityHash64(`week`, `username`, `node_id`, `classification`, `meta_client_name`)
);

ALTER TABLE `${NETWORK_NAME}`.fct_block_first_seen_by_node_weekly_local ON CLUSTER '{cluster}'
ADD PROJECTION p_by_week_node
(
SELECT *
ORDER BY (`week`, `username`, `node_id`, `classification`, `meta_client_name`)
);
2 changes: 2 additions & 0 deletions migrations/035_fct_block_first_seen_by_node_monthly.down.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
DROP TABLE IF EXISTS `${NETWORK_NAME}`.fct_block_first_seen_by_node_monthly ON CLUSTER '{cluster}';
DROP TABLE IF EXISTS `${NETWORK_NAME}`.fct_block_first_seen_by_node_monthly_local ON CLUSTER '{cluster}';
56 changes: 56 additions & 0 deletions migrations/035_fct_block_first_seen_by_node_monthly.up.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
CREATE TABLE `${NETWORK_NAME}`.fct_block_first_seen_by_node_monthly_local on cluster '{cluster}' (
`updated_date_time` DateTime COMMENT 'Timestamp when the record was last updated' CODEC(DoubleDelta, ZSTD(1)),
`month` UInt32 COMMENT 'Year and month in YYYYMM format (e.g., 202507)' CODEC(ZSTD(1)),
`month_start_date_time` DateTime COMMENT 'Start of month at midnight UTC' CODEC(DoubleDelta, ZSTD(1)),
`username` LowCardinality(String) COMMENT 'Username of the node' CODEC(ZSTD(1)),
`node_id` String COMMENT 'ID of the node' CODEC(ZSTD(1)),
`classification` LowCardinality(String) COMMENT 'Classification of the node, e.g. "individual", "corporate", "internal" (aka ethPandaOps) or "unclassified"' CODEC(ZSTD(1)),
`meta_client_name` LowCardinality(String) COMMENT 'Name of the client',
`meta_client_implementation` LowCardinality(String) COMMENT 'Implementation of the client',
`meta_client_geo_city` LowCardinality(String) COMMENT 'City of the client' CODEC(ZSTD(1)),
`meta_client_geo_country` LowCardinality(String) COMMENT 'Country of the client' CODEC(ZSTD(1)),
`meta_client_geo_country_code` LowCardinality(String) COMMENT 'Country code of the client' CODEC(ZSTD(1)),
`meta_client_geo_continent_code` LowCardinality(String) COMMENT 'Continent code of the client' CODEC(ZSTD(1)),
`meta_client_geo_longitude` Nullable(Float64) COMMENT 'Longitude of the client' CODEC(ZSTD(1)),
`meta_client_geo_latitude` Nullable(Float64) COMMENT 'Latitude of the client' CODEC(ZSTD(1)),
`meta_client_geo_autonomous_system_number` Nullable(UInt32) COMMENT 'Autonomous system number of the client' CODEC(ZSTD(1)),
`meta_client_geo_autonomous_system_organization` Nullable(String) COMMENT 'Autonomous system organization of the client' CODEC(ZSTD(1)),
`min_slot` UInt64 COMMENT 'Minimum slot number in this month' CODEC(ZSTD(1)),
`max_slot` UInt64 COMMENT 'Maximum slot number in this month' CODEC(ZSTD(1)),
`day_count` UInt32 COMMENT 'Number of days in this month' CODEC(ZSTD(1)),
`epoch_count` UInt32 COMMENT 'Number of epochs in this month' CODEC(ZSTD(1)),
`slot_count` UInt32 COMMENT 'Number of slots with blocks seen in this month' CODEC(ZSTD(1)),
`min_seen_slot_start_diff_ms` UInt32 COMMENT 'Minimum time from slot start for the node to see a block (milliseconds)' CODEC(ZSTD(1)),
`p05_seen_slot_start_diff_ms` UInt32 COMMENT '5th percentile time from slot start (milliseconds)' CODEC(ZSTD(1)),
`p50_seen_slot_start_diff_ms` UInt32 COMMENT 'Median (p50) time from slot start (milliseconds)' CODEC(ZSTD(1)),
`avg_seen_slot_start_diff_ms` UInt32 COMMENT 'Average time from slot start (milliseconds)' CODEC(ZSTD(1)),
`p90_seen_slot_start_diff_ms` UInt32 COMMENT '90th percentile time from slot start (milliseconds)' CODEC(ZSTD(1)),
`p95_seen_slot_start_diff_ms` UInt32 COMMENT '95th percentile time from slot start (milliseconds)' CODEC(ZSTD(1)),
`p99_seen_slot_start_diff_ms` UInt32 COMMENT '99th percentile time from slot start (milliseconds)' CODEC(ZSTD(1)),
`max_seen_slot_start_diff_ms` UInt32 COMMENT 'Maximum time from slot start (milliseconds)' CODEC(ZSTD(1))
) ENGINE = ReplicatedReplacingMergeTree(
'/clickhouse/{installation}/{cluster}/tables/{shard}/{database}/{table}',
'{replica}',
`updated_date_time`
) PARTITION BY month
ORDER BY
(`month`, `username`, `node_id`, `classification`, `meta_client_name`)
SETTINGS
deduplicate_merge_projection_mode = 'rebuild',
min_age_to_force_merge_seconds = 384,
min_age_to_force_merge_on_partition_only=false
COMMENT 'Block first seen statistics by node aggregated by month';

CREATE TABLE `${NETWORK_NAME}`.fct_block_first_seen_by_node_monthly ON CLUSTER '{cluster}' AS `${NETWORK_NAME}`.fct_block_first_seen_by_node_monthly_local ENGINE = Distributed(
'{cluster}',
'${NETWORK_NAME}',
fct_block_first_seen_by_node_monthly_local,
cityHash64(`month`, `username`, `node_id`, `classification`, `meta_client_name`)
);

ALTER TABLE `${NETWORK_NAME}`.fct_block_first_seen_by_node_monthly_local ON CLUSTER '{cluster}'
ADD PROJECTION p_by_month_node
(
SELECT *
ORDER BY (`month`, `username`, `node_id`, `classification`, `meta_client_name`)
);
2 changes: 2 additions & 0 deletions migrations/036_fct_block_first_seen_by_node_by_epoch.down.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
DROP TABLE IF EXISTS `${NETWORK_NAME}`.fct_block_first_seen_by_node_by_epoch ON CLUSTER '{cluster}';
DROP TABLE IF EXISTS `${NETWORK_NAME}`.fct_block_first_seen_by_node_by_epoch_local ON CLUSTER '{cluster}';
54 changes: 54 additions & 0 deletions migrations/036_fct_block_first_seen_by_node_by_epoch.up.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
CREATE TABLE `${NETWORK_NAME}`.fct_block_first_seen_by_node_by_epoch_local on cluster '{cluster}' (
`updated_date_time` DateTime COMMENT 'Timestamp when the record was last updated' CODEC(DoubleDelta, ZSTD(1)),
`epoch` UInt32 COMMENT 'Epoch number' CODEC(DoubleDelta, ZSTD(1)),
`epoch_start_date_time` DateTime COMMENT 'The wall clock time when the epoch started' CODEC(DoubleDelta, ZSTD(1)),
`username` LowCardinality(String) COMMENT 'Username of the node' CODEC(ZSTD(1)),
`node_id` String COMMENT 'ID of the node' CODEC(ZSTD(1)),
`classification` LowCardinality(String) COMMENT 'Classification of the node, e.g. "individual", "corporate", "internal" (aka ethPandaOps) or "unclassified"' CODEC(ZSTD(1)),
`meta_client_name` LowCardinality(String) COMMENT 'Name of the client',
`meta_client_implementation` LowCardinality(String) COMMENT 'Implementation of the client',
`meta_client_geo_city` LowCardinality(String) COMMENT 'City of the client' CODEC(ZSTD(1)),
`meta_client_geo_country` LowCardinality(String) COMMENT 'Country of the client' CODEC(ZSTD(1)),
`meta_client_geo_country_code` LowCardinality(String) COMMENT 'Country code of the client' CODEC(ZSTD(1)),
`meta_client_geo_continent_code` LowCardinality(String) COMMENT 'Continent code of the client' CODEC(ZSTD(1)),
`meta_client_geo_longitude` Nullable(Float64) COMMENT 'Longitude of the client' CODEC(ZSTD(1)),
`meta_client_geo_latitude` Nullable(Float64) COMMENT 'Latitude of the client' CODEC(ZSTD(1)),
`meta_client_geo_autonomous_system_number` Nullable(UInt32) COMMENT 'Autonomous system number of the client' CODEC(ZSTD(1)),
`meta_client_geo_autonomous_system_organization` Nullable(String) COMMENT 'Autonomous system organization of the client' CODEC(ZSTD(1)),
`min_slot` UInt64 COMMENT 'Minimum slot number in this epoch' CODEC(ZSTD(1)),
`max_slot` UInt64 COMMENT 'Maximum slot number in this epoch' CODEC(ZSTD(1)),
`slot_count` UInt32 COMMENT 'Number of slots with blocks seen in this epoch' CODEC(ZSTD(1)),
`min_seen_slot_start_diff_ms` UInt32 COMMENT 'Minimum time from slot start for the node to see a block (milliseconds)' CODEC(ZSTD(1)),
`p05_seen_slot_start_diff_ms` UInt32 COMMENT '5th percentile time from slot start (milliseconds)' CODEC(ZSTD(1)),
`p50_seen_slot_start_diff_ms` UInt32 COMMENT 'Median (p50) time from slot start (milliseconds)' CODEC(ZSTD(1)),
`avg_seen_slot_start_diff_ms` UInt32 COMMENT 'Average time from slot start (milliseconds)' CODEC(ZSTD(1)),
`p90_seen_slot_start_diff_ms` UInt32 COMMENT '90th percentile time from slot start (milliseconds)' CODEC(ZSTD(1)),
`p95_seen_slot_start_diff_ms` UInt32 COMMENT '95th percentile time from slot start (milliseconds)' CODEC(ZSTD(1)),
`p99_seen_slot_start_diff_ms` UInt32 COMMENT '99th percentile time from slot start (milliseconds)' CODEC(ZSTD(1)),
`max_seen_slot_start_diff_ms` UInt32 COMMENT 'Maximum time from slot start (milliseconds)' CODEC(ZSTD(1))
) ENGINE = ReplicatedReplacingMergeTree(
'/clickhouse/{installation}/{cluster}/tables/{shard}/{database}/{table}',
'{replica}',
`updated_date_time`
) PARTITION BY toStartOfMonth(epoch_start_date_time)
ORDER BY
(`epoch_start_date_time`, `username`, `node_id`, `classification`, `meta_client_name`)
SETTINGS
deduplicate_merge_projection_mode = 'rebuild',
min_age_to_force_merge_seconds = 384,
min_age_to_force_merge_on_partition_only=false
COMMENT 'Block first seen statistics by node aggregated by epoch';

CREATE TABLE `${NETWORK_NAME}`.fct_block_first_seen_by_node_by_epoch ON CLUSTER '{cluster}' AS `${NETWORK_NAME}`.fct_block_first_seen_by_node_by_epoch_local ENGINE = Distributed(
'{cluster}',
'${NETWORK_NAME}',
fct_block_first_seen_by_node_by_epoch_local,
cityHash64(`epoch_start_date_time`, `username`, `node_id`, `classification`, `meta_client_name`)
);

ALTER TABLE `${NETWORK_NAME}`.fct_block_first_seen_by_node_by_epoch_local ON CLUSTER '{cluster}'
ADD PROJECTION p_by_epoch_node
(
SELECT *
ORDER BY (`epoch`, `username`, `node_id`, `classification`, `meta_client_name`)
);
Loading
Loading