From 7bdb29a32da1a6038d95253648dbc409827cd3ef Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Fri, 14 Mar 2025 17:51:13 -0700 Subject: [PATCH 1/7] fix lint --- .../schema.sql | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 clickhouse_db_schema/oss_ci_job_queue_time_historical/schema.sql diff --git a/clickhouse_db_schema/oss_ci_job_queue_time_historical/schema.sql b/clickhouse_db_schema/oss_ci_job_queue_time_historical/schema.sql new file mode 100644 index 0000000000..f58270e760 --- /dev/null +++ b/clickhouse_db_schema/oss_ci_job_queue_time_historical/schema.sql @@ -0,0 +1,26 @@ +CREATE TABLE misc.oss_ci_job_queue_time_historical( + `time` DateTime64(9), + `type` String, + `repo` String DEFAULT 'pytorch/pytorch', + `workflow_name` String, + `job_name` String, + `machine_type` String, + `histogram_version` String, + `histogram` Array(UInt64), + `max_queue_time` UInt64, + `total_count` UInt64, + `extra_info` Map(String,String) +) +ENGINE = SharedMergeTree('/clickhouse/tables/{uuid}/{shard}', '{replica}') +PARTITION BY toYYYYMM(created_at) +ORDER BY ( + job_name, + workflow_name, + machine_type, + job_name, + time, + repo, + type, +) +TTL toDate(time) + toIntervalYear(5) +SETTINGS index_granularity = 8192 From 38618c591d10cae5f727623aa177eeb0a431bc2f Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Fri, 14 Mar 2025 18:35:39 -0700 Subject: [PATCH 2/7] add time --- .../schema.sql | 22 ++++++++----------- .../oss_ci_queue_time_histogram/schema.sql | 15 ++++++------- 2 files changed, 16 insertions(+), 21 deletions(-) diff --git a/clickhouse_db_schema/oss_ci_job_queue_time_historical/schema.sql b/clickhouse_db_schema/oss_ci_job_queue_time_historical/schema.sql index f58270e760..f65477e9d2 100644 --- a/clickhouse_db_schema/oss_ci_job_queue_time_historical/schema.sql +++ b/clickhouse_db_schema/oss_ci_job_queue_time_historical/schema.sql @@ -1,26 +1,22 @@ CREATE TABLE misc.oss_ci_job_queue_time_historical( - `time` DateTime64(9), - `type` String, + `queue_s` UInt64, `repo` String DEFAULT 'pytorch/pytorch', `workflow_name` String, `job_name` String, - `machine_type` String, - `histogram_version` String, - `histogram` Array(UInt64), - `max_queue_time` UInt64, - `total_count` UInt64, - `extra_info` Map(String,String) + `machine_type` String + `time` DateTime64(9), + -- The raw records on S3, this is populated by the s3 replicator + `_meta` Tuple(bucket String, key String) ) ENGINE = SharedMergeTree('/clickhouse/tables/{uuid}/{shard}', '{replica}') -PARTITION BY toYYYYMM(created_at) +PARTITION BY toYYYYMM(time) ORDER BY ( - job_name, + repo, workflow_name, - machine_type, job_name, + machine_type, + queue_s, time, - repo, - type, ) TTL toDate(time) + toIntervalYear(5) SETTINGS index_granularity = 8192 diff --git a/clickhouse_db_schema/oss_ci_queue_time_histogram/schema.sql b/clickhouse_db_schema/oss_ci_queue_time_histogram/schema.sql index 5532197b0e..680d683cc7 100644 --- a/clickhouse_db_schema/oss_ci_queue_time_histogram/schema.sql +++ b/clickhouse_db_schema/oss_ci_queue_time_histogram/schema.sql @@ -1,6 +1,4 @@ CREATE TABLE misc.oss_ci_queue_time_histogram( - `created_at` DateTime64(0, 'UTC'), - `time_stamp` DateTime64(0, 'UTC'), `type` String, `repo` String DEFAULT 'pytorch/pytorch', `workflow_name` String, @@ -9,19 +7,20 @@ CREATE TABLE misc.oss_ci_queue_time_histogram( `histogram_version` String, `histogram` Array(UInt64), `max_queue_time` UInt64, + `sum_queue_time` UInt64, `total_count` UInt64, + `time` DateTime64(9), `extra_info` Map(String,String) ) ENGINE = SharedMergeTree('/clickhouse/tables/{uuid}/{shard}', '{replica}') -PARTITION BY toYYYYMM(created_at) +PARTITION BY toYYYYMM(time) ORDER BY ( - job_name, + type, workflow_name, - machine_type, job_name, - time_stamp, + machine_type, repo, - type, + time, ) -TTL toDate(time_stamp) + toIntervalYear(5) +TTL toDate(time) + toIntervalYear(5) SETTINGS index_granularity = 8192 From 9f0d8ad715a7deed8e6ce638c39f48a621d1c31d Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Mon, 17 Mar 2025 21:02:40 -0700 Subject: [PATCH 3/7] add html_url --- .../oss_ci_job_queue_time_historical/schema.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/clickhouse_db_schema/oss_ci_job_queue_time_historical/schema.sql b/clickhouse_db_schema/oss_ci_job_queue_time_historical/schema.sql index f65477e9d2..dc104e2920 100644 --- a/clickhouse_db_schema/oss_ci_job_queue_time_historical/schema.sql +++ b/clickhouse_db_schema/oss_ci_job_queue_time_historical/schema.sql @@ -3,7 +3,8 @@ CREATE TABLE misc.oss_ci_job_queue_time_historical( `repo` String DEFAULT 'pytorch/pytorch', `workflow_name` String, `job_name` String, - `machine_type` String + `html_url` String, + `machine_type` String, `time` DateTime64(9), -- The raw records on S3, this is populated by the s3 replicator `_meta` Tuple(bucket String, key String) From 0e7d2836cff2f5812751511f7d3abc1591419919 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Mon, 17 Mar 2025 23:26:33 -0700 Subject: [PATCH 4/7] fix test version --- .../oss_ci_job_queue_time_historical/schema.sql | 7 +++---- .../oss_ci_queue_time_histogram/schema.sql | 6 +++--- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/clickhouse_db_schema/oss_ci_job_queue_time_historical/schema.sql b/clickhouse_db_schema/oss_ci_job_queue_time_historical/schema.sql index dc104e2920..e3861adb22 100644 --- a/clickhouse_db_schema/oss_ci_job_queue_time_historical/schema.sql +++ b/clickhouse_db_schema/oss_ci_job_queue_time_historical/schema.sql @@ -13,11 +13,10 @@ ENGINE = SharedMergeTree('/clickhouse/tables/{uuid}/{shard}', '{replica}') PARTITION BY toYYYYMM(time) ORDER BY ( repo, - workflow_name, - job_name, - machine_type, - queue_s, time, + machine_type, + job_name, + workflow_name, ) TTL toDate(time) + toIntervalYear(5) SETTINGS index_granularity = 8192 diff --git a/clickhouse_db_schema/oss_ci_queue_time_histogram/schema.sql b/clickhouse_db_schema/oss_ci_queue_time_histogram/schema.sql index 680d683cc7..19135e25af 100644 --- a/clickhouse_db_schema/oss_ci_queue_time_histogram/schema.sql +++ b/clickhouse_db_schema/oss_ci_queue_time_histogram/schema.sql @@ -16,11 +16,11 @@ ENGINE = SharedMergeTree('/clickhouse/tables/{uuid}/{shard}', '{replica}') PARTITION BY toYYYYMM(time) ORDER BY ( type, - workflow_name, - job_name, - machine_type, repo, time, + machine_type, + job_name, + workflow_name, ) TTL toDate(time) + toIntervalYear(5) SETTINGS index_granularity = 8192 From 465590e147e55fbcecb78d48ac18e90f2a7dd86d Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Mon, 17 Mar 2025 23:46:52 -0700 Subject: [PATCH 5/7] fix test version --- .../oss_ci_job_queue_time_historical/schema.sql | 1 + clickhouse_db_schema/oss_ci_queue_time_histogram/schema.sql | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/clickhouse_db_schema/oss_ci_job_queue_time_historical/schema.sql b/clickhouse_db_schema/oss_ci_job_queue_time_historical/schema.sql index e3861adb22..76301b9e95 100644 --- a/clickhouse_db_schema/oss_ci_job_queue_time_historical/schema.sql +++ b/clickhouse_db_schema/oss_ci_job_queue_time_historical/schema.sql @@ -1,3 +1,4 @@ + -- This table is used to keep track of snapshots of in-queue jobs CREATE TABLE misc.oss_ci_job_queue_time_historical( `queue_s` UInt64, `repo` String DEFAULT 'pytorch/pytorch', diff --git a/clickhouse_db_schema/oss_ci_queue_time_histogram/schema.sql b/clickhouse_db_schema/oss_ci_queue_time_histogram/schema.sql index 19135e25af..68b6fc382c 100644 --- a/clickhouse_db_schema/oss_ci_queue_time_histogram/schema.sql +++ b/clickhouse_db_schema/oss_ci_queue_time_histogram/schema.sql @@ -1,4 +1,7 @@ + -- This table is used to store queue time histogram CREATE TABLE misc.oss_ci_queue_time_histogram( + -- the type of histogram, currently we store two types of histogram: + -- 'in-queue-histogram','completed-queue-histogram' `type` String, `repo` String DEFAULT 'pytorch/pytorch', `workflow_name` String, @@ -7,7 +10,7 @@ CREATE TABLE misc.oss_ci_queue_time_histogram( `histogram_version` String, `histogram` Array(UInt64), `max_queue_time` UInt64, - `sum_queue_time` UInt64, + `avg_queue_time` UInt64, `total_count` UInt64, `time` DateTime64(9), `extra_info` Map(String,String) From 14b7844a081a70c26d0826bc418433e2ab14fdad Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Wed, 19 Mar 2025 00:15:44 -0700 Subject: [PATCH 6/7] add tags --- clickhouse_db_schema/oss_ci_job_queue_time_historical/schema.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/clickhouse_db_schema/oss_ci_job_queue_time_historical/schema.sql b/clickhouse_db_schema/oss_ci_job_queue_time_historical/schema.sql index 76301b9e95..c74d0aef75 100644 --- a/clickhouse_db_schema/oss_ci_job_queue_time_historical/schema.sql +++ b/clickhouse_db_schema/oss_ci_job_queue_time_historical/schema.sql @@ -7,6 +7,7 @@ CREATE TABLE misc.oss_ci_job_queue_time_historical( `html_url` String, `machine_type` String, `time` DateTime64(9), + `runner_labels` Array(String), -- The raw records on S3, this is populated by the s3 replicator `_meta` Tuple(bucket String, key String) ) From 9a24a151a1747c43c0e95b025b4690f8283a3503 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Wed, 19 Mar 2025 00:35:36 -0700 Subject: [PATCH 7/7] typo --- clickhouse_db_schema/oss_ci_queue_time_histogram/schema.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/clickhouse_db_schema/oss_ci_queue_time_histogram/schema.sql b/clickhouse_db_schema/oss_ci_queue_time_histogram/schema.sql index 68b6fc382c..a310278d35 100644 --- a/clickhouse_db_schema/oss_ci_queue_time_histogram/schema.sql +++ b/clickhouse_db_schema/oss_ci_queue_time_histogram/schema.sql @@ -13,6 +13,7 @@ CREATE TABLE misc.oss_ci_queue_time_histogram( `avg_queue_time` UInt64, `total_count` UInt64, `time` DateTime64(9), + `runner_labels` Array(String), `extra_info` Map(String,String) ) ENGINE = SharedMergeTree('/clickhouse/tables/{uuid}/{shard}', '{replica}')