From 5e295ffee544346c5904d04a8720ab4b37379c96 Mon Sep 17 00:00:00 2001 From: James Barford-Evans Date: Mon, 15 Sep 2025 15:58:43 +0100 Subject: [PATCH 1/6] Fix for `completed` query performance --- database/src/pool/postgres.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/database/src/pool/postgres.rs b/database/src/pool/postgres.rs index ce764a617..f3dff0b5a 100644 --- a/database/src/pool/postgres.rs +++ b/database/src/pool/postgres.rs @@ -410,6 +410,11 @@ static MIGRATIONS: &[&str] = &[ CREATE INDEX error_artifact_idx ON error(aid); "#, + // For completed requests we take the last N completed. As the total number + // of requests grows to make things fast we need an index on the completed_at + r#" + CREATE INDEX IF NOT EXISTS benchmark_request_completed_idx ON benchmark_request(completed_at); + "# ]; #[async_trait::async_trait] From f9368487b239d6fd6b9be290c4764e0bdec52bb5 Mon Sep 17 00:00:00 2001 From: James Barford-Evans Date: Mon, 15 Sep 2025 16:37:53 +0100 Subject: [PATCH 2/6] add error.aid index --- database/src/pool/postgres.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/database/src/pool/postgres.rs b/database/src/pool/postgres.rs index f3dff0b5a..8ec2f3811 100644 --- a/database/src/pool/postgres.rs +++ b/database/src/pool/postgres.rs @@ -414,7 +414,13 @@ static MIGRATIONS: &[&str] = &[ // of requests grows to make things fast we need an index on the completed_at r#" CREATE INDEX IF NOT EXISTS benchmark_request_completed_idx ON benchmark_request(completed_at); - "# + "#, + // Artifacts table grows massively and we do a join on artifacts.id = error.aid. + // Thus error.aid needs an index to filter errors for an artifact without + // a full table scan + r#" + CREATE INDEX IF NOT EXISTS error_aid_idx ON error(aid); + "#, ]; #[async_trait::async_trait] From 859355f95b218d4fb0be4b9b4004fe0bb0cd73a7 Mon Sep 17 00:00:00 2001 From: James Barford-Evans Date: Mon, 15 Sep 2025 17:00:59 +0100 Subject: [PATCH 3/6] possible fix for in_progress --- database/src/pool/postgres.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/database/src/pool/postgres.rs b/database/src/pool/postgres.rs index 8ec2f3811..7f693bd15 100644 --- a/database/src/pool/postgres.rs +++ b/database/src/pool/postgres.rs @@ -754,10 +754,8 @@ impl PostgresConnection { UNION SELECT tag FROM parents ) - SELECT job_queue.* - FROM requests - -- Only get requests that have some jobs - RIGHT JOIN job_queue on job_queue.request_tag = requests.tag + -- Only get the jobs of in_progress requests + SELECT * FROM job_queue WHERE job_queue.request_tag IN (SELECT * FROM requests) ")).await.unwrap(), }), conn, From 6ada48987b5fae2f7479103245646d053f154238 Mon Sep 17 00:00:00 2001 From: James Barford-Evans Date: Wed, 17 Sep 2025 09:58:59 +0100 Subject: [PATCH 4/6] removed index on error.aid as previous PR added it --- database/src/pool/postgres.rs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/database/src/pool/postgres.rs b/database/src/pool/postgres.rs index 7f693bd15..6ce609f92 100644 --- a/database/src/pool/postgres.rs +++ b/database/src/pool/postgres.rs @@ -415,12 +415,6 @@ static MIGRATIONS: &[&str] = &[ r#" CREATE INDEX IF NOT EXISTS benchmark_request_completed_idx ON benchmark_request(completed_at); "#, - // Artifacts table grows massively and we do a join on artifacts.id = error.aid. - // Thus error.aid needs an index to filter errors for an artifact without - // a full table scan - r#" - CREATE INDEX IF NOT EXISTS error_aid_idx ON error(aid); - "#, ]; #[async_trait::async_trait] From b9fd3466b56b0266e947f3e16136601589f1b681 Mon Sep 17 00:00:00 2001 From: James Barford-Evans Date: Fri, 19 Sep 2025 09:07:34 +0100 Subject: [PATCH 5/6] PR Feedback; remove `IF NOT EXISTS` from index creation --- database/src/pool/postgres.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database/src/pool/postgres.rs b/database/src/pool/postgres.rs index 6ce609f92..767164c73 100644 --- a/database/src/pool/postgres.rs +++ b/database/src/pool/postgres.rs @@ -413,7 +413,7 @@ static MIGRATIONS: &[&str] = &[ // For completed requests we take the last N completed. As the total number // of requests grows to make things fast we need an index on the completed_at r#" - CREATE INDEX IF NOT EXISTS benchmark_request_completed_idx ON benchmark_request(completed_at); + CREATE INDEX benchmark_request_completed_idx ON benchmark_request(completed_at); "#, ]; From b880a243b3eda94ec2749fed7c6b6ab80e298ad6 Mon Sep 17 00:00:00 2001 From: James Barford-Evans Date: Fri, 19 Sep 2025 10:01:09 +0100 Subject: [PATCH 6/6] PR Feedback; Use an `INNER JOIN` --- database/src/pool/postgres.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database/src/pool/postgres.rs b/database/src/pool/postgres.rs index 767164c73..89eb640f3 100644 --- a/database/src/pool/postgres.rs +++ b/database/src/pool/postgres.rs @@ -749,7 +749,7 @@ impl PostgresConnection { SELECT tag FROM parents ) -- Only get the jobs of in_progress requests - SELECT * FROM job_queue WHERE job_queue.request_tag IN (SELECT * FROM requests) + SELECT * FROM job_queue INNER JOIN requests ON job_queue.request_tag = requests.tag ")).await.unwrap(), }), conn,