From 003cee31bb4c6b4543f59fe4b71ff58aecd79dd9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 3 Dec 2024 16:04:59 +0100 Subject: [PATCH 1/3] fix: f3: Keep retrying the task forever --- tasks/f3/f3_task.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tasks/f3/f3_task.go b/tasks/f3/f3_task.go index 6a77619c1..e8968d245 100644 --- a/tasks/f3/f3_task.go +++ b/tasks/f3/f3_task.go @@ -2,7 +2,9 @@ package f3 import ( "context" + "database/sql" "errors" + "github.com/yugabyte/pgx/v5" "time" logging "github.com/ipfs/go-log/v2" @@ -71,6 +73,12 @@ func (f *F3Task) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done boo var spID int64 err = f.db.QueryRow(ctx, "SELECT sp_id FROM f3_tasks WHERE task_id = $1", taskID).Scan(&spID) if err != nil { + if err == pgx.ErrNoRows { + // can only happen when the SP ID was removed from the cluster configuration + log.Warnw("marking f3 task with no corresponding spid as done", "task", taskID) + return true, nil + } + return false, xerrors.Errorf("failed to get sp_id: %w", err) } @@ -195,7 +203,10 @@ func (f *F3Task) TypeDetails() harmonytask.TaskTypeDetails { Gpu: 0, Ram: 10 << 20, }, - MaxFailures: 1, + // No MaxRetries, never kill this task + RetryWait: func(retries int) time.Duration { + return time.Minute + }, } } From ee46c1c7762a4e7358b055b2826750f8f6c4d42b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 4 Dec 2024 15:59:27 +0100 Subject: [PATCH 2/3] Update tasks/f3/f3_task.go Co-authored-by: Jakub Sztandera --- tasks/f3/f3_task.go | 1 - 1 file changed, 1 deletion(-) diff --git a/tasks/f3/f3_task.go b/tasks/f3/f3_task.go index e8968d245..299dab3f1 100644 --- a/tasks/f3/f3_task.go +++ b/tasks/f3/f3_task.go @@ -2,7 +2,6 @@ package f3 import ( "context" - "database/sql" "errors" "github.com/yugabyte/pgx/v5" "time" From 2dba4e1c5e5515a200fd581ac1d210d6815dc319 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 6 Dec 2024 13:13:27 +0100 Subject: [PATCH 3/3] make gen --- tasks/f3/f3_task.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/f3/f3_task.go b/tasks/f3/f3_task.go index 299dab3f1..5a9437c71 100644 --- a/tasks/f3/f3_task.go +++ b/tasks/f3/f3_task.go @@ -3,10 +3,10 @@ package f3 import ( "context" "errors" - "github.com/yugabyte/pgx/v5" "time" logging "github.com/ipfs/go-log/v2" + "github.com/yugabyte/pgx/v5" "golang.org/x/xerrors" "github.com/filecoin-project/go-address"