From 6f2196f6552e4c5c800bd95141396b48d582a224 Mon Sep 17 00:00:00 2001
From: Hector Sanjuan <code@hector.link>
Date: Fri, 7 Mar 2025 08:42:55 +0100
Subject: [PATCH 1/3] deps: update boxo to main-branch commit
---
docs/examples/kubo-as-a-library/go.mod | 4 ++--
docs/examples/kubo-as-a-library/go.sum | 8 ++++----
go.mod | 4 ++--
go.sum | 8 ++++----
test/dependencies/go.mod | 2 +-
test/dependencies/go.sum | 8 ++++----
6 files changed, 17 insertions(+), 17 deletions(-)
diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod
index 8dffe9c1434..8ca4390ed3c 100644
--- a/docs/examples/kubo-as-a-library/go.mod
+++ b/docs/examples/kubo-as-a-library/go.mod
@@ -7,7 +7,7 @@ go 1.24
replace github.com/ipfs/kubo => ./../../..
require (
- github.com/ipfs/boxo v0.29.0
+ github.com/ipfs/boxo v0.29.1-0.20250307073615-93ea580a3bb0
github.com/ipfs/kubo v0.0.0-00010101000000-000000000000
github.com/libp2p/go-libp2p v0.41.0
github.com/multiformats/go-multiaddr v0.15.0
@@ -52,7 +52,7 @@ require (
github.com/francoispqt/gojay v1.2.13 // indirect
github.com/fsnotify/fsnotify v1.7.0 // indirect
github.com/gabriel-vasile/mimetype v1.4.6 // indirect
- github.com/gammazero/chanqueue v1.0.0 // indirect
+ github.com/gammazero/chanqueue v1.1.0 // indirect
github.com/gammazero/deque v1.0.0 // indirect
github.com/getsentry/sentry-go v0.27.0 // indirect
github.com/go-jose/go-jose/v4 v4.0.5 // indirect
diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum
index 9a5fa3be19e..6207a1d8abd 100644
--- a/docs/examples/kubo-as-a-library/go.sum
+++ b/docs/examples/kubo-as-a-library/go.sum
@@ -166,8 +166,8 @@ github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nos
github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
github.com/gabriel-vasile/mimetype v1.4.6 h1:3+PzJTKLkvgjeTbts6msPJt4DixhT4YtFNf1gtGe3zc=
github.com/gabriel-vasile/mimetype v1.4.6/go.mod h1:JX1qVKqZd40hUPpAfiNTe0Sne7hdfKSbOqqmkq8GCXc=
-github.com/gammazero/chanqueue v1.0.0 h1:FER/sMailGFA3DDvFooEkipAMU+3c9Bg3bheloPSz6o=
-github.com/gammazero/chanqueue v1.0.0/go.mod h1:fMwpwEiuUgpab0sH4VHiVcEoji1pSi+EIzeG4TPeKPc=
+github.com/gammazero/chanqueue v1.1.0 h1:yiwtloc1azhgGLFo2gMloJtQvkYD936Ai7tBfa+rYJw=
+github.com/gammazero/chanqueue v1.1.0/go.mod h1:fMwpwEiuUgpab0sH4VHiVcEoji1pSi+EIzeG4TPeKPc=
github.com/gammazero/deque v1.0.0 h1:LTmimT8H7bXkkCy6gZX7zNLtkbz4NdS2z8LZuor3j34=
github.com/gammazero/deque v1.0.0/go.mod h1:iflpYvtGfM3U8S8j+sZEKIak3SAKYpA5/SQewgfXDKo=
github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps=
@@ -298,8 +298,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcd
github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU=
github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
-github.com/ipfs/boxo v0.29.0 h1:clzd7PglUcE+Ufq1KucS3aKID7pzGVaSgcdRsW395t4=
-github.com/ipfs/boxo v0.29.0/go.mod h1:c3R52nMlgMsN1tADffYcogKoVRsX1RJE1TMYSpJ4uVs=
+github.com/ipfs/boxo v0.29.1-0.20250307073615-93ea580a3bb0 h1:wkFvXf7SqRiAu5ezj6lsieE2PwDk3nUkFOs9/+5lWdo=
+github.com/ipfs/boxo v0.29.1-0.20250307073615-93ea580a3bb0/go.mod h1:MkDJStXiJS9U99cbAijHdcmwNfVn5DKYBmQCOgjY2NU=
github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ=
diff --git a/go.mod b/go.mod
index 0bcc1869f6e..0bcfbc99de4 100644
--- a/go.mod
+++ b/go.mod
@@ -21,7 +21,7 @@ require (
github.com/hashicorp/go-version v1.7.0
github.com/ipfs-shipyard/nopfs v0.0.14
github.com/ipfs-shipyard/nopfs/ipfs v0.25.0
- github.com/ipfs/boxo v0.29.0
+ github.com/ipfs/boxo v0.29.1-0.20250307073615-93ea580a3bb0
github.com/ipfs/go-block-format v0.2.0
github.com/ipfs/go-cid v0.5.0
github.com/ipfs/go-cidutil v0.1.0
@@ -125,7 +125,7 @@ require (
github.com/flynn/noise v1.1.0 // indirect
github.com/francoispqt/gojay v1.2.13 // indirect
github.com/gabriel-vasile/mimetype v1.4.6 // indirect
- github.com/gammazero/chanqueue v1.0.0 // indirect
+ github.com/gammazero/chanqueue v1.1.0 // indirect
github.com/gammazero/deque v1.0.0 // indirect
github.com/getsentry/sentry-go v0.27.0 // indirect
github.com/go-jose/go-jose/v4 v4.0.5 // indirect
diff --git a/go.sum b/go.sum
index e29d9352f3c..7e2b6e63afb 100644
--- a/go.sum
+++ b/go.sum
@@ -200,8 +200,8 @@ github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nos
github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
github.com/gabriel-vasile/mimetype v1.4.6 h1:3+PzJTKLkvgjeTbts6msPJt4DixhT4YtFNf1gtGe3zc=
github.com/gabriel-vasile/mimetype v1.4.6/go.mod h1:JX1qVKqZd40hUPpAfiNTe0Sne7hdfKSbOqqmkq8GCXc=
-github.com/gammazero/chanqueue v1.0.0 h1:FER/sMailGFA3DDvFooEkipAMU+3c9Bg3bheloPSz6o=
-github.com/gammazero/chanqueue v1.0.0/go.mod h1:fMwpwEiuUgpab0sH4VHiVcEoji1pSi+EIzeG4TPeKPc=
+github.com/gammazero/chanqueue v1.1.0 h1:yiwtloc1azhgGLFo2gMloJtQvkYD936Ai7tBfa+rYJw=
+github.com/gammazero/chanqueue v1.1.0/go.mod h1:fMwpwEiuUgpab0sH4VHiVcEoji1pSi+EIzeG4TPeKPc=
github.com/gammazero/deque v1.0.0 h1:LTmimT8H7bXkkCy6gZX7zNLtkbz4NdS2z8LZuor3j34=
github.com/gammazero/deque v1.0.0/go.mod h1:iflpYvtGfM3U8S8j+sZEKIak3SAKYpA5/SQewgfXDKo=
github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps=
@@ -362,8 +362,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcd
github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU=
github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
-github.com/ipfs/boxo v0.29.0 h1:clzd7PglUcE+Ufq1KucS3aKID7pzGVaSgcdRsW395t4=
-github.com/ipfs/boxo v0.29.0/go.mod h1:c3R52nMlgMsN1tADffYcogKoVRsX1RJE1TMYSpJ4uVs=
+github.com/ipfs/boxo v0.29.1-0.20250307073615-93ea580a3bb0 h1:wkFvXf7SqRiAu5ezj6lsieE2PwDk3nUkFOs9/+5lWdo=
+github.com/ipfs/boxo v0.29.1-0.20250307073615-93ea580a3bb0/go.mod h1:MkDJStXiJS9U99cbAijHdcmwNfVn5DKYBmQCOgjY2NU=
github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ=
diff --git a/test/dependencies/go.mod b/test/dependencies/go.mod
index 6fca5043c0a..6fb22bf3d68 100644
--- a/test/dependencies/go.mod
+++ b/test/dependencies/go.mod
@@ -116,7 +116,7 @@ require (
github.com/huin/goupnp v1.3.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/ipfs/bbloom v0.0.4 // indirect
- github.com/ipfs/boxo v0.29.0 // indirect
+ github.com/ipfs/boxo v0.29.1-0.20250307073615-93ea580a3bb0 // indirect
github.com/ipfs/go-block-format v0.2.0 // indirect
github.com/ipfs/go-cid v0.5.0 // indirect
github.com/ipfs/go-datastore v0.8.2 // indirect
diff --git a/test/dependencies/go.sum b/test/dependencies/go.sum
index 5ec6fbf5759..1fa1886037b 100644
--- a/test/dependencies/go.sum
+++ b/test/dependencies/go.sum
@@ -156,8 +156,8 @@ github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nos
github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
github.com/fzipp/gocyclo v0.6.0 h1:lsblElZG7d3ALtGMx9fmxeTKZaLLpU8mET09yN4BBLo=
github.com/fzipp/gocyclo v0.6.0/go.mod h1:rXPyn8fnlpa0R2csP/31uerbiVBugk5whMdlyaLkLoA=
-github.com/gammazero/chanqueue v1.0.0 h1:FER/sMailGFA3DDvFooEkipAMU+3c9Bg3bheloPSz6o=
-github.com/gammazero/chanqueue v1.0.0/go.mod h1:fMwpwEiuUgpab0sH4VHiVcEoji1pSi+EIzeG4TPeKPc=
+github.com/gammazero/chanqueue v1.1.0 h1:yiwtloc1azhgGLFo2gMloJtQvkYD936Ai7tBfa+rYJw=
+github.com/gammazero/chanqueue v1.1.0/go.mod h1:fMwpwEiuUgpab0sH4VHiVcEoji1pSi+EIzeG4TPeKPc=
github.com/gammazero/deque v1.0.0 h1:LTmimT8H7bXkkCy6gZX7zNLtkbz4NdS2z8LZuor3j34=
github.com/gammazero/deque v1.0.0/go.mod h1:iflpYvtGfM3U8S8j+sZEKIak3SAKYpA5/SQewgfXDKo=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
@@ -294,8 +294,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
-github.com/ipfs/boxo v0.29.0 h1:clzd7PglUcE+Ufq1KucS3aKID7pzGVaSgcdRsW395t4=
-github.com/ipfs/boxo v0.29.0/go.mod h1:c3R52nMlgMsN1tADffYcogKoVRsX1RJE1TMYSpJ4uVs=
+github.com/ipfs/boxo v0.29.1-0.20250307073615-93ea580a3bb0 h1:wkFvXf7SqRiAu5ezj6lsieE2PwDk3nUkFOs9/+5lWdo=
+github.com/ipfs/boxo v0.29.1-0.20250307073615-93ea580a3bb0/go.mod h1:MkDJStXiJS9U99cbAijHdcmwNfVn5DKYBmQCOgjY2NU=
github.com/ipfs/go-block-format v0.2.0 h1:ZqrkxBA2ICbDRbK8KJs/u0O3dlp6gmAuuXUJNiW1Ycs=
github.com/ipfs/go-block-format v0.2.0/go.mod h1:+jpL11nFx5A/SPpsoBn6Bzkra/zaArfSmsknbPMYgzM=
github.com/ipfs/go-cid v0.5.0 h1:goEKKhaGm0ul11IHA7I6p1GmKz8kEYniqFopaB5Otwg=
From ba22102a640b3f41804319981d83908c96a96275 Mon Sep 17 00:00:00 2001
From: Hector Sanjuan <code@hector.link>
Date: Thu, 6 Mar 2025 15:19:19 +0100
Subject: [PATCH 2/3] provider: buffer pin providers.
Fixes #10596.
The reproviding process can take long. Currently, each CID to be provided is
obtained by making a query to the pinner and reading one by one as the CIDs
get provided.
While this query is ongoing, the pinner holds a Read mutex on the pinset.
If a pin-add-request arrives, a goroutine will start waiting for a Write mutex
on the pinset. From that point, no new Read mutexes can be taken until the writer
can proceed and finishes.
However, no one can proceed because the read mutex is still held while the
reproviding is ongoing.
The fix is mostly in Boxo, where we add a "buffered" provider which reads the
cids onto memory so that they can be provided at its own pace without making
everyone wait.
The consequence is we will need more RAM memory. Rule of thumb is 1GiB extra per 20M cids to be reprovided.
---
core/node/provider.go | 13 ++++++++++---
docs/changelogs/v0.34.md | 5 +++++
2 files changed, 15 insertions(+), 3 deletions(-)
diff --git a/core/node/provider.go b/core/node/provider.go
index d6ab26b0c0d..78f0761cc14 100644
--- a/core/node/provider.go
+++ b/core/node/provider.go
@@ -167,16 +167,23 @@ func newProvidingStrategy(onlyPinned, onlyRoots bool) interface{} {
IPLDFetcher fetcher.Factory `name:"ipldFetcher"`
}
return func(in input) provider.KeyChanFunc {
+ // Pinner-related CIDs will be buffered in memory to avoid
+ // deadlocking the pinner when the providing process is slow.
+
if onlyRoots {
- return provider.NewPinnedProvider(true, in.Pinner, in.IPLDFetcher)
+ return provider.NewBufferedProvider(
+ provider.NewPinnedProvider(true, in.Pinner, in.IPLDFetcher),
+ )
}
if onlyPinned {
- return provider.NewPinnedProvider(false, in.Pinner, in.IPLDFetcher)
+ return provider.NewBufferedProvider(
+ provider.NewPinnedProvider(false, in.Pinner, in.IPLDFetcher),
+ )
}
return provider.NewPrioritizedProvider(
- provider.NewPinnedProvider(true, in.Pinner, in.IPLDFetcher),
+ provider.NewBufferedProvider(provider.NewPinnedProvider(true, in.Pinner, in.IPLDFetcher)),
provider.NewBlockstoreProvider(in.Blockstore),
)
}
diff --git a/docs/changelogs/v0.34.md b/docs/changelogs/v0.34.md
index b11ec3699e3..91877452589 100644
--- a/docs/changelogs/v0.34.md
+++ b/docs/changelogs/v0.34.md
@@ -14,6 +14,7 @@
- [Badger datastore update](#badger-datastore-update)
- [Datastore Implementation Updates](#datastore-implementation-updates)
- [One Multi-error Package](#one-multi-error-package)
+ - [Fix hanging pinset operations during reprovides](#fix-hanging-pinset-operations-during-reprovides)
- [๐ฆ๏ธ Important dependency updates](#-important-dependency-updates)
- [๐จโ๐ฉโ๐งโ๐ฆ Contributors](#-contributors)
@@ -74,6 +75,10 @@ The go-ds-xxx datastore implementations have been updated to support the updated
Kubo previously depended on multiple multi-error packages, `github.com/hashicorp/go-multierror` and `go.uber.org/multierr`. These have nearly identical functionality so there was no need to use both. Therefore, `go.uber.org/multierr` was selected as the package to depend on. Any future code needing multi-error functionality should use `go.uber.org/multierr` to avoid introducing unneeded dependencies.
+#### Fix hanging pinset operations during reprovides
+
+The reprovide process can be quite slow. In default settings, the reprovide process will start reading CIDs that belong to the pinset. During this operation, starvation can occurr for other operations that need pinset access (see https://github.com/ipfs/kubo/issue/10596). We have now switch to buffering pinset-related cids that are going to be reprovided in memory, so that we can free pinset mutexes as soon as possible so that pinset-writes and subsequent read operations can proceed. The downside is larger pinsets will need some extra memory, with an estimation of ~1GiB of RAM memory-use per 20 million items to be reprovided.
+
#### ๐ฆ๏ธ Important dependency updates
- update `go-libp2p` to [v0.41.0](https://github.com/libp2p/go-libp2p/releases/tag/v0.41.0) (incl. [v0.40.0](https://github.com/libp2p/go-libp2p/releases/tag/v0.40.0))
From 9de938479cca7c223bf87cba17b54ce59245e207 Mon Sep 17 00:00:00 2001
From: Guillaume Michel <guillaumemichel@users.noreply.github.com>
Date: Fri, 7 Mar 2025 16:37:44 +0100
Subject: [PATCH 3/3] Update docs/changelogs/v0.34.md
---
docs/changelogs/v0.34.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/docs/changelogs/v0.34.md b/docs/changelogs/v0.34.md
index 91877452589..8c1c5753e3f 100644
--- a/docs/changelogs/v0.34.md
+++ b/docs/changelogs/v0.34.md
@@ -77,7 +77,7 @@ Kubo previously depended on multiple multi-error packages, `github.com/hashicorp
#### Fix hanging pinset operations during reprovides
-The reprovide process can be quite slow. In default settings, the reprovide process will start reading CIDs that belong to the pinset. During this operation, starvation can occurr for other operations that need pinset access (see https://github.com/ipfs/kubo/issue/10596). We have now switch to buffering pinset-related cids that are going to be reprovided in memory, so that we can free pinset mutexes as soon as possible so that pinset-writes and subsequent read operations can proceed. The downside is larger pinsets will need some extra memory, with an estimation of ~1GiB of RAM memory-use per 20 million items to be reprovided.
+The reprovide process can be quite slow. In default settings, the reprovide process will start reading CIDs that belong to the pinset. During this operation, starvation can occur for other operations that need pinset access (see https://github.com/ipfs/kubo/issue/10596). We have now switch to buffering pinset-related cids that are going to be reprovided in memory, so that we can free pinset mutexes as soon as possible so that pinset-writes and subsequent read operations can proceed. The downside is larger pinsets will need some extra memory, with an estimation of ~1GiB of RAM memory-use per 20 million items to be reprovided.
#### ๐ฆ๏ธ Important dependency updates