Skip to content

Commit da2dc47

Browse files
lidellidel
committed
fix: use offline path resolvers where appropriate
this fixes the problem described in #10161 (comment) by adding explicit offline path resolvers that are backed by offline exchange, and using them in NoFetch gateways instead of the default online ones
1 parent c99068e commit da2dc47

File tree

9 files changed

+91
-67
lines changed

9 files changed

+91
-67
lines changed

Diff for: ‎core/core.go

+31-27
Original file line numberDiff line numberDiff line change
@@ -76,35 +76,39 @@ type IpfsNode struct {
7676
PNetFingerprint libp2p.PNetFingerprint `optional:"true"` // fingerprint of private network
7777

7878
// Services
79-
Peerstore pstore.Peerstore `optional:"true"` // storage for other Peer instances
80-
Blockstore bstore.GCBlockstore // the block store (lower level)
81-
Filestore *filestore.Filestore `optional:"true"` // the filestore blockstore
82-
BaseBlocks node.BaseBlocks // the raw blockstore, no filestore wrapping
83-
GCLocker bstore.GCLocker // the locker used to protect the blockstore during gc
84-
Blocks bserv.BlockService // the block service, get/add blocks.
85-
DAG ipld.DAGService // the merkle dag service, get/add objects.
86-
IPLDFetcherFactory fetcher.Factory `name:"ipldFetcher"` // fetcher that paths over the IPLD data model
87-
UnixFSFetcherFactory fetcher.Factory `name:"unixfsFetcher"` // fetcher that interprets UnixFS data
88-
Reporter *metrics.BandwidthCounter `optional:"true"`
89-
Discovery mdns.Service `optional:"true"`
90-
FilesRoot *mfs.Root
91-
RecordValidator record.Validator
79+
Peerstore pstore.Peerstore `optional:"true"` // storage for other Peer instances
80+
Blockstore bstore.GCBlockstore // the block store (lower level)
81+
Filestore *filestore.Filestore `optional:"true"` // the filestore blockstore
82+
BaseBlocks node.BaseBlocks // the raw blockstore, no filestore wrapping
83+
GCLocker bstore.GCLocker // the locker used to protect the blockstore during gc
84+
Blocks bserv.BlockService // the block service, get/add blocks.
85+
DAG ipld.DAGService // the merkle dag service, get/add objects.
86+
IPLDFetcherFactory fetcher.Factory `name:"ipldFetcher"` // fetcher that paths over the IPLD data model
87+
UnixFSFetcherFactory fetcher.Factory `name:"unixfsFetcher"` // fetcher that interprets UnixFS data
88+
OfflineIPLDFetcherFactory fetcher.Factory `name:"offlineIpldFetcher"` // fetcher that paths over the IPLD data model without fetching new blocks
89+
OfflineUnixFSFetcherFactory fetcher.Factory `name:"offlineUnixfsFetcher"` // fetcher that interprets UnixFS data without fetching new blocks
90+
Reporter *metrics.BandwidthCounter `optional:"true"`
91+
Discovery mdns.Service `optional:"true"`
92+
FilesRoot *mfs.Root
93+
RecordValidator record.Validator
9294

9395
// Online
94-
PeerHost p2phost.Host `optional:"true"` // the network host (server+client)
95-
Peering *peering.PeeringService `optional:"true"`
96-
Filters *ma.Filters `optional:"true"`
97-
Bootstrapper io.Closer `optional:"true"` // the periodic bootstrapper
98-
Routing irouting.ProvideManyRouter `optional:"true"` // the routing system. recommend ipfs-dht
99-
DNSResolver *madns.Resolver // the DNS resolver
100-
IPLDPathResolver pathresolver.Resolver `name:"ipldPathResolver"` // The IPLD path resolver
101-
UnixFSPathResolver pathresolver.Resolver `name:"unixFSPathResolver"` // The UnixFS path resolver
102-
Exchange exchange.Interface // the block exchange + strategy (bitswap)
103-
Namesys namesys.NameSystem // the name system, resolves paths to hashes
104-
Provider provider.System // the value provider system
105-
IpnsRepub *ipnsrp.Republisher `optional:"true"`
106-
GraphExchange graphsync.GraphExchange `optional:"true"`
107-
ResourceManager network.ResourceManager `optional:"true"`
96+
PeerHost p2phost.Host `optional:"true"` // the network host (server+client)
97+
Peering *peering.PeeringService `optional:"true"`
98+
Filters *ma.Filters `optional:"true"`
99+
Bootstrapper io.Closer `optional:"true"` // the periodic bootstrapper
100+
Routing irouting.ProvideManyRouter `optional:"true"` // the routing system. recommend ipfs-dht
101+
DNSResolver *madns.Resolver // the DNS resolver
102+
IPLDPathResolver pathresolver.Resolver `name:"ipldPathResolver"` // The IPLD path resolver
103+
UnixFSPathResolver pathresolver.Resolver `name:"unixFSPathResolver"` // The UnixFS path resolver
104+
OfflineIPLDPathResolver pathresolver.Resolver `name:"offlineIpldPathResolver"` // The IPLD path resolver that uses only locally available blocks
105+
OfflineUnixFSPathResolver pathresolver.Resolver `name:"offlineUnixFSPathResolver"` // The UnixFS path resolver that uses only locally available blocks
106+
Exchange exchange.Interface // the block exchange + strategy (bitswap)
107+
Namesys namesys.NameSystem // the name system, resolves paths to hashes
108+
Provider provider.System // the value provider system
109+
IpnsRepub *ipnsrp.Republisher `optional:"true"`
110+
GraphExchange graphsync.GraphExchange `optional:"true"`
111+
ResourceManager network.ResourceManager `optional:"true"`
108112

109113
PubSub *pubsub.PubSub `optional:"true"`
110114
PSRouter *psrouter.PubsubValueStore `optional:"true"`

Diff for: ‎core/corehttp/gateway.go

+10-6
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,11 @@ func Libp2pGatewayOption() ServeOption {
8181
return func(n *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) {
8282
bserv := blockservice.New(n.Blocks.Blockstore(), offline.Exchange(n.Blocks.Blockstore()))
8383

84-
backend, err := gateway.NewBlocksBackend(bserv)
84+
backend, err := gateway.NewBlocksBackend(bserv,
85+
// GatewayOverLibp2p only returns things that are in local blockstore
86+
// (same as Gateway.NoFetch=true), we have to pass offline path resolver
87+
gateway.WithResolver(n.OfflineUnixFSPathResolver),
88+
)
8589
if err != nil {
8690
return nil, err
8791
}
@@ -111,7 +115,7 @@ func newGatewayBackend(n *core.IpfsNode) (gateway.IPFSBackend, error) {
111115
bserv := n.Blocks
112116
var vsRouting routing.ValueStore = n.Routing
113117
nsys := n.Namesys
114-
resolver := n.UnixFSPathResolver
118+
pathResolver := n.UnixFSPathResolver
115119

116120
if cfg.Gateway.NoFetch {
117121
bserv = blockservice.New(bserv.Blockstore(), offline.Exchange(bserv.Blockstore()))
@@ -133,15 +137,15 @@ func newGatewayBackend(n *core.IpfsNode) (gateway.IPFSBackend, error) {
133137
return nil, fmt.Errorf("error constructing namesys: %w", err)
134138
}
135139

136-
// Let NewBlocksBackend setup the default resolver using the
137-
// offline backend.
138-
resolver = nil
140+
// Gateway.NoFetch=true requires offline path resolver
141+
// to avoid fetching missing blocks during path traversal
142+
pathResolver = n.OfflineUnixFSPathResolver
139143
}
140144

141145
backend, err := gateway.NewBlocksBackend(bserv,
142146
gateway.WithValueStore(vsRouting),
143147
gateway.WithNameSystem(nsys),
144-
gateway.WithResolver(resolver),
148+
gateway.WithResolver(pathResolver),
145149
)
146150
if err != nil {
147151
return nil, err

Diff for: ‎core/node/core.go

+32-19
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"github.com/ipfs/boxo/blockservice"
88
blockstore "github.com/ipfs/boxo/blockstore"
99
exchange "github.com/ipfs/boxo/exchange"
10+
offline "github.com/ipfs/boxo/exchange/offline"
1011
"github.com/ipfs/boxo/fetcher"
1112
bsfetcher "github.com/ipfs/boxo/fetcher/impl/blockservice"
1213
"github.com/ipfs/boxo/filestore"
@@ -21,9 +22,6 @@ import (
2122
format "github.com/ipfs/go-ipld-format"
2223
"github.com/ipfs/go-unixfsnode"
2324
dagpb "github.com/ipld/go-codec-dagpb"
24-
"github.com/ipld/go-ipld-prime"
25-
basicnode "github.com/ipld/go-ipld-prime/node/basic"
26-
"github.com/ipld/go-ipld-prime/schema"
2725
"go.uber.org/fx"
2826

2927
"github.com/ipfs/kubo/core/node/helpers"
@@ -87,43 +85,58 @@ func (s *syncDagService) Session(ctx context.Context) format.NodeGetter {
8785
// FetchersOut allows injection of fetchers.
8886
type FetchersOut struct {
8987
fx.Out
90-
IPLDFetcher fetcher.Factory `name:"ipldFetcher"`
91-
UnixfsFetcher fetcher.Factory `name:"unixfsFetcher"`
88+
IPLDFetcher fetcher.Factory `name:"ipldFetcher"`
89+
UnixfsFetcher fetcher.Factory `name:"unixfsFetcher"`
90+
OfflineIPLDFetcher fetcher.Factory `name:"offlineIpldFetcher"`
91+
OfflineUnixfsFetcher fetcher.Factory `name:"offlineUnixfsFetcher"`
9292
}
9393

9494
// FetchersIn allows using fetchers for other dependencies.
9595
type FetchersIn struct {
9696
fx.In
97-
IPLDFetcher fetcher.Factory `name:"ipldFetcher"`
98-
UnixfsFetcher fetcher.Factory `name:"unixfsFetcher"`
97+
IPLDFetcher fetcher.Factory `name:"ipldFetcher"`
98+
UnixfsFetcher fetcher.Factory `name:"unixfsFetcher"`
99+
OfflineIPLDFetcher fetcher.Factory `name:"offlineIpldFetcher"`
100+
OfflineUnixfsFetcher fetcher.Factory `name:"offlineUnixfsFetcher"`
99101
}
100102

101103
// FetcherConfig returns a fetcher config that can build new fetcher instances
102104
func FetcherConfig(bs blockservice.BlockService) FetchersOut {
103105
ipldFetcher := bsfetcher.NewFetcherConfig(bs)
104-
ipldFetcher.PrototypeChooser = dagpb.AddSupportToChooser(func(lnk ipld.Link, lnkCtx ipld.LinkContext) (ipld.NodePrototype, error) {
105-
if tlnkNd, ok := lnkCtx.LinkNode.(schema.TypedLinkNode); ok {
106-
return tlnkNd.LinkTargetNodePrototype(), nil
107-
}
108-
return basicnode.Prototype.Any, nil
109-
})
110-
106+
ipldFetcher.PrototypeChooser = dagpb.AddSupportToChooser(bsfetcher.DefaultPrototypeChooser)
111107
unixFSFetcher := ipldFetcher.WithReifier(unixfsnode.Reify)
112-
return FetchersOut{IPLDFetcher: ipldFetcher, UnixfsFetcher: unixFSFetcher}
108+
109+
// Construct offline versions which we can safely use in contexts where
110+
// path resolution should not fetch new blocks via exchange.
111+
offlineBs := blockservice.New(bs.Blockstore(), offline.Exchange(bs.Blockstore()))
112+
offlineIpldFetcher := bsfetcher.NewFetcherConfig(offlineBs)
113+
offlineIpldFetcher.PrototypeChooser = dagpb.AddSupportToChooser(bsfetcher.DefaultPrototypeChooser)
114+
offlineUnixFSFetcher := offlineIpldFetcher.WithReifier(unixfsnode.Reify)
115+
116+
return FetchersOut{
117+
IPLDFetcher: ipldFetcher,
118+
UnixfsFetcher: unixFSFetcher,
119+
OfflineIPLDFetcher: offlineIpldFetcher,
120+
OfflineUnixfsFetcher: offlineUnixFSFetcher,
121+
}
113122
}
114123

115124
// PathResolversOut allows injection of path resolvers
116125
type PathResolversOut struct {
117126
fx.Out
118-
IPLDPathResolver pathresolver.Resolver `name:"ipldPathResolver"`
119-
UnixFSPathResolver pathresolver.Resolver `name:"unixFSPathResolver"`
127+
IPLDPathResolver pathresolver.Resolver `name:"ipldPathResolver"`
128+
UnixFSPathResolver pathresolver.Resolver `name:"unixFSPathResolver"`
129+
OfflineIPLDPathResolver pathresolver.Resolver `name:"offlineIpldPathResolver"`
130+
OfflineUnixFSPathResolver pathresolver.Resolver `name:"offlineUnixFSPathResolver"`
120131
}
121132

122133
// PathResolverConfig creates path resolvers with the given fetchers.
123134
func PathResolverConfig(fetchers FetchersIn) PathResolversOut {
124135
return PathResolversOut{
125-
IPLDPathResolver: pathresolver.NewBasicResolver(fetchers.IPLDFetcher),
126-
UnixFSPathResolver: pathresolver.NewBasicResolver(fetchers.UnixfsFetcher),
136+
IPLDPathResolver: pathresolver.NewBasicResolver(fetchers.IPLDFetcher),
137+
UnixFSPathResolver: pathresolver.NewBasicResolver(fetchers.UnixfsFetcher),
138+
OfflineIPLDPathResolver: pathresolver.NewBasicResolver(fetchers.OfflineIPLDFetcher),
139+
OfflineUnixFSPathResolver: pathresolver.NewBasicResolver(fetchers.OfflineUnixfsFetcher),
127140
}
128141
}
129142

Diff for: ‎docs/examples/kubo-as-a-library/go.mod

+2-2
Original file line numberDiff line numberDiff line change
@@ -61,8 +61,8 @@ require (
6161
github.com/hashicorp/golang-lru v0.5.4 // indirect
6262
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
6363
github.com/huin/goupnp v1.2.0 // indirect
64-
github.com/ipfs-shipyard/nopfs v0.0.12-0.20231025203843-544c829c9d80 // indirect
65-
github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231025203843-544c829c9d80 // indirect
64+
github.com/ipfs-shipyard/nopfs v0.0.12-0.20231027223058-cde3b5ba964c // indirect
65+
github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c // indirect
6666
github.com/ipfs/bbloom v0.0.4 // indirect
6767
github.com/ipfs/go-bitfield v1.1.0 // indirect
6868
github.com/ipfs/go-block-format v0.2.0 // indirect

Diff for: ‎docs/examples/kubo-as-a-library/go.sum

+4-4
Original file line numberDiff line numberDiff line change
@@ -299,10 +299,10 @@ github.com/huin/goupnp v1.2.0 h1:uOKW26NG1hsSSbXIZ1IR7XP9Gjd1U8pnLaCMgntmkmY=
299299
github.com/huin/goupnp v1.2.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8=
300300
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
301301
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
302-
github.com/ipfs-shipyard/nopfs v0.0.12-0.20231025203843-544c829c9d80 h1:fGhrSi5CgMQxw0OdTKjrYXMxKjbmCFOFh4GtD8CK6dA=
303-
github.com/ipfs-shipyard/nopfs v0.0.12-0.20231025203843-544c829c9d80/go.mod h1:1oj4+g/mN6JRuZiXHt5iFRG02e62wp5AKcB3gdgknbk=
304-
github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231025203843-544c829c9d80 h1:O2fhQrtzG44UiGOH8YRVtAtfjCTu+/TubeFymhz5Brg=
305-
github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231025203843-544c829c9d80/go.mod h1:6EekK/jo+TynwSE/ZOiOJd4eEvRXoavEC3vquKtv4yI=
302+
github.com/ipfs-shipyard/nopfs v0.0.12-0.20231027223058-cde3b5ba964c h1:17FO7HnKiFhO7iadu3zCgII+EblpdRmJt5qg9FqQo8Y=
303+
github.com/ipfs-shipyard/nopfs v0.0.12-0.20231027223058-cde3b5ba964c/go.mod h1:1oj4+g/mN6JRuZiXHt5iFRG02e62wp5AKcB3gdgknbk=
304+
github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c h1:7UynTbtdlt+w08ggb1UGLGaGjp1mMaZhoTZSctpn5Ak=
305+
github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c/go.mod h1:6EekK/jo+TynwSE/ZOiOJd4eEvRXoavEC3vquKtv4yI=
306306
github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
307307
github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
308308
github.com/ipfs/boxo v0.13.2-0.20231019090647-a7e134e54ff9 h1:dmjRJU0cMxrGIcd9gBgrRATl2DL0xrmJM/jedPKhQbQ=

Diff for: ‎go.mod

+2-2
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,8 @@ require (
1515
github.com/fsnotify/fsnotify v1.6.0
1616
github.com/google/uuid v1.3.1
1717
github.com/hashicorp/go-multierror v1.1.1
18-
github.com/ipfs-shipyard/nopfs v0.0.12-0.20231025203843-544c829c9d80
19-
github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231025203843-544c829c9d80
18+
github.com/ipfs-shipyard/nopfs v0.0.12-0.20231027223058-cde3b5ba964c
19+
github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c
2020
github.com/ipfs/boxo v0.13.2-0.20231019090647-a7e134e54ff9
2121
github.com/ipfs/go-block-format v0.2.0
2222
github.com/ipfs/go-cid v0.4.1

Diff for: ‎go.sum

+4-4
Original file line numberDiff line numberDiff line change
@@ -333,10 +333,10 @@ github.com/huin/goupnp v1.2.0 h1:uOKW26NG1hsSSbXIZ1IR7XP9Gjd1U8pnLaCMgntmkmY=
333333
github.com/huin/goupnp v1.2.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8=
334334
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
335335
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
336-
github.com/ipfs-shipyard/nopfs v0.0.12-0.20231025203843-544c829c9d80 h1:fGhrSi5CgMQxw0OdTKjrYXMxKjbmCFOFh4GtD8CK6dA=
337-
github.com/ipfs-shipyard/nopfs v0.0.12-0.20231025203843-544c829c9d80/go.mod h1:1oj4+g/mN6JRuZiXHt5iFRG02e62wp5AKcB3gdgknbk=
338-
github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231025203843-544c829c9d80 h1:O2fhQrtzG44UiGOH8YRVtAtfjCTu+/TubeFymhz5Brg=
339-
github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231025203843-544c829c9d80/go.mod h1:6EekK/jo+TynwSE/ZOiOJd4eEvRXoavEC3vquKtv4yI=
336+
github.com/ipfs-shipyard/nopfs v0.0.12-0.20231027223058-cde3b5ba964c h1:17FO7HnKiFhO7iadu3zCgII+EblpdRmJt5qg9FqQo8Y=
337+
github.com/ipfs-shipyard/nopfs v0.0.12-0.20231027223058-cde3b5ba964c/go.mod h1:1oj4+g/mN6JRuZiXHt5iFRG02e62wp5AKcB3gdgknbk=
338+
github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c h1:7UynTbtdlt+w08ggb1UGLGaGjp1mMaZhoTZSctpn5Ak=
339+
github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c/go.mod h1:6EekK/jo+TynwSE/ZOiOJd4eEvRXoavEC3vquKtv4yI=
340340
github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
341341
github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
342342
github.com/ipfs/boxo v0.13.2-0.20231019090647-a7e134e54ff9 h1:dmjRJU0cMxrGIcd9gBgrRATl2DL0xrmJM/jedPKhQbQ=

Diff for: ‎plugin/plugins/nopfs/nopfs.go

+4-2
Original file line numberDiff line numberDiff line change
@@ -62,8 +62,10 @@ func MakeBlocker() (*nopfs.Blocker, error) {
6262
func PathResolvers(fetchers node.FetchersIn, blocker *nopfs.Blocker) node.PathResolversOut {
6363
res := node.PathResolverConfig(fetchers)
6464
return node.PathResolversOut{
65-
IPLDPathResolver: ipfs.WrapResolver(res.IPLDPathResolver, blocker),
66-
UnixFSPathResolver: ipfs.WrapResolver(res.UnixFSPathResolver, blocker),
65+
IPLDPathResolver: ipfs.WrapResolver(res.IPLDPathResolver, blocker),
66+
UnixFSPathResolver: ipfs.WrapResolver(res.UnixFSPathResolver, blocker),
67+
OfflineIPLDPathResolver: ipfs.WrapResolver(res.OfflineIPLDPathResolver, blocker),
68+
OfflineUnixFSPathResolver: ipfs.WrapResolver(res.OfflineUnixFSPathResolver, blocker),
6769
}
6870
}
6971

Diff for: ‎test/cli/content_blocking_test.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,8 @@ import (
2121
)
2222

2323
func TestContentBlocking(t *testing.T) {
24-
t.Parallel()
24+
// NOTE: we can't run this with t.Parallel() because we set IPFS_NS_MAP
25+
// and running in parallel could impact other tests
2526

2627
const blockedMsg = "blocked and cannot be provided"
2728
const statusExpl = "HTTP error code is expected"

0 commit comments

Comments
 (0)