From a7459fac7e1c1eb17c21914a5c45bd7c9ca1e61f Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Wed, 2 Aug 2023 14:39:07 +0200 Subject: [PATCH 01/17] feat!: remove all Provide* functions --- routing/http/client/client.go | 134 +-------------- routing/http/client/client_test.go | 150 ---------------- routing/http/contentrouter/contentrouter.go | 58 +------ .../http/contentrouter/contentrouter_test.go | 60 ------- routing/http/server/server.go | 90 +--------- routing/http/server/server_test.go | 16 +- routing/http/types/json/provider.go | 80 +-------- routing/http/types/provider.go | 11 -- routing/http/types/provider_bitswap.go | 161 +----------------- routing/http/types/provider_unknown.go | 7 +- 10 files changed, 19 insertions(+), 748 deletions(-) diff --git a/routing/http/client/client.go b/routing/http/client/client.go index c504a0315..0c5d3f98f 100644 --- a/routing/http/client/client.go +++ b/routing/http/client/client.go @@ -10,28 +10,22 @@ import ( "mime" "net/http" "strings" - "time" "github.com/benbjohnson/clock" ipns "github.com/ipfs/boxo/ipns" "github.com/ipfs/boxo/routing/http/contentrouter" - "github.com/ipfs/boxo/routing/http/internal/drjson" - "github.com/ipfs/boxo/routing/http/server" "github.com/ipfs/boxo/routing/http/types" "github.com/ipfs/boxo/routing/http/types/iter" jsontypes "github.com/ipfs/boxo/routing/http/types/json" "github.com/ipfs/boxo/routing/http/types/ndjson" "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" - record "github.com/libp2p/go-libp2p-record" - "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/peer" - "github.com/multiformats/go-multiaddr" ) var ( _ contentrouter.Client = &client{} - logger = logging.Logger("service/delegatedrouting") + logger = logging.Logger("routing/http/client") defaultHTTPClient = &http.Client{ Transport: &ResponseBodyLimitedTransport{ RoundTripper: http.DefaultTransport, @@ -50,18 +44,8 @@ const ( type client struct { baseURL string httpClient httpClient - validator record.Validator clock clock.Clock - - accepts string - - peerID peer.ID - addrs []types.Multiaddr - identity crypto.PrivKey - - // called immeidately after signing a provide req - // used for testing, e.g. testing the server with a mangled signature - afterSignCallback func(req *types.WriteBitswapProviderRecord) + accepts string } // defaultUserAgent is used as a fallback to inform HTTP server which library @@ -76,12 +60,6 @@ type httpClient interface { type Option func(*client) -func WithIdentity(identity crypto.PrivKey) Option { - return func(c *client) { - c.identity = identity - } -} - func WithHTTPClient(h httpClient) Option { return func(c *client) { c.httpClient = h @@ -105,15 +83,6 @@ func WithUserAgent(ua string) Option { } } -func WithProviderInfo(peerID peer.ID, addrs []multiaddr.Multiaddr) Option { - return func(c *client) { - c.peerID = peerID - for _, a := range addrs { - c.addrs = append(c.addrs, types.Multiaddr{Multiaddr: a}) - } - } -} - func WithStreamResultsRequired() Option { return func(c *client) { c.accepts = mediaTypeNDJSON @@ -121,12 +90,10 @@ func WithStreamResultsRequired() Option { } // New creates a content routing API client. -// The Provider and identity parameters are option. If they are nil, the `Provide` method will not function. func New(baseURL string, opts ...Option) (*client, error) { client := &client{ baseURL: baseURL, httpClient: defaultHTTPClient, - validator: ipns.Validator{}, clock: clock.New(), accepts: strings.Join([]string{mediaTypeNDJSON, mediaTypeJSON}, ","), } @@ -135,10 +102,6 @@ func New(baseURL string, opts ...Option) (*client, error) { opt(client) } - if client.identity != nil && client.peerID.Size() != 0 && !client.peerID.MatchesPublicKey(client.identity.GetPublic()) { - return nil, errors.New("identity does not match provider") - } - return client, nil } @@ -168,7 +131,7 @@ func (c *client) FindProviders(ctx context.Context, key cid.Cid) (provs iter.Res // TODO test measurements m := newMeasurement("FindProviders") - url := c.baseURL + server.ProvidePath + key.String() + url := c.baseURL + "/routing/v1/providers/" + key.String() req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { return nil, err @@ -223,7 +186,7 @@ func (c *client) FindProviders(ctx context.Context, key cid.Cid) (provs iter.Res var it iter.ResultIter[types.ProviderResponse] switch mediaType { case mediaTypeJSON: - parsedResp := &jsontypes.ReadProvidersResponse{} + parsedResp := &jsontypes.ProvidersResponse{} err = json.NewDecoder(resp.Body).Decode(parsedResp) var sliceIt iter.Iter[types.ProviderResponse] = iter.FromSlice(parsedResp.Providers) it = iter.ToResultIter(sliceIt) @@ -238,95 +201,6 @@ func (c *client) FindProviders(ctx context.Context, key cid.Cid) (provs iter.Res return &measuringIter[iter.Result[types.ProviderResponse]]{Iter: it, ctx: ctx, m: m}, nil } -func (c *client) ProvideBitswap(ctx context.Context, keys []cid.Cid, ttl time.Duration) (time.Duration, error) { - if c.identity == nil { - return 0, errors.New("cannot provide Bitswap records without an identity") - } - if c.peerID.Size() == 0 { - return 0, errors.New("cannot provide Bitswap records without a peer ID") - } - - ks := make([]types.CID, len(keys)) - for i, c := range keys { - ks[i] = types.CID{Cid: c} - } - - now := c.clock.Now() - - req := types.WriteBitswapProviderRecord{ - Protocol: "transport-bitswap", - Schema: types.SchemaBitswap, - Payload: types.BitswapPayload{ - Keys: ks, - AdvisoryTTL: &types.Duration{Duration: ttl}, - Timestamp: &types.Time{Time: now}, - ID: &c.peerID, - Addrs: c.addrs, - }, - } - err := req.Sign(c.peerID, c.identity) - if err != nil { - return 0, err - } - - if c.afterSignCallback != nil { - c.afterSignCallback(&req) - } - - advisoryTTL, err := c.provideSignedBitswapRecord(ctx, &req) - if err != nil { - return 0, err - } - - return advisoryTTL, err -} - -// ProvideAsync makes a provide request to a delegated router -func (c *client) provideSignedBitswapRecord(ctx context.Context, bswp *types.WriteBitswapProviderRecord) (time.Duration, error) { - req := jsontypes.WriteProvidersRequest{Providers: []types.WriteProviderRecord{bswp}} - - url := c.baseURL + server.ProvidePath - - b, err := drjson.MarshalJSONBytes(req) - if err != nil { - return 0, err - } - - httpReq, err := http.NewRequestWithContext(ctx, http.MethodPut, url, bytes.NewBuffer(b)) - if err != nil { - return 0, err - } - - resp, err := c.httpClient.Do(httpReq) - if err != nil { - return 0, fmt.Errorf("making HTTP req to provide a signed record: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return 0, httpError(resp.StatusCode, resp.Body) - } - var provideResult jsontypes.WriteProvidersResponse - err = json.NewDecoder(resp.Body).Decode(&provideResult) - if err != nil { - return 0, err - } - if len(provideResult.ProvideResults) != 1 { - return 0, fmt.Errorf("expected 1 result but got %d", len(provideResult.ProvideResults)) - } - - v, ok := provideResult.ProvideResults[0].(*types.WriteBitswapProviderRecordResponse) - if !ok { - return 0, fmt.Errorf("expected AdvisoryTTL field") - } - - if v.AdvisoryTTL != nil { - return v.AdvisoryTTL.Duration, nil - } - - return 0, nil -} - func (c *client) FindIPNSRecord(ctx context.Context, name ipns.Name) (*ipns.Record, error) { url := c.baseURL + "/routing/v1/ipns/" + name.String() diff --git a/routing/http/client/client_test.go b/routing/http/client/client_test.go index c1690b3f2..35ee207c4 100644 --- a/routing/http/client/client_test.go +++ b/routing/http/client/client_test.go @@ -10,7 +10,6 @@ import ( "testing" "time" - "github.com/benbjohnson/clock" "github.com/ipfs/boxo/coreiface/path" ipns "github.com/ipfs/boxo/ipns" ipfspath "github.com/ipfs/boxo/path" @@ -22,7 +21,6 @@ import ( "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/peer" "github.com/multiformats/go-multiaddr" - "github.com/multiformats/go-multibase" "github.com/multiformats/go-multihash" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -36,16 +34,6 @@ func (m *mockContentRouter) FindProviders(ctx context.Context, key cid.Cid, limi return args.Get(0).(iter.ResultIter[types.ProviderResponse]), args.Error(1) } -func (m *mockContentRouter) ProvideBitswap(ctx context.Context, req *server.BitswapWriteProvideRequest) (time.Duration, error) { - args := m.Called(ctx, req) - return args.Get(0).(time.Duration), args.Error(1) -} - -func (m *mockContentRouter) Provide(ctx context.Context, req *server.WriteProvideRequest) (types.ProviderResponse, error) { - args := m.Called(ctx, req) - return args.Get(0).(types.ProviderResponse), args.Error(1) -} - func (m *mockContentRouter) FindIPNSRecord(ctx context.Context, name ipns.Name) (*ipns.Record, error) { args := m.Called(ctx, name) return args.Get(0).(*ipns.Record), args.Error(1) @@ -63,8 +51,6 @@ type testDeps struct { recordingHTTPClient *recordingHTTPClient router *mockContentRouter server *httptest.Server - peerID peer.ID - addrs []multiaddr.Multiaddr client *client } @@ -95,7 +81,6 @@ func (c *recordingHTTPClient) Do(req *http.Request) (*http.Response, error) { func makeTestDeps(t *testing.T, clientsOpts []Option, serverOpts []server.Option) testDeps { const testUserAgent = "testUserAgent" - peerID, addrs, identity := makeProviderAndIdentity() router := &mockContentRouter{} recordingHandler := &recordingHandler{ Handler: server.Handler(router, serverOpts...), @@ -110,8 +95,6 @@ func makeTestDeps(t *testing.T, clientsOpts []Option, serverOpts []server.Option serverAddr := "http://" + server.Listener.Addr().String() recordingHTTPClient := &recordingHTTPClient{httpClient: defaultHTTPClient} defaultClientOpts := []Option{ - WithProviderInfo(peerID, addrs), - WithIdentity(identity), WithUserAgent(testUserAgent), WithHTTPClient(recordingHTTPClient), } @@ -124,8 +107,6 @@ func makeTestDeps(t *testing.T, clientsOpts []Option, serverOpts []server.Option recordingHTTPClient: recordingHTTPClient, router: router, server: server, - peerID: peerID, - addrs: addrs, client: c, } } @@ -151,13 +132,6 @@ func addrsToDRAddrs(addrs []multiaddr.Multiaddr) (drmas []types.Multiaddr) { return } -func drAddrsToAddrs(drmas []types.Multiaddr) (addrs []multiaddr.Multiaddr) { - for _, a := range drmas { - addrs = append(addrs, a.Multiaddr) - } - return -} - func makeBSReadProviderResp() types.ReadBitswapProviderRecord { peerID, addrs, _ := makeProviderAndIdentity() return types.ReadBitswapProviderRecord{ @@ -333,130 +307,6 @@ func TestClient_FindProviders(t *testing.T) { } } -func TestClient_Provide(t *testing.T) { - cases := []struct { - name string - manglePath bool - mangleSignature bool - stopServer bool - noProviderInfo bool - noIdentity bool - - cids []cid.Cid - ttl time.Duration - - routerAdvisoryTTL time.Duration - routerErr error - - expErrContains string - expWinErrContains string - - expAdvisoryTTL time.Duration - }{ - { - name: "happy case", - cids: []cid.Cid{makeCID()}, - ttl: 1 * time.Hour, - routerAdvisoryTTL: 1 * time.Minute, - - expAdvisoryTTL: 1 * time.Minute, - }, - { - name: "should return a 403 if the payload signature verification fails", - cids: []cid.Cid{}, - mangleSignature: true, - expErrContains: "HTTP error with StatusCode=403", - }, - { - name: "should return error if identity is not provided", - noIdentity: true, - expErrContains: "cannot provide Bitswap records without an identity", - }, - { - name: "should return error if provider is not provided", - noProviderInfo: true, - expErrContains: "cannot provide Bitswap records without a peer ID", - }, - { - name: "returns an error if there's a non-200 response", - manglePath: true, - expErrContains: "HTTP error with StatusCode=404: 404 page not found", - }, - { - name: "returns an error if the HTTP client returns a non-HTTP error", - stopServer: true, - expErrContains: "connect: connection refused", - expWinErrContains: "connectex: No connection could be made because the target machine actively refused it.", - }, - } - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - deps := makeTestDeps(t, nil, nil) - client := deps.client - router := deps.router - - if c.noIdentity { - client.identity = nil - } - if c.noProviderInfo { - client.peerID = "" - client.addrs = nil - } - - clock := clock.NewMock() - clock.Set(time.Now()) - client.clock = clock - - ctx := context.Background() - - if c.manglePath { - client.baseURL += "/foo" - } - if c.stopServer { - deps.server.Close() - } - if c.mangleSignature { - client.afterSignCallback = func(req *types.WriteBitswapProviderRecord) { - mh, err := multihash.Encode([]byte("boom"), multihash.SHA2_256) - require.NoError(t, err) - mb, err := multibase.Encode(multibase.Base64, mh) - require.NoError(t, err) - - req.Signature = mb - } - } - - expectedProvReq := &server.BitswapWriteProvideRequest{ - Keys: c.cids, - Timestamp: clock.Now().Truncate(time.Millisecond), - AdvisoryTTL: c.ttl, - Addrs: drAddrsToAddrs(client.addrs), - ID: client.peerID, - } - - router.On("ProvideBitswap", mock.Anything, expectedProvReq). - Return(c.routerAdvisoryTTL, c.routerErr) - - advisoryTTL, err := client.ProvideBitswap(ctx, c.cids, c.ttl) - - var errorString string - if runtime.GOOS == "windows" && c.expWinErrContains != "" { - errorString = c.expWinErrContains - } else { - errorString = c.expErrContains - } - - if errorString != "" { - require.ErrorContains(t, err, errorString) - } else { - require.NoError(t, err) - } - - assert.Equal(t, c.expAdvisoryTTL, advisoryTTL) - }) - } -} - func makeName(t *testing.T) (crypto.PrivKey, ipns.Name) { sk, _, err := crypto.GenerateEd25519Key(rand.Reader) require.NoError(t, err) diff --git a/routing/http/contentrouter/contentrouter.go b/routing/http/contentrouter/contentrouter.go index 8318a3163..0881e5841 100644 --- a/routing/http/contentrouter/contentrouter.go +++ b/routing/http/contentrouter/contentrouter.go @@ -5,11 +5,11 @@ import ( "reflect" "time" - "github.com/ipfs/boxo/routing/http/internal" "github.com/ipfs/boxo/routing/http/types" "github.com/ipfs/boxo/routing/http/types/iter" "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" + routinghelpers "github.com/libp2p/go-libp2p-routing-helpers" "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/routing" "github.com/multiformats/go-multiaddr" @@ -21,37 +21,21 @@ var logger = logging.Logger("service/contentrouting") const ttl = 24 * time.Hour type Client interface { - ProvideBitswap(ctx context.Context, keys []cid.Cid, ttl time.Duration) (time.Duration, error) FindProviders(ctx context.Context, key cid.Cid) (iter.ResultIter[types.ProviderResponse], error) } type contentRouter struct { - client Client - maxProvideConcurrency int - maxProvideBatchSize int + client Client } var _ routing.ContentRouting = (*contentRouter)(nil) +var _ routinghelpers.ProvideManyRouter = (*contentRouter)(nil) type option func(c *contentRouter) -func WithMaxProvideConcurrency(max int) option { - return func(c *contentRouter) { - c.maxProvideConcurrency = max - } -} - -func WithMaxProvideBatchSize(max int) option { - return func(c *contentRouter) { - c.maxProvideBatchSize = max - } -} - func NewContentRoutingClient(c Client, opts ...option) *contentRouter { cr := &contentRouter{ - client: c, - maxProvideConcurrency: 5, - maxProvideBatchSize: 100, + client: c, } for _, opt := range opts { opt(cr) @@ -60,41 +44,11 @@ func NewContentRoutingClient(c Client, opts ...option) *contentRouter { } func (c *contentRouter) Provide(ctx context.Context, key cid.Cid, announce bool) error { - // If 'true' is - // passed, it also announces it, otherwise it is just kept in the local - // accounting of which objects are being provided. - if !announce { - return nil - } - - _, err := c.client.ProvideBitswap(ctx, []cid.Cid{key}, ttl) - return err + return routing.ErrNotSupported } -// ProvideMany provides a set of keys to the remote delegate. -// Large sets of keys are chunked into multiple requests and sent concurrently, according to the concurrency configuration. -// TODO: implement retries through transient errors func (c *contentRouter) ProvideMany(ctx context.Context, mhKeys []multihash.Multihash) error { - keys := make([]cid.Cid, 0, len(mhKeys)) - for _, m := range mhKeys { - keys = append(keys, cid.NewCidV1(cid.Raw, m)) - } - - if len(keys) <= c.maxProvideBatchSize { - _, err := c.client.ProvideBitswap(ctx, keys, ttl) - return err - } - - return internal.DoBatch( - ctx, - c.maxProvideBatchSize, - c.maxProvideConcurrency, - keys, - func(ctx context.Context, batch []cid.Cid) error { - _, err := c.client.ProvideBitswap(ctx, batch, ttl) - return err - }, - ) + return routing.ErrNotSupported } // Ready is part of the existing `ProvideMany` interface. diff --git a/routing/http/contentrouter/contentrouter_test.go b/routing/http/contentrouter/contentrouter_test.go index 3830482e2..9b2861e15 100644 --- a/routing/http/contentrouter/contentrouter_test.go +++ b/routing/http/contentrouter/contentrouter_test.go @@ -4,25 +4,18 @@ import ( "context" "crypto/rand" "testing" - "time" "github.com/ipfs/boxo/routing/http/types" "github.com/ipfs/boxo/routing/http/types/iter" "github.com/ipfs/go-cid" "github.com/libp2p/go-libp2p/core/peer" "github.com/multiformats/go-multihash" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) type mockClient struct{ mock.Mock } -func (m *mockClient) ProvideBitswap(ctx context.Context, keys []cid.Cid, ttl time.Duration) (time.Duration, error) { - args := m.Called(ctx, keys, ttl) - return args.Get(0).(time.Duration), args.Error(1) -} - func (m *mockClient) FindProviders(ctx context.Context, key cid.Cid) (iter.ResultIter[types.ProviderResponse], error) { args := m.Called(ctx, key) return args.Get(0).(iter.ResultIter[types.ProviderResponse]), args.Error(1) @@ -47,59 +40,6 @@ func makeCID() cid.Cid { return c } -func TestProvide(t *testing.T) { - for _, c := range []struct { - name string - announce bool - - expNotProvided bool - }{ - { - name: "announce=false results in no client request", - announce: false, - expNotProvided: true, - }, - { - name: "announce=true results in a client req", - announce: true, - }, - } { - t.Run(c.name, func(t *testing.T) { - ctx := context.Background() - key := makeCID() - client := &mockClient{} - crc := NewContentRoutingClient(client) - - if !c.expNotProvided { - client.On("ProvideBitswap", ctx, []cid.Cid{key}, ttl).Return(time.Minute, nil) - } - - err := crc.Provide(ctx, key, c.announce) - assert.NoError(t, err) - - if c.expNotProvided { - client.AssertNumberOfCalls(t, "ProvideBitswap", 0) - } - }) - } -} - -func TestProvideMany(t *testing.T) { - cids := []cid.Cid{makeCID(), makeCID()} - var mhs []multihash.Multihash - for _, c := range cids { - mhs = append(mhs, c.Hash()) - } - ctx := context.Background() - client := &mockClient{} - crc := NewContentRoutingClient(client) - - client.On("ProvideBitswap", ctx, cids, ttl).Return(time.Minute, nil) - - err := crc.ProvideMany(ctx, mhs) - require.NoError(t, err) -} - func TestFindProvidersAsync(t *testing.T) { key := makeCID() ctx := context.Background() diff --git a/routing/http/server/server.go b/routing/http/server/server.go index 835262990..6b2591e41 100644 --- a/routing/http/server/server.go +++ b/routing/http/server/server.go @@ -3,7 +3,6 @@ package server import ( "bytes" "context" - "encoding/json" "errors" "fmt" "io" @@ -11,7 +10,6 @@ import ( "net/http" "strconv" "strings" - "time" "github.com/cespare/xxhash/v2" "github.com/gorilla/mux" @@ -21,8 +19,6 @@ import ( "github.com/ipfs/boxo/routing/http/types/iter" jsontypes "github.com/ipfs/boxo/routing/http/types/json" "github.com/ipfs/go-cid" - "github.com/libp2p/go-libp2p/core/peer" - "github.com/multiformats/go-multiaddr" logging "github.com/ipfs/go-log/v2" ) @@ -40,7 +36,6 @@ const ( var logger = logging.Logger("service/server/delegatedrouting") const ( - ProvidePath = "/routing/v1/providers/" FindProvidersPath = "/routing/v1/providers/{cid}" IPNSPath = "/routing/v1/ipns/{cid}" ) @@ -54,8 +49,6 @@ type ContentRouter interface { // FindProviders searches for peers who are able to provide a given key. Limit // indicates the maximum amount of results to return. 0 means unbounded. FindProviders(ctx context.Context, key cid.Cid, limit int) (iter.ResultIter[types.ProviderResponse], error) - ProvideBitswap(ctx context.Context, req *BitswapWriteProvideRequest) (time.Duration, error) - Provide(ctx context.Context, req *WriteProvideRequest) (types.ProviderResponse, error) // FindIPNSRecord searches for an [ipns.Record] for the given [ipns.Name]. FindIPNSRecord(ctx context.Context, name ipns.Name) (*ipns.Record, error) @@ -65,20 +58,6 @@ type ContentRouter interface { ProvideIPNSRecord(ctx context.Context, name ipns.Name, record *ipns.Record) error } -type BitswapWriteProvideRequest struct { - Keys []cid.Cid - Timestamp time.Time - AdvisoryTTL time.Duration - ID peer.ID - Addrs []multiaddr.Multiaddr -} - -type WriteProvideRequest struct { - Protocol string - Schema string - Bytes []byte -} - type Option func(s *server) // WithStreamingResultsDisabled disables ndjson responses, so that the server only supports JSON responses. @@ -116,7 +95,6 @@ func Handler(svc ContentRouter, opts ...Option) http.Handler { } r := mux.NewRouter() - r.HandleFunc(ProvidePath, server.provide).Methods(http.MethodPut) r.HandleFunc(FindProvidersPath, server.findProviders).Methods(http.MethodGet) r.HandleFunc(IPNSPath, server.getIPNSRecord).Methods(http.MethodGet) @@ -132,72 +110,6 @@ type server struct { streamingRecordsLimit int } -func (s *server) provide(w http.ResponseWriter, httpReq *http.Request) { - req := jsontypes.WriteProvidersRequest{} - err := json.NewDecoder(httpReq.Body).Decode(&req) - _ = httpReq.Body.Close() - if err != nil { - writeErr(w, "Provide", http.StatusBadRequest, fmt.Errorf("invalid request: %w", err)) - return - } - - resp := jsontypes.WriteProvidersResponse{} - - for i, prov := range req.Providers { - switch v := prov.(type) { - case *types.WriteBitswapProviderRecord: - err := v.Verify() - if err != nil { - logErr("Provide", "signature verification failed", err) - writeErr(w, "Provide", http.StatusForbidden, errors.New("signature verification failed")) - return - } - - keys := make([]cid.Cid, len(v.Payload.Keys)) - for i, k := range v.Payload.Keys { - keys[i] = k.Cid - } - addrs := make([]multiaddr.Multiaddr, len(v.Payload.Addrs)) - for i, a := range v.Payload.Addrs { - addrs[i] = a.Multiaddr - } - advisoryTTL, err := s.svc.ProvideBitswap(httpReq.Context(), &BitswapWriteProvideRequest{ - Keys: keys, - Timestamp: v.Payload.Timestamp.Time, - AdvisoryTTL: v.Payload.AdvisoryTTL.Duration, - ID: *v.Payload.ID, - Addrs: addrs, - }) - if err != nil { - writeErr(w, "Provide", http.StatusInternalServerError, fmt.Errorf("delegate error: %w", err)) - return - } - resp.ProvideResults = append(resp.ProvideResults, - &types.WriteBitswapProviderRecordResponse{ - Protocol: v.Protocol, - Schema: v.Schema, - AdvisoryTTL: &types.Duration{Duration: advisoryTTL}, - }, - ) - case *types.UnknownProviderRecord: - provResp, err := s.svc.Provide(httpReq.Context(), &WriteProvideRequest{ - Protocol: v.Protocol, - Schema: v.Schema, - Bytes: v.Bytes, - }) - if err != nil { - writeErr(w, "Provide", http.StatusInternalServerError, fmt.Errorf("delegate error: %w", err)) - return - } - resp.ProvideResults = append(resp.ProvideResults, provResp) - default: - writeErr(w, "Provide", http.StatusBadRequest, fmt.Errorf("provider record %d does not contain a protocol", i)) - return - } - } - writeJSONResult(w, "Provide", resp) -} - func (s *server) findProviders(w http.ResponseWriter, httpReq *http.Request) { vars := mux.Vars(httpReq) cidStr := vars["cid"] @@ -272,7 +184,7 @@ func (s *server) findProvidersJSON(w http.ResponseWriter, provIter iter.ResultIt providers = append(providers, res.Val) i++ } - response := jsontypes.ReadProvidersResponse{Providers: providers} + response := jsontypes.ProvidersResponse{Providers: providers} writeJSONResult(w, "FindProviders", response) } diff --git a/routing/http/server/server_test.go b/routing/http/server/server_test.go index dfe38f0da..04f2961e4 100644 --- a/routing/http/server/server_test.go +++ b/routing/http/server/server_test.go @@ -43,13 +43,13 @@ func TestHeaders(t *testing.T) { router.On("FindProviders", mock.Anything, cb, DefaultRecordsLimit). Return(results, nil) - resp, err := http.Get(serverAddr + ProvidePath + c) + resp, err := http.Get(serverAddr + "/routing/v1/providers/" + c) require.NoError(t, err) require.Equal(t, 200, resp.StatusCode) header := resp.Header.Get("Content-Type") require.Equal(t, mediaTypeJSON, header) - resp, err = http.Get(serverAddr + ProvidePath + "BAD_CID") + resp, err = http.Get(serverAddr + "/routing/v1/providers/" + "BAD_CID") require.NoError(t, err) defer resp.Body.Close() require.Equal(t, 400, resp.StatusCode) @@ -98,7 +98,7 @@ func TestResponse(t *testing.T) { limit = DefaultStreamingRecordsLimit } router.On("FindProviders", mock.Anything, cid, limit).Return(results, nil) - urlStr := serverAddr + ProvidePath + cidStr + urlStr := serverAddr + "/routing/v1/providers/" + cidStr req, err := http.NewRequest(http.MethodGet, urlStr, nil) require.NoError(t, err) @@ -264,16 +264,6 @@ func (m *mockContentRouter) FindProviders(ctx context.Context, key cid.Cid, limi return args.Get(0).(iter.ResultIter[types.ProviderResponse]), args.Error(1) } -func (m *mockContentRouter) ProvideBitswap(ctx context.Context, req *BitswapWriteProvideRequest) (time.Duration, error) { - args := m.Called(ctx, req) - return args.Get(0).(time.Duration), args.Error(1) -} - -func (m *mockContentRouter) Provide(ctx context.Context, req *WriteProvideRequest) (types.ProviderResponse, error) { - args := m.Called(ctx, req) - return args.Get(0).(types.ProviderResponse), args.Error(1) -} - func (m *mockContentRouter) FindIPNSRecord(ctx context.Context, name ipns.Name) (*ipns.Record, error) { args := m.Called(ctx, name) return args.Get(0).(*ipns.Record), args.Error(1) diff --git a/routing/http/types/json/provider.go b/routing/http/types/json/provider.go index 351197338..3163e1144 100644 --- a/routing/http/types/json/provider.go +++ b/routing/http/types/json/provider.go @@ -6,12 +6,12 @@ import ( "github.com/ipfs/boxo/routing/http/types" ) -// ReadProvidersResponse is the result of a Provide request -type ReadProvidersResponse struct { +// ProvidersResponse is the result of a GET Providers request. +type ProvidersResponse struct { Providers []types.ProviderResponse } -func (r *ReadProvidersResponse) UnmarshalJSON(b []byte) error { +func (r *ProvidersResponse) UnmarshalJSON(b []byte) error { var tempFPR struct{ Providers []json.RawMessage } err := json.Unmarshal(b, &tempFPR) if err != nil { @@ -40,77 +40,3 @@ func (r *ReadProvidersResponse) UnmarshalJSON(b []byte) error { } return nil } - -type WriteProvidersRequest struct { - Providers []types.WriteProviderRecord -} - -func (r *WriteProvidersRequest) UnmarshalJSON(b []byte) error { - type wpr struct{ Providers []json.RawMessage } - var tempWPR wpr - err := json.Unmarshal(b, &tempWPR) - if err != nil { - return err - } - - for _, provBytes := range tempWPR.Providers { - var rawProv types.UnknownProviderRecord - err := json.Unmarshal(provBytes, &rawProv) - if err != nil { - return err - } - - switch rawProv.Schema { - case types.SchemaBitswap: - var prov types.WriteBitswapProviderRecord - err := json.Unmarshal(rawProv.Bytes, &prov) - if err != nil { - return err - } - r.Providers = append(r.Providers, &prov) - default: - var prov types.UnknownProviderRecord - err := json.Unmarshal(b, &prov) - if err != nil { - return err - } - r.Providers = append(r.Providers, &prov) - } - } - return nil -} - -// WriteProvidersResponse is the result of a Provide operation -type WriteProvidersResponse struct { - ProvideResults []types.ProviderResponse -} - -func (r *WriteProvidersResponse) UnmarshalJSON(b []byte) error { - var tempWPR struct{ ProvideResults []json.RawMessage } - err := json.Unmarshal(b, &tempWPR) - if err != nil { - return err - } - - for _, provBytes := range tempWPR.ProvideResults { - var rawProv types.UnknownProviderRecord - err := json.Unmarshal(provBytes, &rawProv) - if err != nil { - return err - } - - switch rawProv.Schema { - case types.SchemaBitswap: - var prov types.WriteBitswapProviderRecordResponse - err := json.Unmarshal(rawProv.Bytes, &prov) - if err != nil { - return err - } - r.ProvideResults = append(r.ProvideResults, &prov) - default: - r.ProvideResults = append(r.ProvideResults, &rawProv) - } - } - - return nil -} diff --git a/routing/http/types/provider.go b/routing/http/types/provider.go index 6e8e303f7..aa311d8ed 100644 --- a/routing/http/types/provider.go +++ b/routing/http/types/provider.go @@ -1,17 +1,6 @@ package types -// WriteProviderRecord is a type that enforces structs to imlement it to avoid confusion -type WriteProviderRecord interface { - IsWriteProviderRecord() -} - -// ReadProviderRecord is a type that enforces structs to imlement it to avoid confusion -type ReadProviderRecord interface { - IsReadProviderRecord() -} - // ProviderResponse is implemented for any ProviderResponse. It needs to have a Protocol field. type ProviderResponse interface { - GetProtocol() string GetSchema() string } diff --git a/routing/http/types/provider_bitswap.go b/routing/http/types/provider_bitswap.go index f0b5056e4..9add2e39b 100644 --- a/routing/http/types/provider_bitswap.go +++ b/routing/http/types/provider_bitswap.go @@ -1,171 +1,12 @@ package types import ( - "crypto/sha256" - "encoding/json" - "errors" - "fmt" - - "github.com/ipfs/boxo/routing/http/internal/drjson" - "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/peer" - "github.com/multiformats/go-multibase" ) const SchemaBitswap = "bitswap" -var _ WriteProviderRecord = &WriteBitswapProviderRecord{} - -// WriteBitswapProviderRecord is used when we want to add a new provider record that is using bitswap. -type WriteBitswapProviderRecord struct { - Protocol string - Schema string - Signature string - - // this content must be untouched because it is signed and we need to verify it - RawPayload json.RawMessage `json:"Payload"` - Payload BitswapPayload `json:"-"` -} - -type BitswapPayload struct { - Keys []CID - Timestamp *Time - AdvisoryTTL *Duration - ID *peer.ID - Addrs []Multiaddr -} - -func (*WriteBitswapProviderRecord) IsWriteProviderRecord() {} - -type tmpBWPR WriteBitswapProviderRecord - -func (p *WriteBitswapProviderRecord) UnmarshalJSON(b []byte) error { - var bwp tmpBWPR - err := json.Unmarshal(b, &bwp) - if err != nil { - return err - } - - p.Protocol = bwp.Protocol - p.Schema = bwp.Schema - p.Signature = bwp.Signature - p.RawPayload = bwp.RawPayload - - return json.Unmarshal(bwp.RawPayload, &p.Payload) -} - -func (p *WriteBitswapProviderRecord) IsSigned() bool { - return p.Signature != "" -} - -func (p *WriteBitswapProviderRecord) setRawPayload() error { - payloadBytes, err := drjson.MarshalJSONBytes(p.Payload) - if err != nil { - return fmt.Errorf("marshaling bitswap write provider payload: %w", err) - } - - p.RawPayload = payloadBytes - - return nil -} - -func (p *WriteBitswapProviderRecord) Sign(peerID peer.ID, key crypto.PrivKey) error { - if p.IsSigned() { - return errors.New("already signed") - } - - if key == nil { - return errors.New("no key provided") - } - - sid, err := peer.IDFromPrivateKey(key) - if err != nil { - return err - } - if sid != peerID { - return errors.New("not the correct signing key") - } - - err = p.setRawPayload() - if err != nil { - return err - } - hash := sha256.Sum256([]byte(p.RawPayload)) - sig, err := key.Sign(hash[:]) - if err != nil { - return err - } - - sigStr, err := multibase.Encode(multibase.Base64, sig) - if err != nil { - return fmt.Errorf("multibase-encoding signature: %w", err) - } - - p.Signature = sigStr - return nil -} - -func (p *WriteBitswapProviderRecord) Verify() error { - if !p.IsSigned() { - return errors.New("not signed") - } - - if p.Payload.ID == nil { - return errors.New("peer ID must be specified") - } - - // note that we only generate and set the payload if it hasn't already been set - // to allow for passing through the payload untouched if it is already provided - if p.RawPayload == nil { - err := p.setRawPayload() - if err != nil { - return err - } - } - - pk, err := p.Payload.ID.ExtractPublicKey() - if err != nil { - return fmt.Errorf("extracing public key from peer ID: %w", err) - } - - _, sigBytes, err := multibase.Decode(p.Signature) - if err != nil { - return fmt.Errorf("multibase-decoding signature to verify: %w", err) - } - - hash := sha256.Sum256([]byte(p.RawPayload)) - ok, err := pk.Verify(hash[:], sigBytes) - if err != nil { - return fmt.Errorf("verifying hash with signature: %w", err) - } - if !ok { - return errors.New("signature failed to verify") - } - - return nil -} - -var _ ProviderResponse = &WriteBitswapProviderRecordResponse{} - -// WriteBitswapProviderRecordResponse will be returned as a result of WriteBitswapProviderRecord -type WriteBitswapProviderRecordResponse struct { - Protocol string - Schema string - AdvisoryTTL *Duration -} - -func (wbprr *WriteBitswapProviderRecordResponse) GetProtocol() string { - return wbprr.Protocol -} - -func (wbprr *WriteBitswapProviderRecordResponse) GetSchema() string { - return wbprr.Schema -} - -var ( - _ ReadProviderRecord = &ReadBitswapProviderRecord{} - _ ProviderResponse = &ReadBitswapProviderRecord{} -) +var _ ProviderResponse = &ReadBitswapProviderRecord{} // ReadBitswapProviderRecord is a provider result with parameters for bitswap providers type ReadBitswapProviderRecord struct { diff --git a/routing/http/types/provider_unknown.go b/routing/http/types/provider_unknown.go index 915cac481..d398c6988 100644 --- a/routing/http/types/provider_unknown.go +++ b/routing/http/types/provider_unknown.go @@ -6,11 +6,7 @@ import ( "github.com/ipfs/boxo/routing/http/internal/drjson" ) -var ( - _ ReadProviderRecord = &UnknownProviderRecord{} - _ WriteProviderRecord = &UnknownProviderRecord{} - _ ProviderResponse = &UnknownProviderRecord{} -) +var _ ProviderResponse = &UnknownProviderRecord{} // UnknownProviderRecord is used when we cannot parse the provider record using `GetProtocol` type UnknownProviderRecord struct { @@ -28,7 +24,6 @@ func (u *UnknownProviderRecord) GetSchema() string { } func (u *UnknownProviderRecord) IsReadProviderRecord() {} -func (u UnknownProviderRecord) IsWriteProviderRecord() {} func (u *UnknownProviderRecord) UnmarshalJSON(b []byte) error { m := map[string]interface{}{} From 621411906853aa78fa72130c425da16b7d3ffe0f Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Wed, 2 Aug 2023 15:27:35 +0200 Subject: [PATCH 02/17] refactor!: use new Peer schema --- routing/http/client/client.go | 12 ++-- routing/http/client/client_test.go | 26 +++---- routing/http/contentrouter/contentrouter.go | 11 ++- .../http/contentrouter/contentrouter_test.go | 30 ++++---- routing/http/server/server.go | 12 ++-- routing/http/server/server_test.go | 46 ++++++------ .../types/json/{provider.go => providers.go} | 10 +-- routing/http/types/ndjson/provider.go | 36 ---------- routing/http/types/ndjson/providers.go | 36 ++++++++++ routing/http/types/provider.go | 6 -- routing/http/types/provider_bitswap.go | 27 ------- routing/http/types/provider_unknown.go | 58 --------------- routing/http/types/record.go | 6 ++ routing/http/types/record_peer.go | 72 +++++++++++++++++++ routing/http/types/record_unknown.go | 47 ++++++++++++ 15 files changed, 233 insertions(+), 202 deletions(-) rename routing/http/types/json/{provider.go => providers.go} (77%) delete mode 100644 routing/http/types/ndjson/provider.go create mode 100644 routing/http/types/ndjson/providers.go delete mode 100644 routing/http/types/provider.go delete mode 100644 routing/http/types/provider_bitswap.go delete mode 100644 routing/http/types/provider_unknown.go create mode 100644 routing/http/types/record.go create mode 100644 routing/http/types/record_peer.go create mode 100644 routing/http/types/record_unknown.go diff --git a/routing/http/client/client.go b/routing/http/client/client.go index 0c5d3f98f..70170abcf 100644 --- a/routing/http/client/client.go +++ b/routing/http/client/client.go @@ -127,7 +127,7 @@ func (c *measuringIter[T]) Close() error { return c.Iter.Close() } -func (c *client) FindProviders(ctx context.Context, key cid.Cid) (provs iter.ResultIter[types.ProviderResponse], err error) { +func (c *client) FindProviders(ctx context.Context, key cid.Cid) (provs iter.ResultIter[types.Record], err error) { // TODO test measurements m := newMeasurement("FindProviders") @@ -155,7 +155,7 @@ func (c *client) FindProviders(ctx context.Context, key cid.Cid) (provs iter.Res if resp.StatusCode == http.StatusNotFound { resp.Body.Close() m.record(ctx) - return iter.FromSlice[iter.Result[types.ProviderResponse]](nil), nil + return iter.FromSlice[iter.Result[types.Record]](nil), nil } if resp.StatusCode != http.StatusOK { @@ -183,22 +183,22 @@ func (c *client) FindProviders(ctx context.Context, key cid.Cid) (provs iter.Res } }() - var it iter.ResultIter[types.ProviderResponse] + var it iter.ResultIter[types.Record] switch mediaType { case mediaTypeJSON: parsedResp := &jsontypes.ProvidersResponse{} err = json.NewDecoder(resp.Body).Decode(parsedResp) - var sliceIt iter.Iter[types.ProviderResponse] = iter.FromSlice(parsedResp.Providers) + var sliceIt iter.Iter[types.Record] = iter.FromSlice(parsedResp.Providers) it = iter.ToResultIter(sliceIt) case mediaTypeNDJSON: skipBodyClose = true - it = ndjson.NewReadProvidersResponseIter(resp.Body) + it = ndjson.NewProvidersResponseIter(resp.Body) default: logger.Errorw("unknown media type", "MediaType", mediaType, "ContentType", respContentType) return nil, errors.New("unknown content type") } - return &measuringIter[iter.Result[types.ProviderResponse]]{Iter: it, ctx: ctx, m: m}, nil + return &measuringIter[iter.Result[types.Record]]{Iter: it, ctx: ctx, m: m}, nil } func (c *client) FindIPNSRecord(ctx context.Context, name ipns.Name) (*ipns.Record, error) { diff --git a/routing/http/client/client_test.go b/routing/http/client/client_test.go index 35ee207c4..39c8c6f99 100644 --- a/routing/http/client/client_test.go +++ b/routing/http/client/client_test.go @@ -3,6 +3,7 @@ package client import ( "context" "crypto/rand" + "encoding/json" "errors" "net/http" "net/http/httptest" @@ -29,9 +30,9 @@ import ( type mockContentRouter struct{ mock.Mock } -func (m *mockContentRouter) FindProviders(ctx context.Context, key cid.Cid, limit int) (iter.ResultIter[types.ProviderResponse], error) { +func (m *mockContentRouter) FindProviders(ctx context.Context, key cid.Cid, limit int) (iter.ResultIter[types.Record], error) { args := m.Called(ctx, key, limit) - return args.Get(0).(iter.ResultIter[types.ProviderResponse]), args.Error(1) + return args.Get(0).(iter.ResultIter[types.Record]), args.Error(1) } func (m *mockContentRouter) FindIPNSRecord(ctx context.Context, name ipns.Name) (*ipns.Record, error) { @@ -132,13 +133,14 @@ func addrsToDRAddrs(addrs []multiaddr.Multiaddr) (drmas []types.Multiaddr) { return } -func makeBSReadProviderResp() types.ReadBitswapProviderRecord { +func makeBSReadProviderResp() types.PeerRecord { peerID, addrs, _ := makeProviderAndIdentity() - return types.ReadBitswapProviderRecord{ - Protocol: "transport-bitswap", - Schema: types.SchemaBitswap, - ID: &peerID, - Addrs: addrsToDRAddrs(addrs), + return types.PeerRecord{ + Schema: types.SchemaPeer, + ID: &peerID, + Protocols: []string{"transport-bitswap"}, + Addrs: addrsToDRAddrs(addrs), + Extra: map[string]json.RawMessage{}, } } @@ -183,7 +185,7 @@ func (e *osErrContains) errContains(t *testing.T, err error) { func TestClient_FindProviders(t *testing.T) { bsReadProvResp := makeBSReadProviderResp() - bitswapProvs := []iter.Result[types.ProviderResponse]{ + bitswapProvs := []iter.Result[types.Record]{ {Val: &bsReadProvResp}, } @@ -191,13 +193,13 @@ func TestClient_FindProviders(t *testing.T) { name string httpStatusCode int stopServer bool - routerProvs []iter.Result[types.ProviderResponse] + routerProvs []iter.Result[types.Record] routerErr error clientRequiresStreaming bool serverStreamingDisabled bool expErrContains osErrContains - expProvs []iter.Result[types.ProviderResponse] + expProvs []iter.Result[types.Record] expStreamingResponse bool expJSONResponse bool }{ @@ -301,7 +303,7 @@ func TestClient_FindProviders(t *testing.T) { c.expErrContains.errContains(t, err) - provs := iter.ReadAll[iter.Result[types.ProviderResponse]](provsIter) + provs := iter.ReadAll[iter.Result[types.Record]](provsIter) assert.Equal(t, c.expProvs, provs) }) } diff --git a/routing/http/contentrouter/contentrouter.go b/routing/http/contentrouter/contentrouter.go index 0881e5841..ac85017d2 100644 --- a/routing/http/contentrouter/contentrouter.go +++ b/routing/http/contentrouter/contentrouter.go @@ -3,7 +3,6 @@ package contentrouter import ( "context" "reflect" - "time" "github.com/ipfs/boxo/routing/http/types" "github.com/ipfs/boxo/routing/http/types/iter" @@ -18,10 +17,8 @@ import ( var logger = logging.Logger("service/contentrouting") -const ttl = 24 * time.Hour - type Client interface { - FindProviders(ctx context.Context, key cid.Cid) (iter.ResultIter[types.ProviderResponse], error) + FindProviders(ctx context.Context, key cid.Cid) (iter.ResultIter[types.Record], error) } type contentRouter struct { @@ -57,7 +54,7 @@ func (c *contentRouter) Ready() bool { } // readProviderResponses reads bitswap records from the iterator into the given channel, dropping non-bitswap records. -func readProviderResponses(iter iter.ResultIter[types.ProviderResponse], ch chan<- peer.AddrInfo) { +func readProviderResponses(iter iter.ResultIter[types.Record], ch chan<- peer.AddrInfo) { defer close(ch) defer iter.Close() for iter.Next() { @@ -67,8 +64,8 @@ func readProviderResponses(iter iter.ResultIter[types.ProviderResponse], ch chan continue } v := res.Val - if v.GetSchema() == types.SchemaBitswap { - result, ok := v.(*types.ReadBitswapProviderRecord) + if v.GetSchema() == types.SchemaPeer { + result, ok := v.(*types.PeerRecord) if !ok { logger.Errorw( "problem casting find providers result", diff --git a/routing/http/contentrouter/contentrouter_test.go b/routing/http/contentrouter/contentrouter_test.go index 9b2861e15..11050178b 100644 --- a/routing/http/contentrouter/contentrouter_test.go +++ b/routing/http/contentrouter/contentrouter_test.go @@ -16,9 +16,9 @@ import ( type mockClient struct{ mock.Mock } -func (m *mockClient) FindProviders(ctx context.Context, key cid.Cid) (iter.ResultIter[types.ProviderResponse], error) { +func (m *mockClient) FindProviders(ctx context.Context, key cid.Cid) (iter.ResultIter[types.Record], error) { args := m.Called(ctx, key) - return args.Get(0).(iter.ResultIter[types.ProviderResponse]), args.Error(1) + return args.Get(0).(iter.ResultIter[types.Record]), args.Error(1) } func (m *mockClient) Ready(ctx context.Context) (bool, error) { @@ -48,22 +48,22 @@ func TestFindProvidersAsync(t *testing.T) { p1 := peer.ID("peer1") p2 := peer.ID("peer2") - ais := []types.ProviderResponse{ - &types.ReadBitswapProviderRecord{ - Protocol: "transport-bitswap", - Schema: types.SchemaBitswap, - ID: &p1, + ais := []types.Record{ + &types.PeerRecord{ + Schema: types.SchemaPeer, + ID: &p1, + Protocols: []string{"transport-bitswap"}, }, - &types.ReadBitswapProviderRecord{ - Protocol: "transport-bitswap", - Schema: types.SchemaBitswap, - ID: &p2, - }, - &types.UnknownProviderRecord{ - Protocol: "UNKNOWN", + &types.PeerRecord{ + Schema: types.SchemaPeer, + ID: &p2, + Protocols: []string{"transport-bitswap"}, }, + // &types.UnknownRecord{ + // Protocol: "UNKNOWN", + // }, } - aisIter := iter.ToResultIter[types.ProviderResponse](iter.FromSlice(ais)) + aisIter := iter.ToResultIter[types.Record](iter.FromSlice(ais)) client.On("FindProviders", ctx, key).Return(aisIter, nil) diff --git a/routing/http/server/server.go b/routing/http/server/server.go index 6b2591e41..5673a14e0 100644 --- a/routing/http/server/server.go +++ b/routing/http/server/server.go @@ -41,14 +41,14 @@ const ( ) type FindProvidersAsyncResponse struct { - ProviderResponse types.ProviderResponse + ProviderResponse types.Record Error error } type ContentRouter interface { // FindProviders searches for peers who are able to provide a given key. Limit // indicates the maximum amount of results to return. 0 means unbounded. - FindProviders(ctx context.Context, key cid.Cid, limit int) (iter.ResultIter[types.ProviderResponse], error) + FindProviders(ctx context.Context, key cid.Cid, limit int) (iter.ResultIter[types.Record], error) // FindIPNSRecord searches for an [ipns.Record] for the given [ipns.Name]. FindIPNSRecord(ctx context.Context, name ipns.Name) (*ipns.Record, error) @@ -119,7 +119,7 @@ func (s *server) findProviders(w http.ResponseWriter, httpReq *http.Request) { return } - var handlerFunc func(w http.ResponseWriter, provIter iter.ResultIter[types.ProviderResponse]) + var handlerFunc func(w http.ResponseWriter, provIter iter.ResultIter[types.Record]) var supportsNDJSON bool var supportsJSON bool @@ -167,11 +167,11 @@ func (s *server) findProviders(w http.ResponseWriter, httpReq *http.Request) { handlerFunc(w, provIter) } -func (s *server) findProvidersJSON(w http.ResponseWriter, provIter iter.ResultIter[types.ProviderResponse]) { +func (s *server) findProvidersJSON(w http.ResponseWriter, provIter iter.ResultIter[types.Record]) { defer provIter.Close() var ( - providers []types.ProviderResponse + providers []types.Record i int ) @@ -188,7 +188,7 @@ func (s *server) findProvidersJSON(w http.ResponseWriter, provIter iter.ResultIt writeJSONResult(w, "FindProviders", response) } -func (s *server) findProvidersNDJSON(w http.ResponseWriter, provIter iter.ResultIter[types.ProviderResponse]) { +func (s *server) findProvidersNDJSON(w http.ResponseWriter, provIter iter.ResultIter[types.Record]) { defer provIter.Close() w.Header().Set("Content-Type", mediaTypeNDJSON) diff --git a/routing/http/server/server_test.go b/routing/http/server/server_test.go index 04f2961e4..64adb705b 100644 --- a/routing/http/server/server_test.go +++ b/routing/http/server/server_test.go @@ -28,12 +28,11 @@ func TestHeaders(t *testing.T) { t.Cleanup(server.Close) serverAddr := "http://" + server.Listener.Addr().String() - results := iter.FromSlice([]iter.Result[types.ProviderResponse]{ - {Val: &types.ReadBitswapProviderRecord{ - Protocol: "transport-bitswap", - Schema: types.SchemaBitswap, - }}, - }, + results := iter.FromSlice([]iter.Result[types.Record]{ + {Val: &types.PeerRecord{ + Schema: types.SchemaPeer, + Protocols: []string{"transport-bitswap"}, + }}}, ) c := "baeabep4vu3ceru7nerjjbk37sxb7wmftteve4hcosmyolsbsiubw2vr6pqzj6mw7kv6tbn6nqkkldnklbjgm5tzbi4hkpkled4xlcr7xz4bq" @@ -73,20 +72,19 @@ func TestResponse(t *testing.T) { runTest := func(t *testing.T, contentType string, expectedStream bool, expectedBody string) { t.Parallel() - results := iter.FromSlice([]iter.Result[types.ProviderResponse]{ - {Val: &types.ReadBitswapProviderRecord{ - Protocol: "transport-bitswap", - Schema: types.SchemaBitswap, - ID: &pid, - Addrs: []types.Multiaddr{}, + results := iter.FromSlice([]iter.Result[types.Record]{ + {Val: &types.PeerRecord{ + Schema: types.SchemaPeer, + ID: &pid, + Protocols: []string{"transport-bitswap"}, + Addrs: []types.Multiaddr{}, }}, - {Val: &types.ReadBitswapProviderRecord{ - Protocol: "transport-bitswap", - Schema: types.SchemaBitswap, - ID: &pid2, - Addrs: []types.Multiaddr{}, - }}, - }, + {Val: &types.PeerRecord{ + Schema: types.SchemaPeer, + ID: &pid2, + Protocols: []string{"transport-bitswap"}, + Addrs: []types.Multiaddr{}, + }}}, ) router := &mockContentRouter{} @@ -113,15 +111,15 @@ func TestResponse(t *testing.T) { body, err := io.ReadAll(resp.Body) require.NoError(t, err) - require.Equal(t, string(body), expectedBody) + require.Equal(t, expectedBody, string(body)) } t.Run("JSON Response", func(t *testing.T) { - runTest(t, mediaTypeJSON, false, `{"Providers":[{"Protocol":"transport-bitswap","Schema":"bitswap","ID":"12D3KooWM8sovaEGU1bmiWGWAzvs47DEcXKZZTuJnpQyVTkRs2Vn","Addrs":[]},{"Protocol":"transport-bitswap","Schema":"bitswap","ID":"12D3KooWM8sovaEGU1bmiWGWAzvs47DEcXKZZTuJnpQyVTkRs2Vz","Addrs":[]}]}`) + runTest(t, mediaTypeJSON, false, `{"Providers":[{"Addrs":[],"ID":"12D3KooWM8sovaEGU1bmiWGWAzvs47DEcXKZZTuJnpQyVTkRs2Vn","Protocols":["transport-bitswap"],"Schema":"peer"},{"Addrs":[],"ID":"12D3KooWM8sovaEGU1bmiWGWAzvs47DEcXKZZTuJnpQyVTkRs2Vz","Protocols":["transport-bitswap"],"Schema":"peer"}]}`) }) t.Run("NDJSON Response", func(t *testing.T) { - runTest(t, mediaTypeNDJSON, true, `{"Protocol":"transport-bitswap","Schema":"bitswap","ID":"12D3KooWM8sovaEGU1bmiWGWAzvs47DEcXKZZTuJnpQyVTkRs2Vn","Addrs":[]}`+"\n"+`{"Protocol":"transport-bitswap","Schema":"bitswap","ID":"12D3KooWM8sovaEGU1bmiWGWAzvs47DEcXKZZTuJnpQyVTkRs2Vz","Addrs":[]}`+"\n") + runTest(t, mediaTypeNDJSON, true, `{"Addrs":[],"ID":"12D3KooWM8sovaEGU1bmiWGWAzvs47DEcXKZZTuJnpQyVTkRs2Vn","Protocols":["transport-bitswap"],"Schema":"peer"}`+"\n"+`{"Addrs":[],"ID":"12D3KooWM8sovaEGU1bmiWGWAzvs47DEcXKZZTuJnpQyVTkRs2Vz","Protocols":["transport-bitswap"],"Schema":"peer"}`+"\n") }) } @@ -259,9 +257,9 @@ func TestIPNS(t *testing.T) { type mockContentRouter struct{ mock.Mock } -func (m *mockContentRouter) FindProviders(ctx context.Context, key cid.Cid, limit int) (iter.ResultIter[types.ProviderResponse], error) { +func (m *mockContentRouter) FindProviders(ctx context.Context, key cid.Cid, limit int) (iter.ResultIter[types.Record], error) { args := m.Called(ctx, key, limit) - return args.Get(0).(iter.ResultIter[types.ProviderResponse]), args.Error(1) + return args.Get(0).(iter.ResultIter[types.Record]), args.Error(1) } func (m *mockContentRouter) FindIPNSRecord(ctx context.Context, name ipns.Name) (*ipns.Record, error) { diff --git a/routing/http/types/json/provider.go b/routing/http/types/json/providers.go similarity index 77% rename from routing/http/types/json/provider.go rename to routing/http/types/json/providers.go index 3163e1144..bd1cc422c 100644 --- a/routing/http/types/json/provider.go +++ b/routing/http/types/json/providers.go @@ -8,7 +8,7 @@ import ( // ProvidersResponse is the result of a GET Providers request. type ProvidersResponse struct { - Providers []types.ProviderResponse + Providers []types.Record } func (r *ProvidersResponse) UnmarshalJSON(b []byte) error { @@ -19,16 +19,16 @@ func (r *ProvidersResponse) UnmarshalJSON(b []byte) error { } for _, provBytes := range tempFPR.Providers { - var readProv types.UnknownProviderRecord + var readProv types.UnknownRecord err := json.Unmarshal(provBytes, &readProv) if err != nil { return err } switch readProv.Schema { - case types.SchemaBitswap: - var prov types.ReadBitswapProviderRecord - err := json.Unmarshal(readProv.Bytes, &prov) + case types.SchemaPeer: + var prov types.PeerRecord + err := json.Unmarshal(provBytes, &prov) if err != nil { return err } diff --git a/routing/http/types/ndjson/provider.go b/routing/http/types/ndjson/provider.go deleted file mode 100644 index 38e28df9a..000000000 --- a/routing/http/types/ndjson/provider.go +++ /dev/null @@ -1,36 +0,0 @@ -package ndjson - -import ( - "encoding/json" - "io" - - "github.com/ipfs/boxo/routing/http/types" - "github.com/ipfs/boxo/routing/http/types/iter" -) - -// NewReadProvidersResponseIter returns an iterator that reads Read Provider Records from the given reader. -func NewReadProvidersResponseIter(r io.Reader) iter.Iter[iter.Result[types.ProviderResponse]] { - jsonIter := iter.FromReaderJSON[types.UnknownProviderRecord](r) - mapFn := func(upr iter.Result[types.UnknownProviderRecord]) iter.Result[types.ProviderResponse] { - var result iter.Result[types.ProviderResponse] - if upr.Err != nil { - result.Err = upr.Err - return result - } - switch upr.Val.Schema { - case types.SchemaBitswap: - var prov types.ReadBitswapProviderRecord - err := json.Unmarshal(upr.Val.Bytes, &prov) - if err != nil { - result.Err = err - return result - } - result.Val = &prov - default: - result.Val = &upr.Val - } - return result - } - - return iter.Map[iter.Result[types.UnknownProviderRecord]](jsonIter, mapFn) -} diff --git a/routing/http/types/ndjson/providers.go b/routing/http/types/ndjson/providers.go new file mode 100644 index 000000000..6471a6a7b --- /dev/null +++ b/routing/http/types/ndjson/providers.go @@ -0,0 +1,36 @@ +package ndjson + +import ( + "encoding/json" + "io" + + "github.com/ipfs/boxo/routing/http/types" + "github.com/ipfs/boxo/routing/http/types/iter" +) + +// NewProvidersResponseIter returns an iterator that reads [types.Record] from the given [io.Reader]. +func NewProvidersResponseIter(r io.Reader) iter.Iter[iter.Result[types.Record]] { + jsonIter := iter.FromReaderJSON[types.UnknownRecord](r) + mapFn := func(upr iter.Result[types.UnknownRecord]) iter.Result[types.Record] { + var result iter.Result[types.Record] + if upr.Err != nil { + result.Err = upr.Err + return result + } + switch upr.Val.Schema { + case types.SchemaPeer: + var prov types.PeerRecord + err := json.Unmarshal(upr.Val.Bytes, &prov) + if err != nil { + result.Err = err + return result + } + result.Val = &prov + default: + result.Val = &upr.Val + } + return result + } + + return iter.Map[iter.Result[types.UnknownRecord]](jsonIter, mapFn) +} diff --git a/routing/http/types/provider.go b/routing/http/types/provider.go deleted file mode 100644 index aa311d8ed..000000000 --- a/routing/http/types/provider.go +++ /dev/null @@ -1,6 +0,0 @@ -package types - -// ProviderResponse is implemented for any ProviderResponse. It needs to have a Protocol field. -type ProviderResponse interface { - GetSchema() string -} diff --git a/routing/http/types/provider_bitswap.go b/routing/http/types/provider_bitswap.go deleted file mode 100644 index 9add2e39b..000000000 --- a/routing/http/types/provider_bitswap.go +++ /dev/null @@ -1,27 +0,0 @@ -package types - -import ( - "github.com/libp2p/go-libp2p/core/peer" -) - -const SchemaBitswap = "bitswap" - -var _ ProviderResponse = &ReadBitswapProviderRecord{} - -// ReadBitswapProviderRecord is a provider result with parameters for bitswap providers -type ReadBitswapProviderRecord struct { - Protocol string - Schema string - ID *peer.ID - Addrs []Multiaddr -} - -func (rbpr *ReadBitswapProviderRecord) GetProtocol() string { - return rbpr.Protocol -} - -func (rbpr *ReadBitswapProviderRecord) GetSchema() string { - return rbpr.Schema -} - -func (*ReadBitswapProviderRecord) IsReadProviderRecord() {} diff --git a/routing/http/types/provider_unknown.go b/routing/http/types/provider_unknown.go deleted file mode 100644 index d398c6988..000000000 --- a/routing/http/types/provider_unknown.go +++ /dev/null @@ -1,58 +0,0 @@ -package types - -import ( - "encoding/json" - - "github.com/ipfs/boxo/routing/http/internal/drjson" -) - -var _ ProviderResponse = &UnknownProviderRecord{} - -// UnknownProviderRecord is used when we cannot parse the provider record using `GetProtocol` -type UnknownProviderRecord struct { - Protocol string - Schema string - Bytes []byte -} - -func (u *UnknownProviderRecord) GetProtocol() string { - return u.Protocol -} - -func (u *UnknownProviderRecord) GetSchema() string { - return u.Schema -} - -func (u *UnknownProviderRecord) IsReadProviderRecord() {} - -func (u *UnknownProviderRecord) UnmarshalJSON(b []byte) error { - m := map[string]interface{}{} - if err := json.Unmarshal(b, &m); err != nil { - return err - } - - ps, ok := m["Protocol"].(string) - if ok { - u.Protocol = ps - } - schema, ok := m["Schema"].(string) - if ok { - u.Schema = schema - } - - u.Bytes = b - - return nil -} - -func (u UnknownProviderRecord) MarshalJSON() ([]byte, error) { - m := map[string]interface{}{} - err := json.Unmarshal(u.Bytes, &m) - if err != nil { - return nil, err - } - m["Protocol"] = u.Protocol - m["Schema"] = u.Schema - - return drjson.MarshalJSONBytes(m) -} diff --git a/routing/http/types/record.go b/routing/http/types/record.go new file mode 100644 index 000000000..4a734d5f5 --- /dev/null +++ b/routing/http/types/record.go @@ -0,0 +1,6 @@ +package types + +// Record is implemented for any record. +type Record interface { + GetSchema() string +} diff --git a/routing/http/types/record_peer.go b/routing/http/types/record_peer.go new file mode 100644 index 000000000..ff4704eb3 --- /dev/null +++ b/routing/http/types/record_peer.go @@ -0,0 +1,72 @@ +package types + +import ( + "encoding/json" + + "github.com/ipfs/boxo/routing/http/internal/drjson" + "github.com/libp2p/go-libp2p/core/peer" +) + +const SchemaPeer = "peer" + +var _ Record = &PeerRecord{} + +type PeerRecord struct { + Schema string + ID *peer.ID + Addrs []Multiaddr + Protocols []string + + // Extra contains extra fields that were included in the original JSON raw + // message, except for the known ones represented by the remaining fields. + Extra map[string]json.RawMessage +} + +func (pr *PeerRecord) GetSchema() string { + return pr.Schema +} + +func (pr *PeerRecord) UnmarshalJSON(b []byte) error { + // Unmarshal all known fields and assign them. + v := struct { + Schema string + ID *peer.ID + Addrs []Multiaddr + Protocols []string + }{} + err := json.Unmarshal(b, &v) + if err != nil { + return err + } + pr.Schema = v.Schema + pr.ID = v.ID + pr.Addrs = v.Addrs + pr.Protocols = v.Protocols + + // Unmarshal everything into the Extra field and remove the + // known fields to avoid conflictual usages of the struct. + err = json.Unmarshal(b, &pr.Extra) + if err != nil { + return err + } + delete(pr.Extra, "Schema") + delete(pr.Extra, "ID") + delete(pr.Extra, "Addrs") + delete(pr.Extra, "Protocols") + + return nil +} + +func (pr PeerRecord) MarshalJSON() ([]byte, error) { + m := map[string]interface{}{} + if pr.Extra != nil { + for key, val := range pr.Extra { + m[key] = val + } + } + m["Schema"] = pr.Schema + m["ID"] = pr.ID + m["Addrs"] = pr.Addrs + m["Protocols"] = pr.Protocols + return drjson.MarshalJSONBytes(m) +} diff --git a/routing/http/types/record_unknown.go b/routing/http/types/record_unknown.go new file mode 100644 index 000000000..9b2f6f960 --- /dev/null +++ b/routing/http/types/record_unknown.go @@ -0,0 +1,47 @@ +package types + +import ( + "encoding/json" + + "github.com/ipfs/boxo/routing/http/internal/drjson" +) + +var _ Record = &UnknownRecord{} + +type UnknownRecord struct { + Schema string + + // Bytes contains the raw JSON bytes that were used to unmarshal this record. + // This value can be used, for example, to unmarshal de record into a different + // type if Schema is of a known value. + Bytes []byte +} + +func (ur *UnknownRecord) GetSchema() string { + return ur.Schema +} + +func (ur *UnknownRecord) UnmarshalJSON(b []byte) error { + v := struct { + Schema string + }{} + err := json.Unmarshal(b, &v) + if err != nil { + return err + } + ur.Schema = v.Schema + ur.Bytes = b + return nil +} + +func (ur UnknownRecord) MarshalJSON() ([]byte, error) { + m := map[string]interface{}{} + if ur.Bytes != nil { + err := json.Unmarshal(ur.Bytes, &m) + if err != nil { + return nil, err + } + } + m["Schema"] = ur.Schema + return drjson.MarshalJSONBytes(m) +} From 59e661777a2c3df8a200155fe4b23693df560020 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Mon, 7 Aug 2023 16:43:37 +0200 Subject: [PATCH 03/17] refactor!: rename Provide* to Put*, and Find* to Get* --- routing/http/client/client.go | 8 +- routing/http/client/client_test.go | 30 ++++---- routing/http/contentrouter/contentrouter.go | 4 +- .../http/contentrouter/contentrouter_test.go | 6 +- routing/http/server/server.go | 76 +++++++++---------- routing/http/server/server_test.go | 14 ++-- 6 files changed, 68 insertions(+), 70 deletions(-) diff --git a/routing/http/client/client.go b/routing/http/client/client.go index 70170abcf..de78281bf 100644 --- a/routing/http/client/client.go +++ b/routing/http/client/client.go @@ -127,9 +127,9 @@ func (c *measuringIter[T]) Close() error { return c.Iter.Close() } -func (c *client) FindProviders(ctx context.Context, key cid.Cid) (provs iter.ResultIter[types.Record], err error) { +func (c *client) GetProviders(ctx context.Context, key cid.Cid) (provs iter.ResultIter[types.Record], err error) { // TODO test measurements - m := newMeasurement("FindProviders") + m := newMeasurement("GetProviders") url := c.baseURL + "/routing/v1/providers/" + key.String() req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) @@ -201,7 +201,7 @@ func (c *client) FindProviders(ctx context.Context, key cid.Cid) (provs iter.Res return &measuringIter[iter.Result[types.Record]]{Iter: it, ctx: ctx, m: m}, nil } -func (c *client) FindIPNSRecord(ctx context.Context, name ipns.Name) (*ipns.Record, error) { +func (c *client) GetIPNSRecord(ctx context.Context, name ipns.Name) (*ipns.Record, error) { url := c.baseURL + "/routing/v1/ipns/" + name.String() httpReq, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) @@ -239,7 +239,7 @@ func (c *client) FindIPNSRecord(ctx context.Context, name ipns.Name) (*ipns.Reco return record, nil } -func (c *client) ProvideIPNSRecord(ctx context.Context, name ipns.Name, record *ipns.Record) error { +func (c *client) PutIPNSRecord(ctx context.Context, name ipns.Name, record *ipns.Record) error { url := c.baseURL + "/routing/v1/ipns/" + name.String() rawRecord, err := ipns.MarshalRecord(record) diff --git a/routing/http/client/client_test.go b/routing/http/client/client_test.go index 39c8c6f99..a0e3f5689 100644 --- a/routing/http/client/client_test.go +++ b/routing/http/client/client_test.go @@ -30,17 +30,17 @@ import ( type mockContentRouter struct{ mock.Mock } -func (m *mockContentRouter) FindProviders(ctx context.Context, key cid.Cid, limit int) (iter.ResultIter[types.Record], error) { +func (m *mockContentRouter) GetProviders(ctx context.Context, key cid.Cid, limit int) (iter.ResultIter[types.Record], error) { args := m.Called(ctx, key, limit) return args.Get(0).(iter.ResultIter[types.Record]), args.Error(1) } -func (m *mockContentRouter) FindIPNSRecord(ctx context.Context, name ipns.Name) (*ipns.Record, error) { +func (m *mockContentRouter) GetIPNSRecord(ctx context.Context, name ipns.Name) (*ipns.Record, error) { args := m.Called(ctx, name) return args.Get(0).(*ipns.Record), args.Error(1) } -func (m *mockContentRouter) ProvideIPNSRecord(ctx context.Context, name ipns.Name, record *ipns.Record) error { +func (m *mockContentRouter) PutIPNSRecord(ctx context.Context, name ipns.Name, record *ipns.Record) error { args := m.Called(ctx, name, record) return args.Error(0) } @@ -183,7 +183,7 @@ func (e *osErrContains) errContains(t *testing.T, err error) { } } -func TestClient_FindProviders(t *testing.T) { +func TestClient_GetProviders(t *testing.T) { bsReadProvResp := makeBSReadProviderResp() bitswapProvs := []iter.Result[types.Record]{ {Val: &bsReadProvResp}, @@ -294,12 +294,12 @@ func TestClient_FindProviders(t *testing.T) { findProvsIter := iter.FromSlice(c.routerProvs) if c.expStreamingResponse { - router.On("FindProviders", mock.Anything, cid, 0).Return(findProvsIter, c.routerErr) + router.On("GetProviders", mock.Anything, cid, 0).Return(findProvsIter, c.routerErr) } else { - router.On("FindProviders", mock.Anything, cid, 20).Return(findProvsIter, c.routerErr) + router.On("GetProviders", mock.Anything, cid, 20).Return(findProvsIter, c.routerErr) } - provsIter, err := client.FindProviders(ctx, cid) + provsIter, err := client.GetProviders(ctx, cid) c.expErrContains.errContains(t, err) @@ -344,9 +344,9 @@ func TestClient_IPNS(t *testing.T) { client := deps.client router := deps.router - router.On("FindIPNSRecord", mock.Anything, name).Return(nil, errors.New("something wrong happened")) + router.On("GetIPNSRecord", mock.Anything, name).Return(nil, errors.New("something wrong happened")) - receivedRecord, err := client.FindIPNSRecord(context.Background(), name) + receivedRecord, err := client.GetIPNSRecord(context.Background(), name) require.Error(t, err) require.Nil(t, receivedRecord) }) @@ -360,9 +360,9 @@ func TestClient_IPNS(t *testing.T) { client := deps.client router := deps.router - router.On("FindIPNSRecord", mock.Anything, name).Return(record, nil) + router.On("GetIPNSRecord", mock.Anything, name).Return(record, nil) - receivedRecord, err := client.FindIPNSRecord(context.Background(), name) + receivedRecord, err := client.GetIPNSRecord(context.Background(), name) require.NoError(t, err) require.Equal(t, record, receivedRecord) }) @@ -376,9 +376,9 @@ func TestClient_IPNS(t *testing.T) { client := deps.client router := deps.router - router.On("FindIPNSRecord", mock.Anything, name2).Return(record, nil) + router.On("GetIPNSRecord", mock.Anything, name2).Return(record, nil) - receivedRecord, err := client.FindIPNSRecord(context.Background(), name2) + receivedRecord, err := client.GetIPNSRecord(context.Background(), name2) require.Error(t, err) require.Nil(t, receivedRecord) }) @@ -391,9 +391,9 @@ func TestClient_IPNS(t *testing.T) { client := deps.client router := deps.router - router.On("ProvideIPNSRecord", mock.Anything, name, record).Return(nil) + router.On("PutIPNSRecord", mock.Anything, name, record).Return(nil) - err := client.ProvideIPNSRecord(context.Background(), name, record) + err := client.PutIPNSRecord(context.Background(), name, record) require.NoError(t, err) }) } diff --git a/routing/http/contentrouter/contentrouter.go b/routing/http/contentrouter/contentrouter.go index ac85017d2..2a06f1012 100644 --- a/routing/http/contentrouter/contentrouter.go +++ b/routing/http/contentrouter/contentrouter.go @@ -18,7 +18,7 @@ import ( var logger = logging.Logger("service/contentrouting") type Client interface { - FindProviders(ctx context.Context, key cid.Cid) (iter.ResultIter[types.Record], error) + GetProviders(ctx context.Context, key cid.Cid) (iter.ResultIter[types.Record], error) } type contentRouter struct { @@ -89,7 +89,7 @@ func readProviderResponses(iter iter.ResultIter[types.Record], ch chan<- peer.Ad } func (c *contentRouter) FindProvidersAsync(ctx context.Context, key cid.Cid, numResults int) <-chan peer.AddrInfo { - resultsIter, err := c.client.FindProviders(ctx, key) + resultsIter, err := c.client.GetProviders(ctx, key) if err != nil { logger.Warnw("error finding providers", "CID", key, "Error", err) ch := make(chan peer.AddrInfo) diff --git a/routing/http/contentrouter/contentrouter_test.go b/routing/http/contentrouter/contentrouter_test.go index 11050178b..bf996d26d 100644 --- a/routing/http/contentrouter/contentrouter_test.go +++ b/routing/http/contentrouter/contentrouter_test.go @@ -16,7 +16,7 @@ import ( type mockClient struct{ mock.Mock } -func (m *mockClient) FindProviders(ctx context.Context, key cid.Cid) (iter.ResultIter[types.Record], error) { +func (m *mockClient) GetProviders(ctx context.Context, key cid.Cid) (iter.ResultIter[types.Record], error) { args := m.Called(ctx, key) return args.Get(0).(iter.ResultIter[types.Record]), args.Error(1) } @@ -40,7 +40,7 @@ func makeCID() cid.Cid { return c } -func TestFindProvidersAsync(t *testing.T) { +func TestGetProvidersAsync(t *testing.T) { key := makeCID() ctx := context.Background() client := &mockClient{} @@ -65,7 +65,7 @@ func TestFindProvidersAsync(t *testing.T) { } aisIter := iter.ToResultIter[types.Record](iter.FromSlice(ais)) - client.On("FindProviders", ctx, key).Return(aisIter, nil) + client.On("GetProviders", ctx, key).Return(aisIter, nil) aiChan := crc.FindProvidersAsync(ctx, key, 2) diff --git a/routing/http/server/server.go b/routing/http/server/server.go index 5673a14e0..9d7267d3a 100644 --- a/routing/http/server/server.go +++ b/routing/http/server/server.go @@ -36,26 +36,26 @@ const ( var logger = logging.Logger("service/server/delegatedrouting") const ( - FindProvidersPath = "/routing/v1/providers/{cid}" - IPNSPath = "/routing/v1/ipns/{cid}" + GetProvidersPath = "/routing/v1/providers/{cid}" + GetIPNSRecordPath = "/routing/v1/ipns/{cid}" ) -type FindProvidersAsyncResponse struct { +type GetProvidersAsyncResponse struct { ProviderResponse types.Record Error error } type ContentRouter interface { - // FindProviders searches for peers who are able to provide a given key. Limit - // indicates the maximum amount of results to return. 0 means unbounded. - FindProviders(ctx context.Context, key cid.Cid, limit int) (iter.ResultIter[types.Record], error) + // GetProviders searches for peers who are able to provide the given [cid.Cid]. + // Limit indicates the maximum amount of results to return; 0 means unbounded. + GetProviders(ctx context.Context, key cid.Cid, limit int) (iter.ResultIter[types.Record], error) - // FindIPNSRecord searches for an [ipns.Record] for the given [ipns.Name]. - FindIPNSRecord(ctx context.Context, name ipns.Name) (*ipns.Record, error) + // GetIPNSRecord searches for an [ipns.Record] for the given [ipns.Name]. + GetIPNSRecord(ctx context.Context, name ipns.Name) (*ipns.Record, error) - // ProvideIPNSRecord stores the provided [ipns.Record] for the given [ipns.Name]. It is - // guaranteed that the record matches the provided name. - ProvideIPNSRecord(ctx context.Context, name ipns.Name, record *ipns.Record) error + // PutIPNSRecord stores the provided [ipns.Record] for the given [ipns.Name]. + // It is guaranteed that the record matches the provided name. + PutIPNSRecord(ctx context.Context, name ipns.Name, record *ipns.Record) error } type Option func(s *server) @@ -67,16 +67,16 @@ func WithStreamingResultsDisabled() Option { } } -// WithRecordsLimit sets a limit that will be passed to ContentRouter.FindProviders -// for non-streaming requests (application/json). Default is DefaultRecordsLimit. +// WithRecordsLimit sets a limit that will be passed to [ContentRouter.GetProviders] +// for non-streaming requests (application/json). Default is [DefaultRecordsLimit]. func WithRecordsLimit(limit int) Option { return func(s *server) { s.recordsLimit = limit } } -// WithStreamingRecordsLimit sets a limit that will be passed to ContentRouter.FindProviders -// for streaming requests (application/x-ndjson). Default is DefaultStreamingRecordsLimit. +// WithStreamingRecordsLimit sets a limit that will be passed to [ContentRouter.GetProviders] +// for streaming requests (application/x-ndjson). Default is [DefaultStreamingRecordsLimit]. func WithStreamingRecordsLimit(limit int) Option { return func(s *server) { s.streamingRecordsLimit = limit @@ -95,11 +95,9 @@ func Handler(svc ContentRouter, opts ...Option) http.Handler { } r := mux.NewRouter() - r.HandleFunc(FindProvidersPath, server.findProviders).Methods(http.MethodGet) - - r.HandleFunc(IPNSPath, server.getIPNSRecord).Methods(http.MethodGet) - r.HandleFunc(IPNSPath, server.putIPNSRecord).Methods(http.MethodPut) - + r.HandleFunc(GetProvidersPath, server.getProviders).Methods(http.MethodGet) + r.HandleFunc(GetIPNSRecordPath, server.getIPNSRecord).Methods(http.MethodGet) + r.HandleFunc(GetIPNSRecordPath, server.putIPNSRecord).Methods(http.MethodPut) return r } @@ -110,12 +108,12 @@ type server struct { streamingRecordsLimit int } -func (s *server) findProviders(w http.ResponseWriter, httpReq *http.Request) { +func (s *server) getProviders(w http.ResponseWriter, httpReq *http.Request) { vars := mux.Vars(httpReq) cidStr := vars["cid"] cid, err := cid.Decode(cidStr) if err != nil { - writeErr(w, "FindProviders", http.StatusBadRequest, fmt.Errorf("unable to parse CID: %w", err)) + writeErr(w, "GetProviders", http.StatusBadRequest, fmt.Errorf("unable to parse CID: %w", err)) return } @@ -126,14 +124,14 @@ func (s *server) findProviders(w http.ResponseWriter, httpReq *http.Request) { var recordsLimit int acceptHeaders := httpReq.Header.Values("Accept") if len(acceptHeaders) == 0 { - handlerFunc = s.findProvidersJSON + handlerFunc = s.getProvidersJSON recordsLimit = s.recordsLimit } else { for _, acceptHeader := range acceptHeaders { for _, accept := range strings.Split(acceptHeader, ",") { mediaType, _, err := mime.ParseMediaType(accept) if err != nil { - writeErr(w, "FindProviders", http.StatusBadRequest, fmt.Errorf("unable to parse Accept header: %w", err)) + writeErr(w, "GetProviders", http.StatusBadRequest, fmt.Errorf("unable to parse Accept header: %w", err)) return } @@ -147,27 +145,27 @@ func (s *server) findProviders(w http.ResponseWriter, httpReq *http.Request) { } if supportsNDJSON && !s.disableNDJSON { - handlerFunc = s.findProvidersNDJSON + handlerFunc = s.getProvidersNDJSON recordsLimit = s.streamingRecordsLimit } else if supportsJSON { - handlerFunc = s.findProvidersJSON + handlerFunc = s.getProvidersJSON recordsLimit = s.recordsLimit } else { - writeErr(w, "FindProviders", http.StatusBadRequest, errors.New("no supported content types")) + writeErr(w, "GetProviders", http.StatusBadRequest, errors.New("no supported content types")) return } } - provIter, err := s.svc.FindProviders(httpReq.Context(), cid, recordsLimit) + provIter, err := s.svc.GetProviders(httpReq.Context(), cid, recordsLimit) if err != nil { - writeErr(w, "FindProviders", http.StatusInternalServerError, fmt.Errorf("delegate error: %w", err)) + writeErr(w, "GetProviders", http.StatusInternalServerError, fmt.Errorf("delegate error: %w", err)) return } handlerFunc(w, provIter) } -func (s *server) findProvidersJSON(w http.ResponseWriter, provIter iter.ResultIter[types.Record]) { +func (s *server) getProvidersJSON(w http.ResponseWriter, provIter iter.ResultIter[types.Record]) { defer provIter.Close() var ( @@ -178,17 +176,17 @@ func (s *server) findProvidersJSON(w http.ResponseWriter, provIter iter.ResultIt for provIter.Next() { res := provIter.Val() if res.Err != nil { - writeErr(w, "FindProviders", http.StatusInternalServerError, fmt.Errorf("delegate error on result %d: %w", i, res.Err)) + writeErr(w, "GetProviders", http.StatusInternalServerError, fmt.Errorf("delegate error on result %d: %w", i, res.Err)) return } providers = append(providers, res.Val) i++ } response := jsontypes.ProvidersResponse{Providers: providers} - writeJSONResult(w, "FindProviders", response) + writeJSONResult(w, "GetProviders", response) } -func (s *server) findProvidersNDJSON(w http.ResponseWriter, provIter iter.ResultIter[types.Record]) { +func (s *server) getProvidersNDJSON(w http.ResponseWriter, provIter iter.ResultIter[types.Record]) { defer provIter.Close() w.Header().Set("Content-Type", mediaTypeNDJSON) @@ -196,25 +194,25 @@ func (s *server) findProvidersNDJSON(w http.ResponseWriter, provIter iter.Result for provIter.Next() { res := provIter.Val() if res.Err != nil { - logger.Errorw("FindProviders ndjson iterator error", "Error", res.Err) + logger.Errorw("GetProviders ndjson iterator error", "Error", res.Err) return } // don't use an encoder because we can't easily differentiate writer errors from encoding errors b, err := drjson.MarshalJSONBytes(res.Val) if err != nil { - logger.Errorw("FindProviders ndjson marshal error", "Error", err) + logger.Errorw("GetProviders ndjson marshal error", "Error", err) return } _, err = w.Write(b) if err != nil { - logger.Warn("FindProviders ndjson write error", "Error", err) + logger.Warn("GetProviders ndjson write error", "Error", err) return } _, err = w.Write([]byte{'\n'}) if err != nil { - logger.Warn("FindProviders ndjson write error", "Error", err) + logger.Warn("GetProviders ndjson write error", "Error", err) return } @@ -244,7 +242,7 @@ func (s *server) getIPNSRecord(w http.ResponseWriter, r *http.Request) { return } - record, err := s.svc.FindIPNSRecord(r.Context(), name) + record, err := s.svc.GetIPNSRecord(r.Context(), name) if err != nil { writeErr(w, "GetIPNSRecord", http.StatusInternalServerError, fmt.Errorf("delegate error: %w", err)) return @@ -307,7 +305,7 @@ func (s *server) putIPNSRecord(w http.ResponseWriter, r *http.Request) { return } - err = s.svc.ProvideIPNSRecord(r.Context(), name, record) + err = s.svc.PutIPNSRecord(r.Context(), name, record) if err != nil { writeErr(w, "PutIPNSRecord", http.StatusInternalServerError, fmt.Errorf("delegate error: %w", err)) return diff --git a/routing/http/server/server_test.go b/routing/http/server/server_test.go index 64adb705b..f9e9279ed 100644 --- a/routing/http/server/server_test.go +++ b/routing/http/server/server_test.go @@ -39,7 +39,7 @@ func TestHeaders(t *testing.T) { cb, err := cid.Decode(c) require.NoError(t, err) - router.On("FindProviders", mock.Anything, cb, DefaultRecordsLimit). + router.On("GetProviders", mock.Anything, cb, DefaultRecordsLimit). Return(results, nil) resp, err := http.Get(serverAddr + "/routing/v1/providers/" + c) @@ -95,7 +95,7 @@ func TestResponse(t *testing.T) { if expectedStream { limit = DefaultStreamingRecordsLimit } - router.On("FindProviders", mock.Anything, cid, limit).Return(results, nil) + router.On("GetProviders", mock.Anything, cid, limit).Return(results, nil) urlStr := serverAddr + "/routing/v1/providers/" + cidStr req, err := http.NewRequest(http.MethodGet, urlStr, nil) @@ -177,7 +177,7 @@ func TestIPNS(t *testing.T) { require.NoError(t, err) router := &mockContentRouter{} - router.On("FindIPNSRecord", mock.Anything, name1).Return(rec, nil) + router.On("GetIPNSRecord", mock.Anything, name1).Return(rec, nil) resp := makeRequest(t, router, "/routing/v1/ipns/"+name1.String()) require.Equal(t, 200, resp.StatusCode) @@ -210,7 +210,7 @@ func TestIPNS(t *testing.T) { t.Parallel() router := &mockContentRouter{} - router.On("ProvideIPNSRecord", mock.Anything, name1, record1).Return(nil) + router.On("PutIPNSRecord", mock.Anything, name1, record1).Return(nil) server := httptest.NewServer(Handler(router)) t.Cleanup(server.Close) @@ -257,17 +257,17 @@ func TestIPNS(t *testing.T) { type mockContentRouter struct{ mock.Mock } -func (m *mockContentRouter) FindProviders(ctx context.Context, key cid.Cid, limit int) (iter.ResultIter[types.Record], error) { +func (m *mockContentRouter) GetProviders(ctx context.Context, key cid.Cid, limit int) (iter.ResultIter[types.Record], error) { args := m.Called(ctx, key, limit) return args.Get(0).(iter.ResultIter[types.Record]), args.Error(1) } -func (m *mockContentRouter) FindIPNSRecord(ctx context.Context, name ipns.Name) (*ipns.Record, error) { +func (m *mockContentRouter) GetIPNSRecord(ctx context.Context, name ipns.Name) (*ipns.Record, error) { args := m.Called(ctx, name) return args.Get(0).(*ipns.Record), args.Error(1) } -func (m *mockContentRouter) ProvideIPNSRecord(ctx context.Context, name ipns.Name, record *ipns.Record) error { +func (m *mockContentRouter) PutIPNSRecord(ctx context.Context, name ipns.Name, record *ipns.Record) error { args := m.Called(ctx, name, record) return args.Error(0) } From d51de47e3caf9c24b82e68f2669eff3fb8877766 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Mon, 7 Aug 2023 17:25:25 +0200 Subject: [PATCH 04/17] feat: routing/http/server supports /routing/v1/peers --- routing/http/client/client.go | 2 +- routing/http/client/client_test.go | 5 + routing/http/server/server.go | 240 ++++++++++++------ routing/http/server/server_test.go | 123 ++++++++- routing/http/types/iter/iter.go | 20 ++ .../types/json/{providers.go => responses.go} | 22 +- .../types/ndjson/{providers.go => records.go} | 4 +- 7 files changed, 323 insertions(+), 93 deletions(-) rename routing/http/types/json/{providers.go => responses.go} (54%) rename routing/http/types/ndjson/{providers.go => records.go} (79%) diff --git a/routing/http/client/client.go b/routing/http/client/client.go index de78281bf..62a736a61 100644 --- a/routing/http/client/client.go +++ b/routing/http/client/client.go @@ -192,7 +192,7 @@ func (c *client) GetProviders(ctx context.Context, key cid.Cid) (provs iter.Resu it = iter.ToResultIter(sliceIt) case mediaTypeNDJSON: skipBodyClose = true - it = ndjson.NewProvidersResponseIter(resp.Body) + it = ndjson.NewRecordsIter(resp.Body) default: logger.Errorw("unknown media type", "MediaType", mediaType, "ContentType", respContentType) return nil, errors.New("unknown content type") diff --git a/routing/http/client/client_test.go b/routing/http/client/client_test.go index a0e3f5689..4c5de7c59 100644 --- a/routing/http/client/client_test.go +++ b/routing/http/client/client_test.go @@ -35,6 +35,11 @@ func (m *mockContentRouter) GetProviders(ctx context.Context, key cid.Cid, limit return args.Get(0).(iter.ResultIter[types.Record]), args.Error(1) } +func (m *mockContentRouter) GetPeers(ctx context.Context, pid peer.ID, limit int) (iter.ResultIter[types.Record], error) { + args := m.Called(ctx, pid, limit) + return args.Get(0).(iter.ResultIter[types.Record]), args.Error(1) +} + func (m *mockContentRouter) GetIPNSRecord(ctx context.Context, name ipns.Name) (*ipns.Record, error) { args := m.Called(ctx, name) return args.Get(0).(*ipns.Record), args.Error(1) diff --git a/routing/http/server/server.go b/routing/http/server/server.go index 9d7267d3a..93878bd0d 100644 --- a/routing/http/server/server.go +++ b/routing/http/server/server.go @@ -19,6 +19,7 @@ import ( "github.com/ipfs/boxo/routing/http/types/iter" jsontypes "github.com/ipfs/boxo/routing/http/types/json" "github.com/ipfs/go-cid" + "github.com/libp2p/go-libp2p/core/peer" logging "github.com/ipfs/go-log/v2" ) @@ -37,6 +38,7 @@ var logger = logging.Logger("service/server/delegatedrouting") const ( GetProvidersPath = "/routing/v1/providers/{cid}" + GetPeersPath = "/routing/v1/peers/{peer-id}" GetIPNSRecordPath = "/routing/v1/ipns/{cid}" ) @@ -50,6 +52,10 @@ type ContentRouter interface { // Limit indicates the maximum amount of results to return; 0 means unbounded. GetProviders(ctx context.Context, key cid.Cid, limit int) (iter.ResultIter[types.Record], error) + // GetPeers searches for peers who have the provided [peer.ID]. + // Limit indicates the maximum amount of results to return; 0 means unbounded. + GetPeers(ctx context.Context, pid peer.ID, limit int) (iter.ResultIter[types.Record], error) + // GetIPNSRecord searches for an [ipns.Record] for the given [ipns.Name]. GetIPNSRecord(ctx context.Context, name ipns.Name) (*ipns.Record, error) @@ -96,8 +102,10 @@ func Handler(svc ContentRouter, opts ...Option) http.Handler { r := mux.NewRouter() r.HandleFunc(GetProvidersPath, server.getProviders).Methods(http.MethodGet) + r.HandleFunc(GetPeersPath, server.getPeers).Methods(http.MethodGet) r.HandleFunc(GetIPNSRecordPath, server.getIPNSRecord).Methods(http.MethodGet) r.HandleFunc(GetIPNSRecordPath, server.putIPNSRecord).Methods(http.MethodPut) + return r } @@ -108,6 +116,43 @@ type server struct { streamingRecordsLimit int } +func (s *server) detectResponseType(r *http.Request) (string, error) { + var ( + supportsNDJSON bool + supportsJSON bool + + acceptHeaders = r.Header.Values("Accept") + ) + + if len(acceptHeaders) == 0 { + return mediaTypeJSON, nil + } + + for _, acceptHeader := range acceptHeaders { + for _, accept := range strings.Split(acceptHeader, ",") { + mediaType, _, err := mime.ParseMediaType(accept) + if err != nil { + return "", fmt.Errorf("unable to parse Accept header: %w", err) + } + + switch mediaType { + case mediaTypeJSON, mediaTypeWildcard: + supportsJSON = true + case mediaTypeNDJSON: + supportsNDJSON = true + } + } + } + + if supportsNDJSON && !s.disableNDJSON { + return mediaTypeNDJSON, nil + } else if supportsJSON { + return mediaTypeJSON, nil + } else { + return "", errors.New("no supported content types") + } +} + func (s *server) getProviders(w http.ResponseWriter, httpReq *http.Request) { vars := mux.Vars(httpReq) cidStr := vars["cid"] @@ -117,43 +162,23 @@ func (s *server) getProviders(w http.ResponseWriter, httpReq *http.Request) { return } - var handlerFunc func(w http.ResponseWriter, provIter iter.ResultIter[types.Record]) + mediaType, err := s.detectResponseType(httpReq) + if err != nil { + writeErr(w, "GetProviders", http.StatusBadRequest, err) + return + } - var supportsNDJSON bool - var supportsJSON bool - var recordsLimit int - acceptHeaders := httpReq.Header.Values("Accept") - if len(acceptHeaders) == 0 { + var ( + handlerFunc func(w http.ResponseWriter, provIter iter.ResultIter[types.Record]) + recordsLimit int + ) + + if mediaType == mediaTypeNDJSON { + handlerFunc = s.getProvidersNDJSON + recordsLimit = s.streamingRecordsLimit + } else { handlerFunc = s.getProvidersJSON recordsLimit = s.recordsLimit - } else { - for _, acceptHeader := range acceptHeaders { - for _, accept := range strings.Split(acceptHeader, ",") { - mediaType, _, err := mime.ParseMediaType(accept) - if err != nil { - writeErr(w, "GetProviders", http.StatusBadRequest, fmt.Errorf("unable to parse Accept header: %w", err)) - return - } - - switch mediaType { - case mediaTypeJSON, mediaTypeWildcard: - supportsJSON = true - case mediaTypeNDJSON: - supportsNDJSON = true - } - } - } - - if supportsNDJSON && !s.disableNDJSON { - handlerFunc = s.getProvidersNDJSON - recordsLimit = s.streamingRecordsLimit - } else if supportsJSON { - handlerFunc = s.getProvidersJSON - recordsLimit = s.recordsLimit - } else { - writeErr(w, "GetProviders", http.StatusBadRequest, errors.New("no supported content types")) - return - } } provIter, err := s.svc.GetProviders(httpReq.Context(), cid, recordsLimit) @@ -168,58 +193,82 @@ func (s *server) getProviders(w http.ResponseWriter, httpReq *http.Request) { func (s *server) getProvidersJSON(w http.ResponseWriter, provIter iter.ResultIter[types.Record]) { defer provIter.Close() - var ( - providers []types.Record - i int - ) - - for provIter.Next() { - res := provIter.Val() - if res.Err != nil { - writeErr(w, "GetProviders", http.StatusInternalServerError, fmt.Errorf("delegate error on result %d: %w", i, res.Err)) - return - } - providers = append(providers, res.Val) - i++ + providers, err := iter.ReadAllResults(provIter) + if err != nil { + writeErr(w, "GetProviders", http.StatusInternalServerError, fmt.Errorf("delegate error: %w", err)) + return } - response := jsontypes.ProvidersResponse{Providers: providers} - writeJSONResult(w, "GetProviders", response) + + writeJSONResult(w, "GetProviders", jsontypes.ProvidersResponse{ + Providers: providers, + }) } func (s *server) getProvidersNDJSON(w http.ResponseWriter, provIter iter.ResultIter[types.Record]) { - defer provIter.Close() + writeResultsIterNDJSON(w, provIter) +} - w.Header().Set("Content-Type", mediaTypeNDJSON) - w.WriteHeader(http.StatusOK) - for provIter.Next() { - res := provIter.Val() - if res.Err != nil { - logger.Errorw("GetProviders ndjson iterator error", "Error", res.Err) - return - } - // don't use an encoder because we can't easily differentiate writer errors from encoding errors - b, err := drjson.MarshalJSONBytes(res.Val) - if err != nil { - logger.Errorw("GetProviders ndjson marshal error", "Error", err) - return - } +func (s *server) getPeers(w http.ResponseWriter, r *http.Request) { + pidStr := mux.Vars(r)["peer-id"] - _, err = w.Write(b) - if err != nil { - logger.Warn("GetProviders ndjson write error", "Error", err) - return - } + // pidStr must be in CIDv1 format. Therefore, use [cid.Decode]. We can't use + // [peer.Decode] because that would allow other formats to pass through. + cid, err := cid.Decode(pidStr) + if err != nil { + writeErr(w, "GetPeers", http.StatusBadRequest, fmt.Errorf("unable to parse peer ID: %w", err)) + return + } - _, err = w.Write([]byte{'\n'}) - if err != nil { - logger.Warn("GetProviders ndjson write error", "Error", err) - return - } + pid, err := peer.FromCid(cid) + if err != nil { + writeErr(w, "GetPeers", http.StatusBadRequest, fmt.Errorf("unable to parse peer ID: %w", err)) + return + } - if f, ok := w.(http.Flusher); ok { - f.Flush() - } + mediaType, err := s.detectResponseType(r) + if err != nil { + writeErr(w, "GetPeers", http.StatusBadRequest, err) + return + } + + var ( + handlerFunc func(w http.ResponseWriter, provIter iter.ResultIter[types.Record]) + recordsLimit int + ) + + if mediaType == mediaTypeNDJSON { + handlerFunc = s.getPeersNDJSON + recordsLimit = s.streamingRecordsLimit + } else { + handlerFunc = s.getPeersJSON + recordsLimit = s.recordsLimit + } + + provIter, err := s.svc.GetPeers(r.Context(), pid, recordsLimit) + if err != nil { + writeErr(w, "GetPeers", http.StatusInternalServerError, fmt.Errorf("delegate error: %w", err)) + return + } + + handlerFunc(w, provIter) +} + +func (s *server) getPeersJSON(w http.ResponseWriter, peersIter iter.ResultIter[types.Record]) { + defer peersIter.Close() + + peers, err := iter.ReadAllResults(peersIter) + if err != nil { + writeErr(w, "GetPeers", http.StatusInternalServerError, fmt.Errorf("delegate error: %w", err)) + return } + + writeJSONResult(w, "GetPeers", jsontypes.PeersResponse{ + Peers: peers, + }) +} + +func (s *server) getPeersNDJSON(w http.ResponseWriter, peersIter iter.ResultIter[types.Record]) { + writeResultsIterNDJSON(w, peersIter) } func (s *server) getIPNSRecord(w http.ResponseWriter, r *http.Request) { @@ -347,3 +396,40 @@ func writeErr(w http.ResponseWriter, method string, statusCode int, cause error) func logErr(method, msg string, err error) { logger.Infow(msg, "Method", method, "Error", err) } + +func writeResultsIterNDJSON(w http.ResponseWriter, resultIter iter.ResultIter[types.Record]) { + defer resultIter.Close() + + w.Header().Set("Content-Type", mediaTypeNDJSON) + w.WriteHeader(http.StatusOK) + + for resultIter.Next() { + res := resultIter.Val() + if res.Err != nil { + logger.Errorw("ndjson iterator error", "Error", res.Err) + return + } + // don't use an encoder because we can't easily differentiate writer errors from encoding errors + b, err := drjson.MarshalJSONBytes(res.Val) + if err != nil { + logger.Errorw("ndjson marshal error", "Error", err) + return + } + + _, err = w.Write(b) + if err != nil { + logger.Warn("ndjson write error", "Error", err) + return + } + + _, err = w.Write([]byte{'\n'}) + if err != nil { + logger.Warn("ndjson write error", "Error", err) + return + } + + if f, ok := w.(http.Flusher); ok { + f.Flush() + } + } +} diff --git a/routing/http/server/server_test.go b/routing/http/server/server_test.go index f9e9279ed..bd6c9c8b8 100644 --- a/routing/http/server/server_test.go +++ b/routing/http/server/server_test.go @@ -18,6 +18,7 @@ import ( "github.com/ipfs/go-cid" "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/peer" + b58 "github.com/mr-tron/base58/base58" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) @@ -56,7 +57,17 @@ func TestHeaders(t *testing.T) { require.Equal(t, "text/plain; charset=utf-8", header) } -func TestResponse(t *testing.T) { +func makePeerID(t *testing.T) (crypto.PrivKey, peer.ID) { + sk, _, err := crypto.GenerateEd25519Key(rand.Reader) + require.NoError(t, err) + + pid, err := peer.IDFromPrivateKey(sk) + require.NoError(t, err) + + return sk, pid +} + +func TestProviders(t *testing.T) { pidStr := "12D3KooWM8sovaEGU1bmiWGWAzvs47DEcXKZZTuJnpQyVTkRs2Vn" pid2Str := "12D3KooWM8sovaEGU1bmiWGWAzvs47DEcXKZZTuJnpQyVTkRs2Vz" cidStr := "bafkreifjjcie6lypi6ny7amxnfftagclbuxndqonfipmb64f2km2devei4" @@ -123,13 +134,108 @@ func TestResponse(t *testing.T) { }) } -func makeName(t *testing.T) (crypto.PrivKey, ipns.Name) { - sk, _, err := crypto.GenerateEd25519Key(rand.Reader) - require.NoError(t, err) +func TestPeers(t *testing.T) { + makeRequest := func(t *testing.T, router *mockContentRouter, contentType, arg string) *http.Response { + server := httptest.NewServer(Handler(router)) + t.Cleanup(server.Close) + req, err := http.NewRequest(http.MethodGet, "http://"+server.Listener.Addr().String()+"/routing/v1/peers/"+arg, nil) + require.NoError(t, err) + req.Header.Set("Accept", contentType) + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + return resp + } - pid, err := peer.IDFromPrivateKey(sk) - require.NoError(t, err) + t.Run("GET /routing/v1/peers/{non-peer-cid} returns 400", func(t *testing.T) { + t.Parallel() + + router := &mockContentRouter{} + resp := makeRequest(t, router, mediaTypeJSON, "bafkqaaa") + require.Equal(t, 400, resp.StatusCode) + }) + + t.Run("GET /routing/v1/peers/{base58-peer-id} returns 400", func(t *testing.T) { + t.Parallel() + + _, pid := makePeerID(t) + router := &mockContentRouter{} + resp := makeRequest(t, router, mediaTypeJSON, b58.Encode([]byte(pid))) + require.Equal(t, 400, resp.StatusCode) + }) + + t.Run("GET /routing/v1/peers/{cid-peer-id} returns 200 with correct body (JSON)", func(t *testing.T) { + t.Parallel() + + _, pid := makePeerID(t) + results := iter.FromSlice([]iter.Result[types.Record]{ + {Val: &types.PeerRecord{ + Schema: types.SchemaPeer, + ID: &pid, + Protocols: []string{"transport-bitswap", "transport-foo"}, + Addrs: []types.Multiaddr{}, + }}, + {Val: &types.PeerRecord{ + Schema: types.SchemaPeer, + ID: &pid, + Protocols: []string{"transport-foo"}, + Addrs: []types.Multiaddr{}, + }}, + }) + router := &mockContentRouter{} + router.On("GetPeers", mock.Anything, pid, 20).Return(results, nil) + + resp := makeRequest(t, router, mediaTypeJSON, peer.ToCid(pid).String()) + require.Equal(t, 200, resp.StatusCode) + + header := resp.Header.Get("Content-Type") + require.Equal(t, mediaTypeJSON, header) + + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + + expectedBody := `{"Peers":[{"Addrs":[],"ID":"` + pid.String() + `","Protocols":["transport-bitswap","transport-foo"],"Schema":"peer"},{"Addrs":[],"ID":"` + pid.String() + `","Protocols":["transport-foo"],"Schema":"peer"}]}` + require.Equal(t, expectedBody, string(body)) + }) + + t.Run("GET /routing/v1/peers/{cid-peer-id} returns 200 with correct body (NDJSON)", func(t *testing.T) { + t.Parallel() + + _, pid := makePeerID(t) + results := iter.FromSlice([]iter.Result[types.Record]{ + {Val: &types.PeerRecord{ + Schema: types.SchemaPeer, + ID: &pid, + Protocols: []string{"transport-bitswap", "transport-foo"}, + Addrs: []types.Multiaddr{}, + }}, + {Val: &types.PeerRecord{ + Schema: types.SchemaPeer, + ID: &pid, + Protocols: []string{"transport-foo"}, + Addrs: []types.Multiaddr{}, + }}, + }) + + router := &mockContentRouter{} + router.On("GetPeers", mock.Anything, pid, 0).Return(results, nil) + + resp := makeRequest(t, router, mediaTypeNDJSON, peer.ToCid(pid).String()) + require.Equal(t, 200, resp.StatusCode) + + header := resp.Header.Get("Content-Type") + require.Equal(t, mediaTypeNDJSON, header) + + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + + expectedBody := `{"Addrs":[],"ID":"` + pid.String() + `","Protocols":["transport-bitswap","transport-foo"],"Schema":"peer"}` + "\n" + `{"Addrs":[],"ID":"` + pid.String() + `","Protocols":["transport-foo"],"Schema":"peer"}` + "\n" + require.Equal(t, expectedBody, string(body)) + }) +} + +func makeName(t *testing.T) (crypto.PrivKey, ipns.Name) { + sk, pid := makePeerID(t) return sk, ipns.NameFromPeer(pid) } @@ -262,6 +368,11 @@ func (m *mockContentRouter) GetProviders(ctx context.Context, key cid.Cid, limit return args.Get(0).(iter.ResultIter[types.Record]), args.Error(1) } +func (m *mockContentRouter) GetPeers(ctx context.Context, pid peer.ID, limit int) (iter.ResultIter[types.Record], error) { + args := m.Called(ctx, pid, limit) + return args.Get(0).(iter.ResultIter[types.Record]), args.Error(1) +} + func (m *mockContentRouter) GetIPNSRecord(ctx context.Context, name ipns.Name) (*ipns.Record, error) { args := m.Called(ctx, name) return args.Get(0).(*ipns.Record), args.Error(1) diff --git a/routing/http/types/iter/iter.go b/routing/http/types/iter/iter.go index 67c6dde00..2e9801d46 100644 --- a/routing/http/types/iter/iter.go +++ b/routing/http/types/iter/iter.go @@ -1,5 +1,7 @@ package iter +import "fmt" + // Iter is an iterator of arbitrary values. // Iterators are generally not goroutine-safe, to make them safe just read from them into a channel. // For our use cases, these usually have a single reader. This motivates iterators instead of channels, @@ -44,3 +46,21 @@ func ReadAll[T any](iter Iter[T]) []T { } return vs } + +func ReadAllResults[T any](iter ResultIter[T]) ([]T, error) { + var ( + vs []T + i int + ) + + for iter.Next() { + res := iter.Val() + if res.Err != nil { + return nil, fmt.Errorf("error on result %d: %w", i, res.Err) + } + vs = append(vs, res.Val) + i++ + } + + return vs, nil +} diff --git a/routing/http/types/json/providers.go b/routing/http/types/json/responses.go similarity index 54% rename from routing/http/types/json/providers.go rename to routing/http/types/json/responses.go index bd1cc422c..b7b5dcbb9 100644 --- a/routing/http/types/json/providers.go +++ b/routing/http/types/json/responses.go @@ -8,17 +8,25 @@ import ( // ProvidersResponse is the result of a GET Providers request. type ProvidersResponse struct { - Providers []types.Record + Providers RecordsArray } -func (r *ProvidersResponse) UnmarshalJSON(b []byte) error { - var tempFPR struct{ Providers []json.RawMessage } - err := json.Unmarshal(b, &tempFPR) +// PeersResponse is the result of a GET Peers request. +type PeersResponse struct { + Peers RecordsArray +} + +// RecordsArray is an array of [types.Record] +type RecordsArray []types.Record + +func (r *RecordsArray) UnmarshalJSON(b []byte) error { + var tempRecords []json.RawMessage + err := json.Unmarshal(b, &tempRecords) if err != nil { return err } - for _, provBytes := range tempFPR.Providers { + for _, provBytes := range tempRecords { var readProv types.UnknownRecord err := json.Unmarshal(provBytes, &readProv) if err != nil { @@ -32,9 +40,9 @@ func (r *ProvidersResponse) UnmarshalJSON(b []byte) error { if err != nil { return err } - r.Providers = append(r.Providers, &prov) + *r = append(*r, &prov) default: - r.Providers = append(r.Providers, &readProv) + *r = append(*r, &readProv) } } diff --git a/routing/http/types/ndjson/providers.go b/routing/http/types/ndjson/records.go similarity index 79% rename from routing/http/types/ndjson/providers.go rename to routing/http/types/ndjson/records.go index 6471a6a7b..32e9199a1 100644 --- a/routing/http/types/ndjson/providers.go +++ b/routing/http/types/ndjson/records.go @@ -8,8 +8,8 @@ import ( "github.com/ipfs/boxo/routing/http/types/iter" ) -// NewProvidersResponseIter returns an iterator that reads [types.Record] from the given [io.Reader]. -func NewProvidersResponseIter(r io.Reader) iter.Iter[iter.Result[types.Record]] { +// NewRecordsIter returns an iterator that reads [types.Record] from the given [io.Reader]. +func NewRecordsIter(r io.Reader) iter.Iter[iter.Result[types.Record]] { jsonIter := iter.FromReaderJSON[types.UnknownRecord](r) mapFn := func(upr iter.Result[types.UnknownRecord]) iter.Result[types.Record] { var result iter.Result[types.Record] From 1456de73d30a2b9f47067da6e565a9aab36c4074 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Tue, 8 Aug 2023 14:35:39 +0200 Subject: [PATCH 05/17] feat: routing/http/client supports /routing/v1/peers --- routing/http/client/client.go | 75 +++++++++++++- routing/http/client/client_test.go | 159 ++++++++++++++++++++++++++--- 2 files changed, 217 insertions(+), 17 deletions(-) diff --git a/routing/http/client/client.go b/routing/http/client/client.go index 62a736a61..76a42f8e9 100644 --- a/routing/http/client/client.go +++ b/routing/http/client/client.go @@ -127,7 +127,7 @@ func (c *measuringIter[T]) Close() error { return c.Iter.Close() } -func (c *client) GetProviders(ctx context.Context, key cid.Cid) (provs iter.ResultIter[types.Record], err error) { +func (c *client) GetProviders(ctx context.Context, key cid.Cid) (providers iter.ResultIter[types.Record], err error) { // TODO test measurements m := newMeasurement("GetProviders") @@ -201,6 +201,79 @@ func (c *client) GetProviders(ctx context.Context, key cid.Cid) (provs iter.Resu return &measuringIter[iter.Result[types.Record]]{Iter: it, ctx: ctx, m: m}, nil } +func (c *client) GetPeers(ctx context.Context, pid peer.ID) (peers iter.ResultIter[types.Record], err error) { + m := newMeasurement("GetPeers") + + url := c.baseURL + "/routing/v1/peers/" + peer.ToCid(pid).String() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + req.Header.Set("Accept", c.accepts) + + m.host = req.Host + + start := c.clock.Now() + resp, err := c.httpClient.Do(req) + + m.err = err + m.latency = c.clock.Since(start) + + if err != nil { + m.record(ctx) + return nil, err + } + + m.statusCode = resp.StatusCode + if resp.StatusCode == http.StatusNotFound { + resp.Body.Close() + m.record(ctx) + return iter.FromSlice[iter.Result[types.Record]](nil), nil + } + + if resp.StatusCode != http.StatusOK { + err := httpError(resp.StatusCode, resp.Body) + resp.Body.Close() + m.record(ctx) + return nil, err + } + + respContentType := resp.Header.Get("Content-Type") + mediaType, _, err := mime.ParseMediaType(respContentType) + if err != nil { + resp.Body.Close() + m.err = err + m.record(ctx) + return nil, fmt.Errorf("parsing Content-Type: %w", err) + } + + m.mediaType = mediaType + + var skipBodyClose bool + defer func() { + if !skipBodyClose { + resp.Body.Close() + } + }() + + var it iter.ResultIter[types.Record] + switch mediaType { + case mediaTypeJSON: + parsedResp := &jsontypes.PeersResponse{} + err = json.NewDecoder(resp.Body).Decode(parsedResp) + var sliceIt iter.Iter[types.Record] = iter.FromSlice(parsedResp.Peers) + it = iter.ToResultIter(sliceIt) + case mediaTypeNDJSON: + skipBodyClose = true + it = ndjson.NewRecordsIter(resp.Body) + default: + logger.Errorw("unknown media type", "MediaType", mediaType, "ContentType", respContentType) + return nil, errors.New("unknown content type") + } + + return &measuringIter[iter.Result[types.Record]]{Iter: it, ctx: ctx, m: m}, nil +} + func (c *client) GetIPNSRecord(ctx context.Context, name ipns.Name) (*ipns.Record, error) { url := c.baseURL + "/routing/v1/ipns/" + name.String() diff --git a/routing/http/client/client_test.go b/routing/http/client/client_test.go index 4c5de7c59..8bc23a2ae 100644 --- a/routing/http/client/client_test.go +++ b/routing/http/client/client_test.go @@ -138,7 +138,7 @@ func addrsToDRAddrs(addrs []multiaddr.Multiaddr) (drmas []types.Multiaddr) { return } -func makeBSReadProviderResp() types.PeerRecord { +func makePeerRecord() types.PeerRecord { peerID, addrs, _ := makeProviderAndIdentity() return types.PeerRecord{ Schema: types.SchemaPeer, @@ -189,7 +189,7 @@ func (e *osErrContains) errContains(t *testing.T, err error) { } func TestClient_GetProviders(t *testing.T) { - bsReadProvResp := makeBSReadProviderResp() + bsReadProvResp := makePeerRecord() bitswapProvs := []iter.Result[types.Record]{ {Val: &bsReadProvResp}, } @@ -198,26 +198,26 @@ func TestClient_GetProviders(t *testing.T) { name string httpStatusCode int stopServer bool - routerProvs []iter.Result[types.Record] + routerResult []iter.Result[types.Record] routerErr error clientRequiresStreaming bool serverStreamingDisabled bool expErrContains osErrContains - expProvs []iter.Result[types.Record] + expResult []iter.Result[types.Record] expStreamingResponse bool expJSONResponse bool }{ { name: "happy case", - routerProvs: bitswapProvs, - expProvs: bitswapProvs, + routerResult: bitswapProvs, + expResult: bitswapProvs, expStreamingResponse: true, }, { name: "server doesn't support streaming", - routerProvs: bitswapProvs, - expProvs: bitswapProvs, + routerResult: bitswapProvs, + expResult: bitswapProvs, serverStreamingDisabled: true, expJSONResponse: true, }, @@ -243,7 +243,7 @@ func TestClient_GetProviders(t *testing.T) { { name: "returns no providers if the HTTP server returns a 404 respones", httpStatusCode: 404, - expProvs: nil, + expResult: nil, }, } for _, c := range cases { @@ -268,6 +268,7 @@ func TestClient_GetProviders(t *testing.T) { assert.Equal(t, mediaTypeNDJSON, r.Header.Get("Content-Type")) }) } + if c.expJSONResponse { onRespReceived = append(onRespReceived, func(r *http.Response) { assert.Equal(t, mediaTypeJSON, r.Header.Get("Content-Type")) @@ -296,20 +297,146 @@ func TestClient_GetProviders(t *testing.T) { } cid := makeCID() - findProvsIter := iter.FromSlice(c.routerProvs) - + routerResultIter := iter.FromSlice(c.routerResult) if c.expStreamingResponse { - router.On("GetProviders", mock.Anything, cid, 0).Return(findProvsIter, c.routerErr) + router.On("GetProviders", mock.Anything, cid, 0).Return(routerResultIter, c.routerErr) } else { - router.On("GetProviders", mock.Anything, cid, 20).Return(findProvsIter, c.routerErr) + router.On("GetProviders", mock.Anything, cid, 20).Return(routerResultIter, c.routerErr) } - provsIter, err := client.GetProviders(ctx, cid) + resultIter, err := client.GetProviders(ctx, cid) + c.expErrContains.errContains(t, err) + + results := iter.ReadAll[iter.Result[types.Record]](resultIter) + assert.Equal(t, c.expResult, results) + }) + } +} + +func TestClient_GetPeers(t *testing.T) { + peerRecord := makePeerRecord() + peerRecords := []iter.Result[types.Record]{ + {Val: &peerRecord}, + } + pid := *peerRecord.ID + + cases := []struct { + name string + httpStatusCode int + stopServer bool + routerResult []iter.Result[types.Record] + routerErr error + clientRequiresStreaming bool + serverStreamingDisabled bool + + expErrContains osErrContains + expResult []iter.Result[types.Record] + expStreamingResponse bool + expJSONResponse bool + }{ + { + name: "happy case", + routerResult: peerRecords, + expResult: peerRecords, + expStreamingResponse: true, + }, + { + name: "server doesn't support streaming", + routerResult: peerRecords, + expResult: peerRecords, + serverStreamingDisabled: true, + expJSONResponse: true, + }, + { + name: "client requires streaming but server doesn't support it", + serverStreamingDisabled: true, + clientRequiresStreaming: true, + expErrContains: osErrContains{expContains: "HTTP error with StatusCode=400: no supported content types"}, + }, + { + name: "returns an error if there's a non-200 response", + httpStatusCode: 500, + expErrContains: osErrContains{expContains: "HTTP error with StatusCode=500"}, + }, + { + name: "returns an error if the HTTP client returns a non-HTTP error", + stopServer: true, + expErrContains: osErrContains{ + expContains: "connect: connection refused", + expContainsWin: "connectex: No connection could be made because the target machine actively refused it.", + }, + }, + { + name: "returns no providers if the HTTP server returns a 404 respones", + httpStatusCode: 404, + expResult: nil, + }, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + var ( + clientOpts []Option + serverOpts []server.Option + onRespReceived []func(*http.Response) + onReqReceived []func(*http.Request) + ) + + if c.serverStreamingDisabled { + serverOpts = append(serverOpts, server.WithStreamingResultsDisabled()) + } + + if c.clientRequiresStreaming { + clientOpts = append(clientOpts, WithStreamResultsRequired()) + onReqReceived = append(onReqReceived, func(r *http.Request) { + assert.Equal(t, mediaTypeNDJSON, r.Header.Get("Accept")) + }) + } + + if c.expStreamingResponse { + onRespReceived = append(onRespReceived, func(r *http.Response) { + assert.Equal(t, mediaTypeNDJSON, r.Header.Get("Content-Type")) + }) + } + + if c.expJSONResponse { + onRespReceived = append(onRespReceived, func(r *http.Response) { + assert.Equal(t, mediaTypeJSON, r.Header.Get("Content-Type")) + }) + } + + deps := makeTestDeps(t, clientOpts, serverOpts) + + deps.recordingHTTPClient.f = append(deps.recordingHTTPClient.f, onRespReceived...) + deps.recordingHandler.f = append(deps.recordingHandler.f, onReqReceived...) + + client := deps.client + router := deps.router + + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + if c.httpStatusCode != 0 { + deps.server.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(c.httpStatusCode) + }) + } + + if c.stopServer { + deps.server.Close() + } + + routerResultIter := iter.FromSlice(c.routerResult) + if c.expStreamingResponse { + router.On("GetPeers", mock.Anything, pid, 0).Return(routerResultIter, c.routerErr) + } else { + router.On("GetPeers", mock.Anything, pid, 20).Return(routerResultIter, c.routerErr) + } + resultIter, err := client.GetPeers(ctx, pid) c.expErrContains.errContains(t, err) - provs := iter.ReadAll[iter.Result[types.Record]](provsIter) - assert.Equal(t, c.expProvs, provs) + results := iter.ReadAll[iter.Result[types.Record]](resultIter) + assert.Equal(t, c.expResult, results) }) } } From 02149da0bd9f6a1d0217a22d9ee5f48131f54131 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Tue, 8 Aug 2023 14:55:50 +0200 Subject: [PATCH 06/17] docs: misc & changelog --- CHANGELOG.md | 15 +++++++++++ routing/http/README.md | 23 +++-------------- routing/http/contentrouter/contentrouter.go | 5 ++-- .../http/contentrouter/contentrouter_test.go | 6 ++--- routing/http/server/server.go | 25 ++++++++++--------- 5 files changed, 38 insertions(+), 36 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7afe79f54..c1dfa35eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,14 +34,29 @@ The following emojis are used to highlight certain changes: reducing globals efforts. * The `blockservice` and `provider` packages has been updated to accommodate for changes in `verifycid`. +* ✨ The `routing/http` package has received the following additions: + * Supports Delegated IPNS as per [IPIP-379](https://specs.ipfs.tech/ipips/ipip-0379/). + * Supports Delegated Peer Routing as per [IPIP-417](https://github.com/ipfs/specs/pull/417). ### Changed * 🛠 `blockservice.New` now accepts a variadic of func options following the [Functional Options pattern](https://www.sohamkamani.com/golang/options-pattern/). +* 🛠 The `routing/http` package has suffered the following modifications: + * Client `FindProviders` has been renamed to `GetProviders`. Similarly, the + required function names in the server `ContentRouter` have also been updated + for higher consistency with the remaining code and the specifications. + * Many types regarding response types were updated to conform to the updated + Peer Schema discussed in [IPIP-417](https://github.com/ipfs/specs/pull/417). ### Removed +* 🛠 The `routing/http` package has suffered the following removals: + * Server and client no longer support the `Provide*` methods for content routing. + These methods did not conform to any specification, as it is still being worked + out in [IPIP-378](https://github.com/ipfs/specs/pull/378). + * Server no longer exports `FindProvidersPath` and `ProvidePath`. + ### Fixed - HTTP Gateway API: Not having a block will result in a 5xx error rather than 404 diff --git a/routing/http/README.md b/routing/http/README.md index 65650ed50..0f0281f8f 100644 --- a/routing/http/README.md +++ b/routing/http/README.md @@ -1,24 +1,9 @@ -go-delegated-routing +Routing V1 Server and Client ======================= -> Delegated routing Client and Server over Reframe RPC - -This package provides delegated routing implementation in Go: -- Client (for IPFS nodes like [Kubo](https://github.com/ipfs/kubo/blob/master/docs/config.md#routingrouters-parameters)), -- Server (for public indexers such as https://cid.contact) +> Delegated Routing V1 Server and Client over HTTP API. ## Documentation -- Go docs: https://pkg.go.dev/github.com/ipfs/boxo/routing/http/ - -## Lead Maintainer - -🦗🎶 - -## Contributing - -Contributions are welcome! This repository is part of the IPFS project and therefore governed by our [contributing guidelines](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md). - -## License - -[SPDX-License-Identifier: Apache-2.0 OR MIT](LICENSE.md) \ No newline at end of file +- Go Documentation: https://pkg.go.dev/github.com/ipfs/boxo/routing/http +- Routing V1 Specification: https://specs.ipfs.tech/routing/http-routing-v1/ diff --git a/routing/http/contentrouter/contentrouter.go b/routing/http/contentrouter/contentrouter.go index 2a06f1012..a3a9e2024 100644 --- a/routing/http/contentrouter/contentrouter.go +++ b/routing/http/contentrouter/contentrouter.go @@ -15,7 +15,7 @@ import ( "github.com/multiformats/go-multihash" ) -var logger = logging.Logger("service/contentrouting") +var logger = logging.Logger("routing/http/contentrouter") type Client interface { GetProviders(ctx context.Context, key cid.Cid) (iter.ResultIter[types.Record], error) @@ -27,6 +27,7 @@ type contentRouter struct { var _ routing.ContentRouting = (*contentRouter)(nil) var _ routinghelpers.ProvideManyRouter = (*contentRouter)(nil) +var _ routinghelpers.ReadyAbleRouter = (*contentRouter)(nil) type option func(c *contentRouter) @@ -48,7 +49,7 @@ func (c *contentRouter) ProvideMany(ctx context.Context, mhKeys []multihash.Mult return routing.ErrNotSupported } -// Ready is part of the existing `ProvideMany` interface. +// Ready is part of the existing [routing.ReadyAbleRouter] interface. func (c *contentRouter) Ready() bool { return true } diff --git a/routing/http/contentrouter/contentrouter_test.go b/routing/http/contentrouter/contentrouter_test.go index bf996d26d..286ce8b30 100644 --- a/routing/http/contentrouter/contentrouter_test.go +++ b/routing/http/contentrouter/contentrouter_test.go @@ -59,9 +59,9 @@ func TestGetProvidersAsync(t *testing.T) { ID: &p2, Protocols: []string{"transport-bitswap"}, }, - // &types.UnknownRecord{ - // Protocol: "UNKNOWN", - // }, + &types.UnknownRecord{ + Schema: "UNKNOWN", + }, } aisIter := iter.ToResultIter[types.Record](iter.FromSlice(ais)) diff --git a/routing/http/server/server.go b/routing/http/server/server.go index 93878bd0d..7737fecf1 100644 --- a/routing/http/server/server.go +++ b/routing/http/server/server.go @@ -34,12 +34,12 @@ const ( DefaultStreamingRecordsLimit = 0 ) -var logger = logging.Logger("service/server/delegatedrouting") +var logger = logging.Logger("routing/http/server") const ( - GetProvidersPath = "/routing/v1/providers/{cid}" - GetPeersPath = "/routing/v1/peers/{peer-id}" - GetIPNSRecordPath = "/routing/v1/ipns/{cid}" + getProvidersPath = "/routing/v1/providers/{cid}" + getPeersPath = "/routing/v1/peers/{peer-id}" + getIPNSRecordPath = "/routing/v1/ipns/{cid}" ) type GetProvidersAsyncResponse struct { @@ -50,7 +50,7 @@ type GetProvidersAsyncResponse struct { type ContentRouter interface { // GetProviders searches for peers who are able to provide the given [cid.Cid]. // Limit indicates the maximum amount of results to return; 0 means unbounded. - GetProviders(ctx context.Context, key cid.Cid, limit int) (iter.ResultIter[types.Record], error) + GetProviders(ctx context.Context, cid cid.Cid, limit int) (iter.ResultIter[types.Record], error) // GetPeers searches for peers who have the provided [peer.ID]. // Limit indicates the maximum amount of results to return; 0 means unbounded. @@ -74,7 +74,8 @@ func WithStreamingResultsDisabled() Option { } // WithRecordsLimit sets a limit that will be passed to [ContentRouter.GetProviders] -// for non-streaming requests (application/json). Default is [DefaultRecordsLimit]. +// and [ContentRouter.GetPeers] for non-streaming requests (application/json). +// Default is [DefaultRecordsLimit]. func WithRecordsLimit(limit int) Option { return func(s *server) { s.recordsLimit = limit @@ -82,7 +83,8 @@ func WithRecordsLimit(limit int) Option { } // WithStreamingRecordsLimit sets a limit that will be passed to [ContentRouter.GetProviders] -// for streaming requests (application/x-ndjson). Default is [DefaultStreamingRecordsLimit]. +// and [ContentRouter.GetPeers] for streaming requests (application/x-ndjson). +// Default is [DefaultStreamingRecordsLimit]. func WithStreamingRecordsLimit(limit int) Option { return func(s *server) { s.streamingRecordsLimit = limit @@ -101,11 +103,10 @@ func Handler(svc ContentRouter, opts ...Option) http.Handler { } r := mux.NewRouter() - r.HandleFunc(GetProvidersPath, server.getProviders).Methods(http.MethodGet) - r.HandleFunc(GetPeersPath, server.getPeers).Methods(http.MethodGet) - r.HandleFunc(GetIPNSRecordPath, server.getIPNSRecord).Methods(http.MethodGet) - r.HandleFunc(GetIPNSRecordPath, server.putIPNSRecord).Methods(http.MethodPut) - + r.HandleFunc(getProvidersPath, server.getProviders).Methods(http.MethodGet) + r.HandleFunc(getPeersPath, server.getPeers).Methods(http.MethodGet) + r.HandleFunc(getIPNSRecordPath, server.getIPNSRecord).Methods(http.MethodGet) + r.HandleFunc(getIPNSRecordPath, server.putIPNSRecord).Methods(http.MethodPut) return r } From 872eb0a64bbf543690702393cafe2c323bb7a8e9 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Tue, 8 Aug 2023 16:15:12 +0200 Subject: [PATCH 07/17] feat: contentrouter implements PeerRouting and ValueStore --- routing/http/contentrouter/contentrouter.go | 115 ++++++++++++++++ .../http/contentrouter/contentrouter_test.go | 130 +++++++++++++++++- 2 files changed, 243 insertions(+), 2 deletions(-) diff --git a/routing/http/contentrouter/contentrouter.go b/routing/http/contentrouter/contentrouter.go index a3a9e2024..255963f24 100644 --- a/routing/http/contentrouter/contentrouter.go +++ b/routing/http/contentrouter/contentrouter.go @@ -3,7 +3,9 @@ package contentrouter import ( "context" "reflect" + "strings" + "github.com/ipfs/boxo/ipns" "github.com/ipfs/boxo/routing/http/types" "github.com/ipfs/boxo/routing/http/types/iter" "github.com/ipfs/go-cid" @@ -19,6 +21,9 @@ var logger = logging.Logger("routing/http/contentrouter") type Client interface { GetProviders(ctx context.Context, key cid.Cid) (iter.ResultIter[types.Record], error) + GetPeers(ctx context.Context, pid peer.ID) (peers iter.ResultIter[types.Record], err error) + GetIPNSRecord(ctx context.Context, name ipns.Name) (*ipns.Record, error) + PutIPNSRecord(ctx context.Context, name ipns.Name, record *ipns.Record) error } type contentRouter struct { @@ -26,6 +31,8 @@ type contentRouter struct { } var _ routing.ContentRouting = (*contentRouter)(nil) +var _ routing.PeerRouting = (*contentRouter)(nil) +var _ routing.ValueStore = (*contentRouter)(nil) var _ routinghelpers.ProvideManyRouter = (*contentRouter)(nil) var _ routinghelpers.ReadyAbleRouter = (*contentRouter)(nil) @@ -101,3 +108,111 @@ func (c *contentRouter) FindProvidersAsync(ctx context.Context, key cid.Cid, num go readProviderResponses(resultsIter, ch) return ch } + +func (c *contentRouter) FindPeer(ctx context.Context, pid peer.ID) (peer.AddrInfo, error) { + iter, err := c.client.GetPeers(ctx, pid) + if err != nil { + return peer.AddrInfo{}, err + } + defer iter.Close() + + for iter.Next() { + res := iter.Val() + if res.Err != nil { + logger.Warnw("error iterating provider responses: %s", res.Err) + continue + } + v := res.Val + if v.GetSchema() == types.SchemaPeer { + result, ok := v.(*types.PeerRecord) + if !ok { + logger.Errorw( + "problem casting find providers result", + "Schema", v.GetSchema(), + "Type", reflect.TypeOf(v).String(), + ) + continue + } + + var addrs []multiaddr.Multiaddr + for _, a := range result.Addrs { + addrs = append(addrs, a.Multiaddr) + } + + return peer.AddrInfo{ + ID: *result.ID, + Addrs: addrs, + }, nil + } + } + + return peer.AddrInfo{}, err +} + +func (c *contentRouter) PutValue(ctx context.Context, key string, data []byte, opts ...routing.Option) error { + if !strings.HasPrefix(key, "/ipns/") { + return routing.ErrNotSupported + } + + name, err := ipns.NameFromRoutingKey([]byte(key)) + if err != nil { + return err + } + + record, err := ipns.UnmarshalRecord(data) + if err != nil { + return err + } + + return c.client.PutIPNSRecord(ctx, name, record) +} + +func (c *contentRouter) GetValue(ctx context.Context, key string, opts ...routing.Option) ([]byte, error) { + if !strings.HasPrefix(key, "/ipns/") { + return nil, routing.ErrNotSupported + } + + name, err := ipns.NameFromRoutingKey([]byte(key)) + if err != nil { + return nil, err + } + + record, err := c.client.GetIPNSRecord(ctx, name) + if err != nil { + return nil, err + } + + return ipns.MarshalRecord(record) +} + +func (c *contentRouter) SearchValue(ctx context.Context, key string, opts ...routing.Option) (<-chan []byte, error) { + if !strings.HasPrefix(key, "/ipns/") { + return nil, routing.ErrNotSupported + } + + name, err := ipns.NameFromRoutingKey([]byte(key)) + if err != nil { + return nil, err + } + + ch := make(chan []byte) + + go func() { + record, err := c.client.GetIPNSRecord(ctx, name) + if err != nil { + close(ch) + return + } + + raw, err := ipns.MarshalRecord(record) + if err != nil { + close(ch) + return + } + + ch <- raw + close(ch) + }() + + return ch, nil +} diff --git a/routing/http/contentrouter/contentrouter_test.go b/routing/http/contentrouter/contentrouter_test.go index 286ce8b30..e1d52fc22 100644 --- a/routing/http/contentrouter/contentrouter_test.go +++ b/routing/http/contentrouter/contentrouter_test.go @@ -4,11 +4,17 @@ import ( "context" "crypto/rand" "testing" + "time" + "github.com/ipfs/boxo/coreiface/path" + "github.com/ipfs/boxo/ipns" + ipfspath "github.com/ipfs/boxo/path" "github.com/ipfs/boxo/routing/http/types" "github.com/ipfs/boxo/routing/http/types/iter" "github.com/ipfs/go-cid" + "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/peer" + "github.com/libp2p/go-libp2p/core/routing" "github.com/multiformats/go-multihash" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" @@ -20,11 +26,22 @@ func (m *mockClient) GetProviders(ctx context.Context, key cid.Cid) (iter.Result args := m.Called(ctx, key) return args.Get(0).(iter.ResultIter[types.Record]), args.Error(1) } - +func (m *mockClient) GetPeers(ctx context.Context, pid peer.ID) (iter.ResultIter[types.Record], error) { + args := m.Called(ctx, pid) + return args.Get(0).(iter.ResultIter[types.Record]), args.Error(1) +} func (m *mockClient) Ready(ctx context.Context) (bool, error) { args := m.Called(ctx) return args.Bool(0), args.Error(1) } +func (m *mockClient) GetIPNSRecord(ctx context.Context, name ipns.Name) (*ipns.Record, error) { + args := m.Called(ctx, name) + return args.Get(0).(*ipns.Record), args.Error(1) +} +func (m *mockClient) PutIPNSRecord(ctx context.Context, name ipns.Name, record *ipns.Record) error { + args := m.Called(ctx, name, record) + return args.Error(0) +} func makeCID() cid.Cid { buf := make([]byte, 63) @@ -40,7 +57,7 @@ func makeCID() cid.Cid { return c } -func TestGetProvidersAsync(t *testing.T) { +func TestFindProvidersAsync(t *testing.T) { key := makeCID() ctx := context.Background() client := &mockClient{} @@ -81,3 +98,112 @@ func TestGetProvidersAsync(t *testing.T) { require.Equal(t, expected, actualAIs) } + +func TestFindPeer(t *testing.T) { + ctx := context.Background() + client := &mockClient{} + crc := NewContentRoutingClient(client) + + p1 := peer.ID("peer1") + ais := []types.Record{ + &types.UnknownRecord{ + Schema: "Unknown", + }, + &types.PeerRecord{ + Schema: types.SchemaPeer, + ID: &p1, + Protocols: []string{"transport-bitswap"}, + }, + } + aisIter := iter.ToResultIter[types.Record](iter.FromSlice(ais)) + + client.On("GetPeers", ctx, p1).Return(aisIter, nil) + + peer, err := crc.FindPeer(ctx, p1) + require.NoError(t, err) + require.Equal(t, peer.ID, p1) +} + +func makeName(t *testing.T) (crypto.PrivKey, ipns.Name) { + sk, _, err := crypto.GenerateEd25519Key(rand.Reader) + require.NoError(t, err) + + pid, err := peer.IDFromPrivateKey(sk) + require.NoError(t, err) + + return sk, ipns.NameFromPeer(pid) +} + +func makeIPNSRecord(t *testing.T, sk crypto.PrivKey, opts ...ipns.Option) (*ipns.Record, []byte) { + cid, err := cid.Decode("bafkreifjjcie6lypi6ny7amxnfftagclbuxndqonfipmb64f2km2devei4") + require.NoError(t, err) + + path := path.IpfsPath(cid) + eol := time.Now().Add(time.Hour * 48) + ttl := time.Second * 20 + + record, err := ipns.NewRecord(sk, ipfspath.FromString(path.String()), 1, eol, ttl, opts...) + require.NoError(t, err) + + rawRecord, err := ipns.MarshalRecord(record) + require.NoError(t, err) + + return record, rawRecord +} + +func TestGetValue(t *testing.T) { + ctx := context.Background() + client := &mockClient{} + crc := NewContentRoutingClient(client) + + t.Run("Fail On Unsupported Key", func(t *testing.T) { + v, err := crc.GetValue(ctx, "/something/unsupported") + require.Nil(t, v) + require.ErrorIs(t, err, routing.ErrNotSupported) + }) + + t.Run("Fail On Invalid IPNS Name", func(t *testing.T) { + v, err := crc.GetValue(ctx, "/ipns/invalid") + require.Nil(t, v) + require.Error(t, err) + }) + + t.Run("Succeeds On Valid IPNS Name", func(t *testing.T) { + sk, name := makeName(t) + rec, rawRec := makeIPNSRecord(t, sk) + client.On("GetIPNSRecord", ctx, name).Return(rec, nil) + v, err := crc.GetValue(ctx, string(name.RoutingKey())) + require.NoError(t, err) + require.Equal(t, rawRec, v) + }) +} + +func TestPutValue(t *testing.T) { + ctx := context.Background() + client := &mockClient{} + crc := NewContentRoutingClient(client) + + sk, name := makeName(t) + _, rawRec := makeIPNSRecord(t, sk) + + t.Run("Fail On Unsupported Key", func(t *testing.T) { + err := crc.PutValue(ctx, "/something/unsupported", rawRec) + require.ErrorIs(t, err, routing.ErrNotSupported) + }) + + t.Run("Fail On Invalid IPNS Name", func(t *testing.T) { + err := crc.PutValue(ctx, "/ipns/invalid", rawRec) + require.Error(t, err) + }) + + t.Run("Fail On Invalid IPNS Record", func(t *testing.T) { + err := crc.PutValue(ctx, string(name.RoutingKey()), []byte("gibberish")) + require.Error(t, err) + }) + + t.Run("Succeeds On Valid IPNS Name & Record", func(t *testing.T) { + client.On("PutIPNSRecord", ctx, name, mock.Anything).Return(nil) + err := crc.PutValue(ctx, string(name.RoutingKey()), rawRec) + require.NoError(t, err) + }) +} From afc89b90ce56a93b06bd27c95225896c5b530b6f Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Tue, 22 Aug 2023 11:30:40 +0200 Subject: [PATCH 08/17] refactor: rename Get/Put to Find/Provide --- CHANGELOG.md | 2 +- gateway/blocks_backend.go | 2 +- gateway/gateway.go | 4 +- gateway/gateway_test.go | 4 +- gateway/handler_ipns_record.go | 2 +- gateway/metrics.go | 6 +- gateway/utilities_test.go | 2 +- routing/http/client/client.go | 12 +- routing/http/client/client_test.go | 40 +++--- routing/http/contentrouter/contentrouter.go | 18 +-- .../http/contentrouter/contentrouter_test.go | 16 +-- routing/http/server/server.go | 118 +++++++++--------- routing/http/server/server_test.go | 20 +-- 13 files changed, 123 insertions(+), 123 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c1dfa35eb..88e2b6837 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,7 +43,7 @@ The following emojis are used to highlight certain changes: * 🛠 `blockservice.New` now accepts a variadic of func options following the [Functional Options pattern](https://www.sohamkamani.com/golang/options-pattern/). * 🛠 The `routing/http` package has suffered the following modifications: - * Client `FindProviders` has been renamed to `GetProviders`. Similarly, the + * Client `FindProviders` has been renamed to `FindProviders`. Similarly, the required function names in the server `ContentRouter` have also been updated for higher consistency with the remaining code and the specifications. * Many types regarding response types were updated to conform to the updated diff --git a/gateway/blocks_backend.go b/gateway/blocks_backend.go index 208c92062..3509991f6 100644 --- a/gateway/blocks_backend.go +++ b/gateway/blocks_backend.go @@ -586,7 +586,7 @@ func (bb *BlocksBackend) ResolveMutable(ctx context.Context, p ifacepath.Path) ( } } -func (bb *BlocksBackend) GetIPNSRecord(ctx context.Context, c cid.Cid) ([]byte, error) { +func (bb *BlocksBackend) FindIPNS(ctx context.Context, c cid.Cid) ([]byte, error) { if bb.routing == nil { return nil, NewErrorStatusCode(errors.New("IPNS Record responses are not supported by this gateway"), http.StatusNotImplemented) } diff --git a/gateway/gateway.go b/gateway/gateway.go index 780691a45..d5d00d484 100644 --- a/gateway/gateway.go +++ b/gateway/gateway.go @@ -330,9 +330,9 @@ type IPFSBackend interface { // IsCached returns whether or not the path exists locally. IsCached(context.Context, path.Path) bool - // GetIPNSRecord retrieves the best IPNS record for a given CID (libp2p-key) + // FindIPNS retrieves the best IPNS record for a given CID (libp2p-key) // from the routing system. - GetIPNSRecord(context.Context, cid.Cid) ([]byte, error) + FindIPNS(context.Context, cid.Cid) ([]byte, error) // ResolveMutable takes a mutable path and resolves it into an immutable one. This means recursively resolving any // DNSLink or IPNS records. diff --git a/gateway/gateway_test.go b/gateway/gateway_test.go index 98996acb3..4e7c76998 100644 --- a/gateway/gateway_test.go +++ b/gateway/gateway_test.go @@ -731,7 +731,7 @@ func (mb *errorMockBackend) ResolveMutable(ctx context.Context, path ipath.Path) return ImmutablePath{}, mb.err } -func (mb *errorMockBackend) GetIPNSRecord(ctx context.Context, c cid.Cid) ([]byte, error) { +func (mb *errorMockBackend) FindIPNS(ctx context.Context, c cid.Cid) ([]byte, error) { return nil, mb.err } @@ -815,7 +815,7 @@ func (mb *panicMockBackend) ResolveMutable(ctx context.Context, p ipath.Path) (I panic("i am panicking") } -func (mb *panicMockBackend) GetIPNSRecord(ctx context.Context, c cid.Cid) ([]byte, error) { +func (mb *panicMockBackend) FindIPNS(ctx context.Context, c cid.Cid) ([]byte, error) { panic("i am panicking") } diff --git a/gateway/handler_ipns_record.go b/gateway/handler_ipns_record.go index b077fa59a..41e8e6268 100644 --- a/gateway/handler_ipns_record.go +++ b/gateway/handler_ipns_record.go @@ -41,7 +41,7 @@ func (i *handler) serveIpnsRecord(ctx context.Context, w http.ResponseWriter, r return false } - rawRecord, err := i.backend.GetIPNSRecord(ctx, c) + rawRecord, err := i.backend.FindIPNS(ctx, c) if err != nil { i.webError(w, r, err, http.StatusInternalServerError) return false diff --git a/gateway/metrics.go b/gateway/metrics.go index 69e81425f..f6f71a486 100644 --- a/gateway/metrics.go +++ b/gateway/metrics.go @@ -143,13 +143,13 @@ func (b *ipfsBackendWithMetrics) IsCached(ctx context.Context, path path.Path) b return bln } -func (b *ipfsBackendWithMetrics) GetIPNSRecord(ctx context.Context, cid cid.Cid) ([]byte, error) { +func (b *ipfsBackendWithMetrics) FindIPNS(ctx context.Context, cid cid.Cid) ([]byte, error) { begin := time.Now() - name := "IPFSBackend.GetIPNSRecord" + name := "IPFSBackend.FindIPNS" ctx, span := spanTrace(ctx, name, trace.WithAttributes(attribute.String("cid", cid.String()))) defer span.End() - r, err := b.backend.GetIPNSRecord(ctx, cid) + r, err := b.backend.FindIPNS(ctx, cid) b.updateBackendCallMetric(name, err, begin) return r, err diff --git a/gateway/utilities_test.go b/gateway/utilities_test.go index 27ba43a14..6c59f133d 100644 --- a/gateway/utilities_test.go +++ b/gateway/utilities_test.go @@ -157,7 +157,7 @@ func (mb *mockBackend) ResolveMutable(ctx context.Context, p ipath.Path) (Immuta return mb.gw.ResolveMutable(ctx, p) } -func (mb *mockBackend) GetIPNSRecord(ctx context.Context, c cid.Cid) ([]byte, error) { +func (mb *mockBackend) FindIPNS(ctx context.Context, c cid.Cid) ([]byte, error) { return nil, routing.ErrNotSupported } diff --git a/routing/http/client/client.go b/routing/http/client/client.go index 76a42f8e9..bed3603d9 100644 --- a/routing/http/client/client.go +++ b/routing/http/client/client.go @@ -127,9 +127,9 @@ func (c *measuringIter[T]) Close() error { return c.Iter.Close() } -func (c *client) GetProviders(ctx context.Context, key cid.Cid) (providers iter.ResultIter[types.Record], err error) { +func (c *client) FindProviders(ctx context.Context, key cid.Cid) (providers iter.ResultIter[types.Record], err error) { // TODO test measurements - m := newMeasurement("GetProviders") + m := newMeasurement("FindProviders") url := c.baseURL + "/routing/v1/providers/" + key.String() req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) @@ -201,8 +201,8 @@ func (c *client) GetProviders(ctx context.Context, key cid.Cid) (providers iter. return &measuringIter[iter.Result[types.Record]]{Iter: it, ctx: ctx, m: m}, nil } -func (c *client) GetPeers(ctx context.Context, pid peer.ID) (peers iter.ResultIter[types.Record], err error) { - m := newMeasurement("GetPeers") +func (c *client) FindPeers(ctx context.Context, pid peer.ID) (peers iter.ResultIter[types.Record], err error) { + m := newMeasurement("FindPeers") url := c.baseURL + "/routing/v1/peers/" + peer.ToCid(pid).String() req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) @@ -274,7 +274,7 @@ func (c *client) GetPeers(ctx context.Context, pid peer.ID) (peers iter.ResultIt return &measuringIter[iter.Result[types.Record]]{Iter: it, ctx: ctx, m: m}, nil } -func (c *client) GetIPNSRecord(ctx context.Context, name ipns.Name) (*ipns.Record, error) { +func (c *client) FindIPNS(ctx context.Context, name ipns.Name) (*ipns.Record, error) { url := c.baseURL + "/routing/v1/ipns/" + name.String() httpReq, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) @@ -312,7 +312,7 @@ func (c *client) GetIPNSRecord(ctx context.Context, name ipns.Name) (*ipns.Recor return record, nil } -func (c *client) PutIPNSRecord(ctx context.Context, name ipns.Name, record *ipns.Record) error { +func (c *client) ProvideIPNS(ctx context.Context, name ipns.Name, record *ipns.Record) error { url := c.baseURL + "/routing/v1/ipns/" + name.String() rawRecord, err := ipns.MarshalRecord(record) diff --git a/routing/http/client/client_test.go b/routing/http/client/client_test.go index 8bc23a2ae..374e69221 100644 --- a/routing/http/client/client_test.go +++ b/routing/http/client/client_test.go @@ -30,22 +30,22 @@ import ( type mockContentRouter struct{ mock.Mock } -func (m *mockContentRouter) GetProviders(ctx context.Context, key cid.Cid, limit int) (iter.ResultIter[types.Record], error) { +func (m *mockContentRouter) FindProviders(ctx context.Context, key cid.Cid, limit int) (iter.ResultIter[types.Record], error) { args := m.Called(ctx, key, limit) return args.Get(0).(iter.ResultIter[types.Record]), args.Error(1) } -func (m *mockContentRouter) GetPeers(ctx context.Context, pid peer.ID, limit int) (iter.ResultIter[types.Record], error) { +func (m *mockContentRouter) FindPeers(ctx context.Context, pid peer.ID, limit int) (iter.ResultIter[types.Record], error) { args := m.Called(ctx, pid, limit) return args.Get(0).(iter.ResultIter[types.Record]), args.Error(1) } -func (m *mockContentRouter) GetIPNSRecord(ctx context.Context, name ipns.Name) (*ipns.Record, error) { +func (m *mockContentRouter) FindIPNS(ctx context.Context, name ipns.Name) (*ipns.Record, error) { args := m.Called(ctx, name) return args.Get(0).(*ipns.Record), args.Error(1) } -func (m *mockContentRouter) PutIPNSRecord(ctx context.Context, name ipns.Name, record *ipns.Record) error { +func (m *mockContentRouter) ProvideIPNS(ctx context.Context, name ipns.Name, record *ipns.Record) error { args := m.Called(ctx, name, record) return args.Error(0) } @@ -188,7 +188,7 @@ func (e *osErrContains) errContains(t *testing.T, err error) { } } -func TestClient_GetProviders(t *testing.T) { +func TestClient_FindProviders(t *testing.T) { bsReadProvResp := makePeerRecord() bitswapProvs := []iter.Result[types.Record]{ {Val: &bsReadProvResp}, @@ -299,12 +299,12 @@ func TestClient_GetProviders(t *testing.T) { routerResultIter := iter.FromSlice(c.routerResult) if c.expStreamingResponse { - router.On("GetProviders", mock.Anything, cid, 0).Return(routerResultIter, c.routerErr) + router.On("FindProviders", mock.Anything, cid, 0).Return(routerResultIter, c.routerErr) } else { - router.On("GetProviders", mock.Anything, cid, 20).Return(routerResultIter, c.routerErr) + router.On("FindProviders", mock.Anything, cid, 20).Return(routerResultIter, c.routerErr) } - resultIter, err := client.GetProviders(ctx, cid) + resultIter, err := client.FindProviders(ctx, cid) c.expErrContains.errContains(t, err) results := iter.ReadAll[iter.Result[types.Record]](resultIter) @@ -313,7 +313,7 @@ func TestClient_GetProviders(t *testing.T) { } } -func TestClient_GetPeers(t *testing.T) { +func TestClient_FindPeers(t *testing.T) { peerRecord := makePeerRecord() peerRecords := []iter.Result[types.Record]{ {Val: &peerRecord}, @@ -427,12 +427,12 @@ func TestClient_GetPeers(t *testing.T) { routerResultIter := iter.FromSlice(c.routerResult) if c.expStreamingResponse { - router.On("GetPeers", mock.Anything, pid, 0).Return(routerResultIter, c.routerErr) + router.On("FindPeers", mock.Anything, pid, 0).Return(routerResultIter, c.routerErr) } else { - router.On("GetPeers", mock.Anything, pid, 20).Return(routerResultIter, c.routerErr) + router.On("FindPeers", mock.Anything, pid, 20).Return(routerResultIter, c.routerErr) } - resultIter, err := client.GetPeers(ctx, pid) + resultIter, err := client.FindPeers(ctx, pid) c.expErrContains.errContains(t, err) results := iter.ReadAll[iter.Result[types.Record]](resultIter) @@ -476,9 +476,9 @@ func TestClient_IPNS(t *testing.T) { client := deps.client router := deps.router - router.On("GetIPNSRecord", mock.Anything, name).Return(nil, errors.New("something wrong happened")) + router.On("FindIPNS", mock.Anything, name).Return(nil, errors.New("something wrong happened")) - receivedRecord, err := client.GetIPNSRecord(context.Background(), name) + receivedRecord, err := client.FindIPNS(context.Background(), name) require.Error(t, err) require.Nil(t, receivedRecord) }) @@ -492,9 +492,9 @@ func TestClient_IPNS(t *testing.T) { client := deps.client router := deps.router - router.On("GetIPNSRecord", mock.Anything, name).Return(record, nil) + router.On("FindIPNS", mock.Anything, name).Return(record, nil) - receivedRecord, err := client.GetIPNSRecord(context.Background(), name) + receivedRecord, err := client.FindIPNS(context.Background(), name) require.NoError(t, err) require.Equal(t, record, receivedRecord) }) @@ -508,9 +508,9 @@ func TestClient_IPNS(t *testing.T) { client := deps.client router := deps.router - router.On("GetIPNSRecord", mock.Anything, name2).Return(record, nil) + router.On("FindIPNS", mock.Anything, name2).Return(record, nil) - receivedRecord, err := client.GetIPNSRecord(context.Background(), name2) + receivedRecord, err := client.FindIPNS(context.Background(), name2) require.Error(t, err) require.Nil(t, receivedRecord) }) @@ -523,9 +523,9 @@ func TestClient_IPNS(t *testing.T) { client := deps.client router := deps.router - router.On("PutIPNSRecord", mock.Anything, name, record).Return(nil) + router.On("ProvideIPNS", mock.Anything, name, record).Return(nil) - err := client.PutIPNSRecord(context.Background(), name, record) + err := client.ProvideIPNS(context.Background(), name, record) require.NoError(t, err) }) } diff --git a/routing/http/contentrouter/contentrouter.go b/routing/http/contentrouter/contentrouter.go index 255963f24..353398329 100644 --- a/routing/http/contentrouter/contentrouter.go +++ b/routing/http/contentrouter/contentrouter.go @@ -20,10 +20,10 @@ import ( var logger = logging.Logger("routing/http/contentrouter") type Client interface { - GetProviders(ctx context.Context, key cid.Cid) (iter.ResultIter[types.Record], error) - GetPeers(ctx context.Context, pid peer.ID) (peers iter.ResultIter[types.Record], err error) - GetIPNSRecord(ctx context.Context, name ipns.Name) (*ipns.Record, error) - PutIPNSRecord(ctx context.Context, name ipns.Name, record *ipns.Record) error + FindProviders(ctx context.Context, key cid.Cid) (iter.ResultIter[types.Record], error) + FindPeers(ctx context.Context, pid peer.ID) (peers iter.ResultIter[types.Record], err error) + FindIPNS(ctx context.Context, name ipns.Name) (*ipns.Record, error) + ProvideIPNS(ctx context.Context, name ipns.Name, record *ipns.Record) error } type contentRouter struct { @@ -97,7 +97,7 @@ func readProviderResponses(iter iter.ResultIter[types.Record], ch chan<- peer.Ad } func (c *contentRouter) FindProvidersAsync(ctx context.Context, key cid.Cid, numResults int) <-chan peer.AddrInfo { - resultsIter, err := c.client.GetProviders(ctx, key) + resultsIter, err := c.client.FindProviders(ctx, key) if err != nil { logger.Warnw("error finding providers", "CID", key, "Error", err) ch := make(chan peer.AddrInfo) @@ -110,7 +110,7 @@ func (c *contentRouter) FindProvidersAsync(ctx context.Context, key cid.Cid, num } func (c *contentRouter) FindPeer(ctx context.Context, pid peer.ID) (peer.AddrInfo, error) { - iter, err := c.client.GetPeers(ctx, pid) + iter, err := c.client.FindPeers(ctx, pid) if err != nil { return peer.AddrInfo{}, err } @@ -164,7 +164,7 @@ func (c *contentRouter) PutValue(ctx context.Context, key string, data []byte, o return err } - return c.client.PutIPNSRecord(ctx, name, record) + return c.client.ProvideIPNS(ctx, name, record) } func (c *contentRouter) GetValue(ctx context.Context, key string, opts ...routing.Option) ([]byte, error) { @@ -177,7 +177,7 @@ func (c *contentRouter) GetValue(ctx context.Context, key string, opts ...routin return nil, err } - record, err := c.client.GetIPNSRecord(ctx, name) + record, err := c.client.FindIPNS(ctx, name) if err != nil { return nil, err } @@ -198,7 +198,7 @@ func (c *contentRouter) SearchValue(ctx context.Context, key string, opts ...rou ch := make(chan []byte) go func() { - record, err := c.client.GetIPNSRecord(ctx, name) + record, err := c.client.FindIPNS(ctx, name) if err != nil { close(ch) return diff --git a/routing/http/contentrouter/contentrouter_test.go b/routing/http/contentrouter/contentrouter_test.go index e1d52fc22..f11075af1 100644 --- a/routing/http/contentrouter/contentrouter_test.go +++ b/routing/http/contentrouter/contentrouter_test.go @@ -22,11 +22,11 @@ import ( type mockClient struct{ mock.Mock } -func (m *mockClient) GetProviders(ctx context.Context, key cid.Cid) (iter.ResultIter[types.Record], error) { +func (m *mockClient) FindProviders(ctx context.Context, key cid.Cid) (iter.ResultIter[types.Record], error) { args := m.Called(ctx, key) return args.Get(0).(iter.ResultIter[types.Record]), args.Error(1) } -func (m *mockClient) GetPeers(ctx context.Context, pid peer.ID) (iter.ResultIter[types.Record], error) { +func (m *mockClient) FindPeers(ctx context.Context, pid peer.ID) (iter.ResultIter[types.Record], error) { args := m.Called(ctx, pid) return args.Get(0).(iter.ResultIter[types.Record]), args.Error(1) } @@ -34,11 +34,11 @@ func (m *mockClient) Ready(ctx context.Context) (bool, error) { args := m.Called(ctx) return args.Bool(0), args.Error(1) } -func (m *mockClient) GetIPNSRecord(ctx context.Context, name ipns.Name) (*ipns.Record, error) { +func (m *mockClient) FindIPNS(ctx context.Context, name ipns.Name) (*ipns.Record, error) { args := m.Called(ctx, name) return args.Get(0).(*ipns.Record), args.Error(1) } -func (m *mockClient) PutIPNSRecord(ctx context.Context, name ipns.Name, record *ipns.Record) error { +func (m *mockClient) ProvideIPNS(ctx context.Context, name ipns.Name, record *ipns.Record) error { args := m.Called(ctx, name, record) return args.Error(0) } @@ -82,7 +82,7 @@ func TestFindProvidersAsync(t *testing.T) { } aisIter := iter.ToResultIter[types.Record](iter.FromSlice(ais)) - client.On("GetProviders", ctx, key).Return(aisIter, nil) + client.On("FindProviders", ctx, key).Return(aisIter, nil) aiChan := crc.FindProvidersAsync(ctx, key, 2) @@ -117,7 +117,7 @@ func TestFindPeer(t *testing.T) { } aisIter := iter.ToResultIter[types.Record](iter.FromSlice(ais)) - client.On("GetPeers", ctx, p1).Return(aisIter, nil) + client.On("FindPeers", ctx, p1).Return(aisIter, nil) peer, err := crc.FindPeer(ctx, p1) require.NoError(t, err) @@ -171,7 +171,7 @@ func TestGetValue(t *testing.T) { t.Run("Succeeds On Valid IPNS Name", func(t *testing.T) { sk, name := makeName(t) rec, rawRec := makeIPNSRecord(t, sk) - client.On("GetIPNSRecord", ctx, name).Return(rec, nil) + client.On("FindIPNS", ctx, name).Return(rec, nil) v, err := crc.GetValue(ctx, string(name.RoutingKey())) require.NoError(t, err) require.Equal(t, rawRec, v) @@ -202,7 +202,7 @@ func TestPutValue(t *testing.T) { }) t.Run("Succeeds On Valid IPNS Name & Record", func(t *testing.T) { - client.On("PutIPNSRecord", ctx, name, mock.Anything).Return(nil) + client.On("ProvideIPNS", ctx, name, mock.Anything).Return(nil) err := crc.PutValue(ctx, string(name.RoutingKey()), rawRec) require.NoError(t, err) }) diff --git a/routing/http/server/server.go b/routing/http/server/server.go index 7737fecf1..aa91bab01 100644 --- a/routing/http/server/server.go +++ b/routing/http/server/server.go @@ -37,31 +37,31 @@ const ( var logger = logging.Logger("routing/http/server") const ( - getProvidersPath = "/routing/v1/providers/{cid}" - getPeersPath = "/routing/v1/peers/{peer-id}" - getIPNSRecordPath = "/routing/v1/ipns/{cid}" + findProvidersPath = "/routing/v1/providers/{cid}" + findPeersPath = "/routing/v1/peers/{peer-id}" + findIPNSPath = "/routing/v1/ipns/{cid}" ) -type GetProvidersAsyncResponse struct { +type FindProvidersAsyncResponse struct { ProviderResponse types.Record Error error } type ContentRouter interface { - // GetProviders searches for peers who are able to provide the given [cid.Cid]. + // FindProviders searches for peers who are able to provide the given [cid.Cid]. // Limit indicates the maximum amount of results to return; 0 means unbounded. - GetProviders(ctx context.Context, cid cid.Cid, limit int) (iter.ResultIter[types.Record], error) + FindProviders(ctx context.Context, cid cid.Cid, limit int) (iter.ResultIter[types.Record], error) - // GetPeers searches for peers who have the provided [peer.ID]. + // FindPeers searches for peers who have the provided [peer.ID]. // Limit indicates the maximum amount of results to return; 0 means unbounded. - GetPeers(ctx context.Context, pid peer.ID, limit int) (iter.ResultIter[types.Record], error) + FindPeers(ctx context.Context, pid peer.ID, limit int) (iter.ResultIter[types.Record], error) - // GetIPNSRecord searches for an [ipns.Record] for the given [ipns.Name]. - GetIPNSRecord(ctx context.Context, name ipns.Name) (*ipns.Record, error) + // FindIPNS searches for an [ipns.Record] for the given [ipns.Name]. + FindIPNS(ctx context.Context, name ipns.Name) (*ipns.Record, error) - // PutIPNSRecord stores the provided [ipns.Record] for the given [ipns.Name]. + // ProvideIPNS stores the provided [ipns.Record] for the given [ipns.Name]. // It is guaranteed that the record matches the provided name. - PutIPNSRecord(ctx context.Context, name ipns.Name, record *ipns.Record) error + ProvideIPNS(ctx context.Context, name ipns.Name, record *ipns.Record) error } type Option func(s *server) @@ -73,8 +73,8 @@ func WithStreamingResultsDisabled() Option { } } -// WithRecordsLimit sets a limit that will be passed to [ContentRouter.GetProviders] -// and [ContentRouter.GetPeers] for non-streaming requests (application/json). +// WithRecordsLimit sets a limit that will be passed to [ContentRouter.FindProviders] +// and [ContentRouter.FindPeers] for non-streaming requests (application/json). // Default is [DefaultRecordsLimit]. func WithRecordsLimit(limit int) Option { return func(s *server) { @@ -82,8 +82,8 @@ func WithRecordsLimit(limit int) Option { } } -// WithStreamingRecordsLimit sets a limit that will be passed to [ContentRouter.GetProviders] -// and [ContentRouter.GetPeers] for streaming requests (application/x-ndjson). +// WithStreamingRecordsLimit sets a limit that will be passed to [ContentRouter.FindProviders] +// and [ContentRouter.FindPeers] for streaming requests (application/x-ndjson). // Default is [DefaultStreamingRecordsLimit]. func WithStreamingRecordsLimit(limit int) Option { return func(s *server) { @@ -103,10 +103,10 @@ func Handler(svc ContentRouter, opts ...Option) http.Handler { } r := mux.NewRouter() - r.HandleFunc(getProvidersPath, server.getProviders).Methods(http.MethodGet) - r.HandleFunc(getPeersPath, server.getPeers).Methods(http.MethodGet) - r.HandleFunc(getIPNSRecordPath, server.getIPNSRecord).Methods(http.MethodGet) - r.HandleFunc(getIPNSRecordPath, server.putIPNSRecord).Methods(http.MethodPut) + r.HandleFunc(findProvidersPath, server.findProviders).Methods(http.MethodGet) + r.HandleFunc(findPeersPath, server.findPeers).Methods(http.MethodGet) + r.HandleFunc(findIPNSPath, server.findIPNS).Methods(http.MethodGet) + r.HandleFunc(findIPNSPath, server.provideIPNS).Methods(http.MethodPut) return r } @@ -154,18 +154,18 @@ func (s *server) detectResponseType(r *http.Request) (string, error) { } } -func (s *server) getProviders(w http.ResponseWriter, httpReq *http.Request) { +func (s *server) findProviders(w http.ResponseWriter, httpReq *http.Request) { vars := mux.Vars(httpReq) cidStr := vars["cid"] cid, err := cid.Decode(cidStr) if err != nil { - writeErr(w, "GetProviders", http.StatusBadRequest, fmt.Errorf("unable to parse CID: %w", err)) + writeErr(w, "FindProviders", http.StatusBadRequest, fmt.Errorf("unable to parse CID: %w", err)) return } mediaType, err := s.detectResponseType(httpReq) if err != nil { - writeErr(w, "GetProviders", http.StatusBadRequest, err) + writeErr(w, "FindProviders", http.StatusBadRequest, err) return } @@ -175,60 +175,60 @@ func (s *server) getProviders(w http.ResponseWriter, httpReq *http.Request) { ) if mediaType == mediaTypeNDJSON { - handlerFunc = s.getProvidersNDJSON + handlerFunc = s.findProvidersNDJSON recordsLimit = s.streamingRecordsLimit } else { - handlerFunc = s.getProvidersJSON + handlerFunc = s.findProvidersJSON recordsLimit = s.recordsLimit } - provIter, err := s.svc.GetProviders(httpReq.Context(), cid, recordsLimit) + provIter, err := s.svc.FindProviders(httpReq.Context(), cid, recordsLimit) if err != nil { - writeErr(w, "GetProviders", http.StatusInternalServerError, fmt.Errorf("delegate error: %w", err)) + writeErr(w, "FindProviders", http.StatusInternalServerError, fmt.Errorf("delegate error: %w", err)) return } handlerFunc(w, provIter) } -func (s *server) getProvidersJSON(w http.ResponseWriter, provIter iter.ResultIter[types.Record]) { +func (s *server) findProvidersJSON(w http.ResponseWriter, provIter iter.ResultIter[types.Record]) { defer provIter.Close() providers, err := iter.ReadAllResults(provIter) if err != nil { - writeErr(w, "GetProviders", http.StatusInternalServerError, fmt.Errorf("delegate error: %w", err)) + writeErr(w, "FindProviders", http.StatusInternalServerError, fmt.Errorf("delegate error: %w", err)) return } - writeJSONResult(w, "GetProviders", jsontypes.ProvidersResponse{ + writeJSONResult(w, "FindProviders", jsontypes.ProvidersResponse{ Providers: providers, }) } -func (s *server) getProvidersNDJSON(w http.ResponseWriter, provIter iter.ResultIter[types.Record]) { +func (s *server) findProvidersNDJSON(w http.ResponseWriter, provIter iter.ResultIter[types.Record]) { writeResultsIterNDJSON(w, provIter) } -func (s *server) getPeers(w http.ResponseWriter, r *http.Request) { +func (s *server) findPeers(w http.ResponseWriter, r *http.Request) { pidStr := mux.Vars(r)["peer-id"] // pidStr must be in CIDv1 format. Therefore, use [cid.Decode]. We can't use // [peer.Decode] because that would allow other formats to pass through. cid, err := cid.Decode(pidStr) if err != nil { - writeErr(w, "GetPeers", http.StatusBadRequest, fmt.Errorf("unable to parse peer ID: %w", err)) + writeErr(w, "FindPeers", http.StatusBadRequest, fmt.Errorf("unable to parse peer ID: %w", err)) return } pid, err := peer.FromCid(cid) if err != nil { - writeErr(w, "GetPeers", http.StatusBadRequest, fmt.Errorf("unable to parse peer ID: %w", err)) + writeErr(w, "FindPeers", http.StatusBadRequest, fmt.Errorf("unable to parse peer ID: %w", err)) return } mediaType, err := s.detectResponseType(r) if err != nil { - writeErr(w, "GetPeers", http.StatusBadRequest, err) + writeErr(w, "FindPeers", http.StatusBadRequest, err) return } @@ -238,43 +238,43 @@ func (s *server) getPeers(w http.ResponseWriter, r *http.Request) { ) if mediaType == mediaTypeNDJSON { - handlerFunc = s.getPeersNDJSON + handlerFunc = s.findPeersNDJSON recordsLimit = s.streamingRecordsLimit } else { - handlerFunc = s.getPeersJSON + handlerFunc = s.findPeersJSON recordsLimit = s.recordsLimit } - provIter, err := s.svc.GetPeers(r.Context(), pid, recordsLimit) + provIter, err := s.svc.FindPeers(r.Context(), pid, recordsLimit) if err != nil { - writeErr(w, "GetPeers", http.StatusInternalServerError, fmt.Errorf("delegate error: %w", err)) + writeErr(w, "FindPeers", http.StatusInternalServerError, fmt.Errorf("delegate error: %w", err)) return } handlerFunc(w, provIter) } -func (s *server) getPeersJSON(w http.ResponseWriter, peersIter iter.ResultIter[types.Record]) { +func (s *server) findPeersJSON(w http.ResponseWriter, peersIter iter.ResultIter[types.Record]) { defer peersIter.Close() peers, err := iter.ReadAllResults(peersIter) if err != nil { - writeErr(w, "GetPeers", http.StatusInternalServerError, fmt.Errorf("delegate error: %w", err)) + writeErr(w, "FindPeers", http.StatusInternalServerError, fmt.Errorf("delegate error: %w", err)) return } - writeJSONResult(w, "GetPeers", jsontypes.PeersResponse{ + writeJSONResult(w, "FindPeers", jsontypes.PeersResponse{ Peers: peers, }) } -func (s *server) getPeersNDJSON(w http.ResponseWriter, peersIter iter.ResultIter[types.Record]) { +func (s *server) findPeersNDJSON(w http.ResponseWriter, peersIter iter.ResultIter[types.Record]) { writeResultsIterNDJSON(w, peersIter) } -func (s *server) getIPNSRecord(w http.ResponseWriter, r *http.Request) { +func (s *server) findIPNS(w http.ResponseWriter, r *http.Request) { if !strings.Contains(r.Header.Get("Accept"), mediaTypeIPNSRecord) { - writeErr(w, "GetIPNSRecord", http.StatusNotAcceptable, errors.New("content type in 'Accept' header is missing or not supported")) + writeErr(w, "FindIPNS", http.StatusNotAcceptable, errors.New("content type in 'Accept' header is missing or not supported")) return } @@ -282,25 +282,25 @@ func (s *server) getIPNSRecord(w http.ResponseWriter, r *http.Request) { cidStr := vars["cid"] cid, err := cid.Decode(cidStr) if err != nil { - writeErr(w, "GetIPNSRecord", http.StatusBadRequest, fmt.Errorf("unable to parse CID: %w", err)) + writeErr(w, "FindIPNS", http.StatusBadRequest, fmt.Errorf("unable to parse CID: %w", err)) return } name, err := ipns.NameFromCid(cid) if err != nil { - writeErr(w, "GetIPNSRecord", http.StatusBadRequest, fmt.Errorf("peer ID CID is not valid: %w", err)) + writeErr(w, "FindIPNS", http.StatusBadRequest, fmt.Errorf("peer ID CID is not valid: %w", err)) return } - record, err := s.svc.GetIPNSRecord(r.Context(), name) + record, err := s.svc.FindIPNS(r.Context(), name) if err != nil { - writeErr(w, "GetIPNSRecord", http.StatusInternalServerError, fmt.Errorf("delegate error: %w", err)) + writeErr(w, "FindIPNS", http.StatusInternalServerError, fmt.Errorf("delegate error: %w", err)) return } rawRecord, err := ipns.MarshalRecord(record) if err != nil { - writeErr(w, "GetIPNSRecord", http.StatusInternalServerError, err) + writeErr(w, "FindIPNS", http.StatusInternalServerError, err) return } @@ -316,9 +316,9 @@ func (s *server) getIPNSRecord(w http.ResponseWriter, r *http.Request) { w.Write(rawRecord) } -func (s *server) putIPNSRecord(w http.ResponseWriter, r *http.Request) { +func (s *server) provideIPNS(w http.ResponseWriter, r *http.Request) { if !strings.Contains(r.Header.Get("Content-Type"), mediaTypeIPNSRecord) { - writeErr(w, "PutIPNSRecord", http.StatusNotAcceptable, errors.New("content type in 'Content-Type' header is missing or not supported")) + writeErr(w, "ProvideIPNS", http.StatusNotAcceptable, errors.New("content type in 'Content-Type' header is missing or not supported")) return } @@ -326,38 +326,38 @@ func (s *server) putIPNSRecord(w http.ResponseWriter, r *http.Request) { cidStr := vars["cid"] cid, err := cid.Decode(cidStr) if err != nil { - writeErr(w, "PutIPNSRecord", http.StatusBadRequest, fmt.Errorf("unable to parse CID: %w", err)) + writeErr(w, "ProvideIPNS", http.StatusBadRequest, fmt.Errorf("unable to parse CID: %w", err)) return } name, err := ipns.NameFromCid(cid) if err != nil { - writeErr(w, "PutIPNSRecord", http.StatusBadRequest, fmt.Errorf("peer ID CID is not valid: %w", err)) + writeErr(w, "ProvideIPNS", http.StatusBadRequest, fmt.Errorf("peer ID CID is not valid: %w", err)) return } // Limit the reader to the maximum record size. rawRecord, err := io.ReadAll(io.LimitReader(r.Body, int64(ipns.MaxRecordSize))) if err != nil { - writeErr(w, "PutIPNSRecord", http.StatusBadRequest, fmt.Errorf("provided record is too long: %w", err)) + writeErr(w, "ProvideIPNS", http.StatusBadRequest, fmt.Errorf("provided record is too long: %w", err)) return } record, err := ipns.UnmarshalRecord(rawRecord) if err != nil { - writeErr(w, "PutIPNSRecord", http.StatusBadRequest, fmt.Errorf("provided record is invalid: %w", err)) + writeErr(w, "ProvideIPNS", http.StatusBadRequest, fmt.Errorf("provided record is invalid: %w", err)) return } err = ipns.ValidateWithName(record, name) if err != nil { - writeErr(w, "PutIPNSRecord", http.StatusBadRequest, fmt.Errorf("provided record is invalid: %w", err)) + writeErr(w, "ProvideIPNS", http.StatusBadRequest, fmt.Errorf("provided record is invalid: %w", err)) return } - err = s.svc.PutIPNSRecord(r.Context(), name, record) + err = s.svc.ProvideIPNS(r.Context(), name, record) if err != nil { - writeErr(w, "PutIPNSRecord", http.StatusInternalServerError, fmt.Errorf("delegate error: %w", err)) + writeErr(w, "ProvideIPNS", http.StatusInternalServerError, fmt.Errorf("delegate error: %w", err)) return } diff --git a/routing/http/server/server_test.go b/routing/http/server/server_test.go index bd6c9c8b8..d23390366 100644 --- a/routing/http/server/server_test.go +++ b/routing/http/server/server_test.go @@ -40,7 +40,7 @@ func TestHeaders(t *testing.T) { cb, err := cid.Decode(c) require.NoError(t, err) - router.On("GetProviders", mock.Anything, cb, DefaultRecordsLimit). + router.On("FindProviders", mock.Anything, cb, DefaultRecordsLimit). Return(results, nil) resp, err := http.Get(serverAddr + "/routing/v1/providers/" + c) @@ -106,7 +106,7 @@ func TestProviders(t *testing.T) { if expectedStream { limit = DefaultStreamingRecordsLimit } - router.On("GetProviders", mock.Anything, cid, limit).Return(results, nil) + router.On("FindProviders", mock.Anything, cid, limit).Return(results, nil) urlStr := serverAddr + "/routing/v1/providers/" + cidStr req, err := http.NewRequest(http.MethodGet, urlStr, nil) @@ -183,7 +183,7 @@ func TestPeers(t *testing.T) { }) router := &mockContentRouter{} - router.On("GetPeers", mock.Anything, pid, 20).Return(results, nil) + router.On("FindPeers", mock.Anything, pid, 20).Return(results, nil) resp := makeRequest(t, router, mediaTypeJSON, peer.ToCid(pid).String()) require.Equal(t, 200, resp.StatusCode) @@ -218,7 +218,7 @@ func TestPeers(t *testing.T) { }) router := &mockContentRouter{} - router.On("GetPeers", mock.Anything, pid, 0).Return(results, nil) + router.On("FindPeers", mock.Anything, pid, 0).Return(results, nil) resp := makeRequest(t, router, mediaTypeNDJSON, peer.ToCid(pid).String()) require.Equal(t, 200, resp.StatusCode) @@ -283,7 +283,7 @@ func TestIPNS(t *testing.T) { require.NoError(t, err) router := &mockContentRouter{} - router.On("GetIPNSRecord", mock.Anything, name1).Return(rec, nil) + router.On("FindIPNS", mock.Anything, name1).Return(rec, nil) resp := makeRequest(t, router, "/routing/v1/ipns/"+name1.String()) require.Equal(t, 200, resp.StatusCode) @@ -316,7 +316,7 @@ func TestIPNS(t *testing.T) { t.Parallel() router := &mockContentRouter{} - router.On("PutIPNSRecord", mock.Anything, name1, record1).Return(nil) + router.On("ProvideIPNS", mock.Anything, name1, record1).Return(nil) server := httptest.NewServer(Handler(router)) t.Cleanup(server.Close) @@ -363,22 +363,22 @@ func TestIPNS(t *testing.T) { type mockContentRouter struct{ mock.Mock } -func (m *mockContentRouter) GetProviders(ctx context.Context, key cid.Cid, limit int) (iter.ResultIter[types.Record], error) { +func (m *mockContentRouter) FindProviders(ctx context.Context, key cid.Cid, limit int) (iter.ResultIter[types.Record], error) { args := m.Called(ctx, key, limit) return args.Get(0).(iter.ResultIter[types.Record]), args.Error(1) } -func (m *mockContentRouter) GetPeers(ctx context.Context, pid peer.ID, limit int) (iter.ResultIter[types.Record], error) { +func (m *mockContentRouter) FindPeers(ctx context.Context, pid peer.ID, limit int) (iter.ResultIter[types.Record], error) { args := m.Called(ctx, pid, limit) return args.Get(0).(iter.ResultIter[types.Record]), args.Error(1) } -func (m *mockContentRouter) GetIPNSRecord(ctx context.Context, name ipns.Name) (*ipns.Record, error) { +func (m *mockContentRouter) FindIPNS(ctx context.Context, name ipns.Name) (*ipns.Record, error) { args := m.Called(ctx, name) return args.Get(0).(*ipns.Record), args.Error(1) } -func (m *mockContentRouter) PutIPNSRecord(ctx context.Context, name ipns.Name, record *ipns.Record) error { +func (m *mockContentRouter) ProvideIPNS(ctx context.Context, name ipns.Name, record *ipns.Record) error { args := m.Called(ctx, name, record) return args.Error(0) } From 9b8f70d2e1d824e9af1edd2dd8144ad65139803b Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Thu, 24 Aug 2023 10:48:13 +0200 Subject: [PATCH 09/17] feat: revive SchemaBitswap, ProvideBitswap (marked as deprecated) --- CHANGELOG.md | 11 +- routing/http/client/client.go | 131 ++++++++++++ routing/http/client/client_test.go | 186 +++++++++++++++++- routing/http/contentrouter/contentrouter.go | 89 ++++++++- .../http/contentrouter/contentrouter_test.go | 82 +++++++- routing/http/server/server.go | 89 +++++++++ routing/http/server/server_test.go | 5 + routing/http/types/json/requests.go | 51 +++++ routing/http/types/json/responses.go | 48 +++++ routing/http/types/ndjson/records.go | 10 + routing/http/types/record_bitswap.go | 186 ++++++++++++++++++ 11 files changed, 869 insertions(+), 19 deletions(-) create mode 100644 routing/http/types/json/requests.go create mode 100644 routing/http/types/record_bitswap.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 88e2b6837..997711a54 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,11 +43,12 @@ The following emojis are used to highlight certain changes: * 🛠 `blockservice.New` now accepts a variadic of func options following the [Functional Options pattern](https://www.sohamkamani.com/golang/options-pattern/). * 🛠 The `routing/http` package has suffered the following modifications: - * Client `FindProviders` has been renamed to `FindProviders`. Similarly, the - required function names in the server `ContentRouter` have also been updated - for higher consistency with the remaining code and the specifications. - * Many types regarding response types were updated to conform to the updated - Peer Schema discussed in [IPIP-417](https://github.com/ipfs/specs/pull/417). + * Client `GetIPNSRecord` and `PutIPNSRecord` have been renamed to `FindIPNS` and + `ProvideIPNS`, respectively. Similarly, the required function names in the server + `ContentRouter` have also been updated. + * `ReadBitswapProviderRecord` has been renamed to `BitswapRecord` and marked as deprecated. + From now on, please use the protocol-agnostic `PeerRecord` for most use cases. The new + Peer Schema has been introduced in [IPIP-417](https://github.com/ipfs/specs/pull/417). ### Removed diff --git a/routing/http/client/client.go b/routing/http/client/client.go index bed3603d9..750d2f93e 100644 --- a/routing/http/client/client.go +++ b/routing/http/client/client.go @@ -10,17 +10,21 @@ import ( "mime" "net/http" "strings" + "time" "github.com/benbjohnson/clock" ipns "github.com/ipfs/boxo/ipns" "github.com/ipfs/boxo/routing/http/contentrouter" + "github.com/ipfs/boxo/routing/http/internal/drjson" "github.com/ipfs/boxo/routing/http/types" "github.com/ipfs/boxo/routing/http/types/iter" jsontypes "github.com/ipfs/boxo/routing/http/types/json" "github.com/ipfs/boxo/routing/http/types/ndjson" "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" + "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/peer" + "github.com/multiformats/go-multiaddr" ) var ( @@ -46,6 +50,15 @@ type client struct { httpClient httpClient clock clock.Clock accepts string + + peerID peer.ID + addrs []types.Multiaddr + identity crypto.PrivKey + + // Called immediately after signing a provide request. It is used + // for testing, e.g., testing the server with a mangled signature. + //lint:ignore SA1019 // ignore staticcheck + afterSignCallback func(req *types.WriteBitswapRecord) } // defaultUserAgent is used as a fallback to inform HTTP server which library @@ -60,6 +73,12 @@ type httpClient interface { type Option func(*client) +func WithIdentity(identity crypto.PrivKey) Option { + return func(c *client) { + c.identity = identity + } +} + func WithHTTPClient(h httpClient) Option { return func(c *client) { c.httpClient = h @@ -83,6 +102,15 @@ func WithUserAgent(ua string) Option { } } +func WithProviderInfo(peerID peer.ID, addrs []multiaddr.Multiaddr) Option { + return func(c *client) { + c.peerID = peerID + for _, a := range addrs { + c.addrs = append(c.addrs, types.Multiaddr{Multiaddr: a}) + } + } +} + func WithStreamResultsRequired() Option { return func(c *client) { c.accepts = mediaTypeNDJSON @@ -90,6 +118,7 @@ func WithStreamResultsRequired() Option { } // New creates a content routing API client. +// The Provider and identity parameters are option. If they are nil, the [client.ProvideBitswap] method will not function. func New(baseURL string, opts ...Option) (*client, error) { client := &client{ baseURL: baseURL, @@ -102,6 +131,10 @@ func New(baseURL string, opts ...Option) (*client, error) { opt(client) } + if client.identity != nil && client.peerID.Size() != 0 && !client.peerID.MatchesPublicKey(client.identity.GetPublic()) { + return nil, errors.New("identity does not match provider") + } + return client, nil } @@ -201,6 +234,104 @@ func (c *client) FindProviders(ctx context.Context, key cid.Cid) (providers iter return &measuringIter[iter.Result[types.Record]]{Iter: it, ctx: ctx, m: m}, nil } +// Deprecated: protocol-agnostic provide is being worked on in [IPIP-378]: +// +// [IPIP-378]: https://github.com/ipfs/specs/pull/378 +func (c *client) ProvideBitswap(ctx context.Context, keys []cid.Cid, ttl time.Duration) (time.Duration, error) { + if c.identity == nil { + return 0, errors.New("cannot provide Bitswap records without an identity") + } + if c.peerID.Size() == 0 { + return 0, errors.New("cannot provide Bitswap records without a peer ID") + } + + ks := make([]types.CID, len(keys)) + for i, c := range keys { + ks[i] = types.CID{Cid: c} + } + + now := c.clock.Now() + + req := types.WriteBitswapRecord{ + Protocol: "transport-bitswap", + Schema: types.SchemaBitswap, + Payload: types.BitswapPayload{ + Keys: ks, + AdvisoryTTL: &types.Duration{Duration: ttl}, + Timestamp: &types.Time{Time: now}, + ID: &c.peerID, + Addrs: c.addrs, + }, + } + err := req.Sign(c.peerID, c.identity) + if err != nil { + return 0, err + } + + if c.afterSignCallback != nil { + c.afterSignCallback(&req) + } + + advisoryTTL, err := c.provideSignedBitswapRecord(ctx, &req) + if err != nil { + return 0, err + } + + return advisoryTTL, err +} + +// ProvideAsync makes a provide request to a delegated router +// +//lint:ignore SA1019 // ignore staticcheck +func (c *client) provideSignedBitswapRecord(ctx context.Context, bswp *types.WriteBitswapRecord) (time.Duration, error) { + //lint:ignore SA1019 // ignore staticcheck + req := jsontypes.WriteProvidersRequest{Providers: []types.Record{bswp}} + + url := c.baseURL + "/routing/v1/providers/" + + b, err := drjson.MarshalJSONBytes(req) + if err != nil { + return 0, err + } + + httpReq, err := http.NewRequestWithContext(ctx, http.MethodPut, url, bytes.NewBuffer(b)) + if err != nil { + return 0, err + } + + resp, err := c.httpClient.Do(httpReq) + if err != nil { + return 0, fmt.Errorf("making HTTP req to provide a signed record: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return 0, httpError(resp.StatusCode, resp.Body) + } + + //lint:ignore SA1019 // ignore staticcheck + var provideResult jsontypes.WriteProvidersResponse + err = json.NewDecoder(resp.Body).Decode(&provideResult) + if err != nil { + return 0, err + } + if len(provideResult.ProvideResults) != 1 { + return 0, fmt.Errorf("expected 1 result but got %d", len(provideResult.ProvideResults)) + } + + //lint:ignore SA1019 // ignore staticcheck + v, ok := provideResult.ProvideResults[0].(*types.WriteBitswapRecordResponse) + if !ok { + return 0, fmt.Errorf("expected AdvisoryTTL field") + } + + if v.AdvisoryTTL != nil { + return v.AdvisoryTTL.Duration, nil + } + + return 0, nil +} + func (c *client) FindPeers(ctx context.Context, pid peer.ID) (peers iter.ResultIter[types.Record], err error) { m := newMeasurement("FindPeers") diff --git a/routing/http/client/client_test.go b/routing/http/client/client_test.go index 374e69221..27cd4eb04 100644 --- a/routing/http/client/client_test.go +++ b/routing/http/client/client_test.go @@ -11,6 +11,7 @@ import ( "testing" "time" + "github.com/benbjohnson/clock" "github.com/ipfs/boxo/coreiface/path" ipns "github.com/ipfs/boxo/ipns" ipfspath "github.com/ipfs/boxo/path" @@ -22,6 +23,7 @@ import ( "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/peer" "github.com/multiformats/go-multiaddr" + "github.com/multiformats/go-multibase" "github.com/multiformats/go-multihash" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -35,6 +37,12 @@ func (m *mockContentRouter) FindProviders(ctx context.Context, key cid.Cid, limi return args.Get(0).(iter.ResultIter[types.Record]), args.Error(1) } +//lint:ignore SA1019 // ignore staticcheck +func (m *mockContentRouter) ProvideBitswap(ctx context.Context, req *server.BitswapWriteProvideRequest) (time.Duration, error) { + args := m.Called(ctx, req) + return args.Get(0).(time.Duration), args.Error(1) +} + func (m *mockContentRouter) FindPeers(ctx context.Context, pid peer.ID, limit int) (iter.ResultIter[types.Record], error) { args := m.Called(ctx, pid, limit) return args.Get(0).(iter.ResultIter[types.Record]), args.Error(1) @@ -57,6 +65,8 @@ type testDeps struct { recordingHTTPClient *recordingHTTPClient router *mockContentRouter server *httptest.Server + peerID peer.ID + addrs []multiaddr.Multiaddr client *client } @@ -87,6 +97,7 @@ func (c *recordingHTTPClient) Do(req *http.Request) (*http.Response, error) { func makeTestDeps(t *testing.T, clientsOpts []Option, serverOpts []server.Option) testDeps { const testUserAgent = "testUserAgent" + peerID, addrs, identity := makeProviderAndIdentity() router := &mockContentRouter{} recordingHandler := &recordingHandler{ Handler: server.Handler(router, serverOpts...), @@ -101,6 +112,8 @@ func makeTestDeps(t *testing.T, clientsOpts []Option, serverOpts []server.Option serverAddr := "http://" + server.Listener.Addr().String() recordingHTTPClient := &recordingHTTPClient{httpClient: defaultHTTPClient} defaultClientOpts := []Option{ + WithProviderInfo(peerID, addrs), + WithIdentity(identity), WithUserAgent(testUserAgent), WithHTTPClient(recordingHTTPClient), } @@ -113,6 +126,8 @@ func makeTestDeps(t *testing.T, clientsOpts []Option, serverOpts []server.Option recordingHTTPClient: recordingHTTPClient, router: router, server: server, + peerID: peerID, + addrs: addrs, client: c, } } @@ -131,6 +146,13 @@ func makeCID() cid.Cid { return c } +func drAddrsToAddrs(drmas []types.Multiaddr) (addrs []multiaddr.Multiaddr) { + for _, a := range drmas { + addrs = append(addrs, a.Multiaddr) + } + return +} + func addrsToDRAddrs(addrs []multiaddr.Multiaddr) (drmas []types.Multiaddr) { for _, a := range addrs { drmas = append(drmas, types.Multiaddr{Multiaddr: a}) @@ -149,6 +171,19 @@ func makePeerRecord() types.PeerRecord { } } +//lint:ignore SA1019 // ignore staticcheck +func makeBitswapRecord() types.BitswapRecord { + peerID, addrs, _ := makeProviderAndIdentity() + //lint:ignore SA1019 // ignore staticcheck + return types.BitswapRecord{ + //lint:ignore SA1019 // ignore staticcheck + Schema: types.SchemaBitswap, + ID: &peerID, + Protocol: "transport-bitswap", + Addrs: addrsToDRAddrs(addrs), + } +} + func makeProviderAndIdentity() (peer.ID, []multiaddr.Multiaddr, crypto.PrivKey) { priv, _, err := crypto.GenerateEd25519Key(rand.Reader) if err != nil { @@ -189,9 +224,14 @@ func (e *osErrContains) errContains(t *testing.T, err error) { } func TestClient_FindProviders(t *testing.T) { - bsReadProvResp := makePeerRecord() - bitswapProvs := []iter.Result[types.Record]{ - {Val: &bsReadProvResp}, + peerRecord := makePeerRecord() + peerProviders := []iter.Result[types.Record]{ + {Val: &peerRecord}, + } + + bitswapRecord := makeBitswapRecord() + bitswapProviders := []iter.Result[types.Record]{ + {Val: &bitswapRecord}, } cases := []struct { @@ -210,14 +250,20 @@ func TestClient_FindProviders(t *testing.T) { }{ { name: "happy case", - routerResult: bitswapProvs, - expResult: bitswapProvs, + routerResult: peerProviders, + expResult: peerProviders, + expStreamingResponse: true, + }, + { + name: "happy case (with deprecated bitswap schema)", + routerResult: bitswapProviders, + expResult: bitswapProviders, expStreamingResponse: true, }, { name: "server doesn't support streaming", - routerResult: bitswapProvs, - expResult: bitswapProvs, + routerResult: peerProviders, + expResult: peerProviders, serverStreamingDisabled: true, expJSONResponse: true, }, @@ -313,6 +359,132 @@ func TestClient_FindProviders(t *testing.T) { } } +func TestClient_Provide(t *testing.T) { + cases := []struct { + name string + manglePath bool + mangleSignature bool + stopServer bool + noProviderInfo bool + noIdentity bool + + cids []cid.Cid + ttl time.Duration + + routerAdvisoryTTL time.Duration + routerErr error + + expErrContains string + expWinErrContains string + + expAdvisoryTTL time.Duration + }{ + { + name: "happy case", + cids: []cid.Cid{makeCID()}, + ttl: 1 * time.Hour, + routerAdvisoryTTL: 1 * time.Minute, + + expAdvisoryTTL: 1 * time.Minute, + }, + { + name: "should return a 403 if the payload signature verification fails", + cids: []cid.Cid{}, + mangleSignature: true, + expErrContains: "HTTP error with StatusCode=403", + }, + { + name: "should return error if identity is not provided", + noIdentity: true, + expErrContains: "cannot provide Bitswap records without an identity", + }, + { + name: "should return error if provider is not provided", + noProviderInfo: true, + expErrContains: "cannot provide Bitswap records without a peer ID", + }, + { + name: "returns an error if there's a non-200 response", + manglePath: true, + expErrContains: "HTTP error with StatusCode=404: 404 page not found", + }, + { + name: "returns an error if the HTTP client returns a non-HTTP error", + stopServer: true, + expErrContains: "connect: connection refused", + expWinErrContains: "connectex: No connection could be made because the target machine actively refused it.", + }, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + deps := makeTestDeps(t, nil, nil) + client := deps.client + router := deps.router + + if c.noIdentity { + client.identity = nil + } + if c.noProviderInfo { + client.peerID = "" + client.addrs = nil + } + + clock := clock.NewMock() + clock.Set(time.Now()) + client.clock = clock + + ctx := context.Background() + + if c.manglePath { + client.baseURL += "/foo" + } + if c.stopServer { + deps.server.Close() + } + if c.mangleSignature { + //lint:ignore SA1019 // ignore staticcheck + client.afterSignCallback = func(req *types.WriteBitswapRecord) { + mh, err := multihash.Encode([]byte("boom"), multihash.SHA2_256) + require.NoError(t, err) + mb, err := multibase.Encode(multibase.Base64, mh) + require.NoError(t, err) + + req.Signature = mb + } + } + + //lint:ignore SA1019 // ignore staticcheck + expectedProvReq := &server.BitswapWriteProvideRequest{ + Keys: c.cids, + Timestamp: clock.Now().Truncate(time.Millisecond), + AdvisoryTTL: c.ttl, + Addrs: drAddrsToAddrs(client.addrs), + ID: client.peerID, + } + + router.On("ProvideBitswap", mock.Anything, expectedProvReq). + Return(c.routerAdvisoryTTL, c.routerErr) + + advisoryTTL, err := client.ProvideBitswap(ctx, c.cids, c.ttl) + + var errorString string + if runtime.GOOS == "windows" && c.expWinErrContains != "" { + errorString = c.expWinErrContains + } else { + errorString = c.expErrContains + } + + if errorString != "" { + require.ErrorContains(t, err, errorString) + } else { + require.NoError(t, err) + } + + assert.Equal(t, c.expAdvisoryTTL, advisoryTTL) + }) + } +} + func TestClient_FindPeers(t *testing.T) { peerRecord := makePeerRecord() peerRecords := []iter.Result[types.Record]{ diff --git a/routing/http/contentrouter/contentrouter.go b/routing/http/contentrouter/contentrouter.go index 353398329..8322e37ef 100644 --- a/routing/http/contentrouter/contentrouter.go +++ b/routing/http/contentrouter/contentrouter.go @@ -4,8 +4,10 @@ import ( "context" "reflect" "strings" + "time" "github.com/ipfs/boxo/ipns" + "github.com/ipfs/boxo/routing/http/internal" "github.com/ipfs/boxo/routing/http/types" "github.com/ipfs/boxo/routing/http/types/iter" "github.com/ipfs/go-cid" @@ -15,19 +17,25 @@ import ( "github.com/libp2p/go-libp2p/core/routing" "github.com/multiformats/go-multiaddr" "github.com/multiformats/go-multihash" + "github.com/samber/lo" ) var logger = logging.Logger("routing/http/contentrouter") +const ttl = 24 * time.Hour + type Client interface { FindProviders(ctx context.Context, key cid.Cid) (iter.ResultIter[types.Record], error) + ProvideBitswap(ctx context.Context, keys []cid.Cid, ttl time.Duration) (time.Duration, error) FindPeers(ctx context.Context, pid peer.ID) (peers iter.ResultIter[types.Record], err error) FindIPNS(ctx context.Context, name ipns.Name) (*ipns.Record, error) ProvideIPNS(ctx context.Context, name ipns.Name, record *ipns.Record) error } type contentRouter struct { - client Client + client Client + maxProvideConcurrency int + maxProvideBatchSize int } var _ routing.ContentRouting = (*contentRouter)(nil) @@ -38,9 +46,23 @@ var _ routinghelpers.ReadyAbleRouter = (*contentRouter)(nil) type option func(c *contentRouter) +func WithMaxProvideConcurrency(max int) option { + return func(c *contentRouter) { + c.maxProvideConcurrency = max + } +} + +func WithMaxProvideBatchSize(max int) option { + return func(c *contentRouter) { + c.maxProvideBatchSize = max + } +} + func NewContentRoutingClient(c Client, opts ...option) *contentRouter { cr := &contentRouter{ - client: c, + client: c, + maxProvideConcurrency: 5, + maxProvideBatchSize: 100, } for _, opt := range opts { opt(cr) @@ -49,11 +71,40 @@ func NewContentRoutingClient(c Client, opts ...option) *contentRouter { } func (c *contentRouter) Provide(ctx context.Context, key cid.Cid, announce bool) error { - return routing.ErrNotSupported + // If 'true' is passed, it also announces it, otherwise it is just kept in the local + // accounting of which objects are being provided. + if !announce { + return nil + } + + _, err := c.client.ProvideBitswap(ctx, []cid.Cid{key}, ttl) + return err } +// ProvideMany provides a set of keys to the remote delegate. +// Large sets of keys are chunked into multiple requests and sent concurrently, according to the concurrency configuration. +// TODO: switch to use [client.Provide] when ready. func (c *contentRouter) ProvideMany(ctx context.Context, mhKeys []multihash.Multihash) error { - return routing.ErrNotSupported + keys := make([]cid.Cid, 0, len(mhKeys)) + for _, m := range mhKeys { + keys = append(keys, cid.NewCidV1(cid.Raw, m)) + } + + if len(keys) <= c.maxProvideBatchSize { + _, err := c.client.ProvideBitswap(ctx, keys, ttl) + return err + } + + return internal.DoBatch( + ctx, + c.maxProvideBatchSize, + c.maxProvideConcurrency, + keys, + func(ctx context.Context, batch []cid.Cid) error { + _, err := c.client.ProvideBitswap(ctx, batch, ttl) + return err + }, + ) } // Ready is part of the existing [routing.ReadyAbleRouter] interface. @@ -72,7 +123,8 @@ func readProviderResponses(iter iter.ResultIter[types.Record], ch chan<- peer.Ad continue } v := res.Val - if v.GetSchema() == types.SchemaPeer { + switch v.GetSchema() { + case types.SchemaPeer: result, ok := v.(*types.PeerRecord) if !ok { logger.Errorw( @@ -83,6 +135,33 @@ func readProviderResponses(iter iter.ResultIter[types.Record], ch chan<- peer.Ad continue } + // We only care about Bitswap records here. + if !lo.Contains(result.Protocols, "transport-bitswap") { + continue + } + + var addrs []multiaddr.Multiaddr + for _, a := range result.Addrs { + addrs = append(addrs, a.Multiaddr) + } + + ch <- peer.AddrInfo{ + ID: *result.ID, + Addrs: addrs, + } + //lint:ignore SA1019 // ignore staticcheck + case types.SchemaBitswap: + //lint:ignore SA1019 // ignore staticcheck + result, ok := v.(*types.BitswapRecord) + if !ok { + logger.Errorw( + "problem casting find providers result", + "Schema", v.GetSchema(), + "Type", reflect.TypeOf(v).String(), + ) + continue + } + var addrs []multiaddr.Multiaddr for _, a := range result.Addrs { addrs = append(addrs, a.Multiaddr) diff --git a/routing/http/contentrouter/contentrouter_test.go b/routing/http/contentrouter/contentrouter_test.go index f11075af1..b3445640f 100644 --- a/routing/http/contentrouter/contentrouter_test.go +++ b/routing/http/contentrouter/contentrouter_test.go @@ -16,33 +16,96 @@ import ( "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/routing" "github.com/multiformats/go-multihash" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) type mockClient struct{ mock.Mock } +func (m *mockClient) ProvideBitswap(ctx context.Context, keys []cid.Cid, ttl time.Duration) (time.Duration, error) { + args := m.Called(ctx, keys, ttl) + return args.Get(0).(time.Duration), args.Error(1) +} + func (m *mockClient) FindProviders(ctx context.Context, key cid.Cid) (iter.ResultIter[types.Record], error) { args := m.Called(ctx, key) return args.Get(0).(iter.ResultIter[types.Record]), args.Error(1) } + func (m *mockClient) FindPeers(ctx context.Context, pid peer.ID) (iter.ResultIter[types.Record], error) { args := m.Called(ctx, pid) return args.Get(0).(iter.ResultIter[types.Record]), args.Error(1) } + func (m *mockClient) Ready(ctx context.Context) (bool, error) { args := m.Called(ctx) return args.Bool(0), args.Error(1) } + func (m *mockClient) FindIPNS(ctx context.Context, name ipns.Name) (*ipns.Record, error) { args := m.Called(ctx, name) return args.Get(0).(*ipns.Record), args.Error(1) } + func (m *mockClient) ProvideIPNS(ctx context.Context, name ipns.Name, record *ipns.Record) error { args := m.Called(ctx, name, record) return args.Error(0) } +func TestProvide(t *testing.T) { + for _, c := range []struct { + name string + announce bool + + expNotProvided bool + }{ + { + name: "announce=false results in no client request", + announce: false, + expNotProvided: true, + }, + { + name: "announce=true results in a client req", + announce: true, + }, + } { + t.Run(c.name, func(t *testing.T) { + ctx := context.Background() + key := makeCID() + client := &mockClient{} + crc := NewContentRoutingClient(client) + + if !c.expNotProvided { + client.On("ProvideBitswap", ctx, []cid.Cid{key}, ttl).Return(time.Minute, nil) + } + + err := crc.Provide(ctx, key, c.announce) + assert.NoError(t, err) + + if c.expNotProvided { + client.AssertNumberOfCalls(t, "ProvideBitswap", 0) + } + }) + } +} + +func TestProvideMany(t *testing.T) { + cids := []cid.Cid{makeCID(), makeCID()} + var mhs []multihash.Multihash + for _, c := range cids { + mhs = append(mhs, c.Hash()) + } + ctx := context.Background() + client := &mockClient{} + crc := NewContentRoutingClient(client) + + client.On("ProvideBitswap", ctx, cids, ttl).Return(time.Minute, nil) + + err := crc.ProvideMany(ctx, mhs) + require.NoError(t, err) +} + func makeCID() cid.Cid { buf := make([]byte, 63) _, err := rand.Read(buf) @@ -65,17 +128,31 @@ func TestFindProvidersAsync(t *testing.T) { p1 := peer.ID("peer1") p2 := peer.ID("peer2") + p3 := peer.ID("peer3") + p4 := peer.ID("peer4") ais := []types.Record{ &types.PeerRecord{ Schema: types.SchemaPeer, ID: &p1, Protocols: []string{"transport-bitswap"}, }, + //lint:ignore SA1019 // ignore staticcheck + &types.BitswapRecord{ + //lint:ignore SA1019 // ignore staticcheck + Schema: types.SchemaBitswap, + ID: &p2, + Protocol: "transport-bitswap", + }, &types.PeerRecord{ Schema: types.SchemaPeer, - ID: &p2, + ID: &p3, Protocols: []string{"transport-bitswap"}, }, + &types.PeerRecord{ + Schema: types.SchemaPeer, + ID: &p4, + Protocols: []string{"transport-horse"}, + }, &types.UnknownRecord{ Schema: "UNKNOWN", }, @@ -84,7 +161,7 @@ func TestFindProvidersAsync(t *testing.T) { client.On("FindProviders", ctx, key).Return(aisIter, nil) - aiChan := crc.FindProvidersAsync(ctx, key, 2) + aiChan := crc.FindProvidersAsync(ctx, key, 3) var actualAIs []peer.AddrInfo for ai := range aiChan { @@ -94,6 +171,7 @@ func TestFindProvidersAsync(t *testing.T) { expected := []peer.AddrInfo{ {ID: p1}, {ID: p2}, + {ID: p3}, } require.Equal(t, expected, actualAIs) diff --git a/routing/http/server/server.go b/routing/http/server/server.go index aa91bab01..d32d2d4b7 100644 --- a/routing/http/server/server.go +++ b/routing/http/server/server.go @@ -3,6 +3,7 @@ package server import ( "bytes" "context" + "encoding/json" "errors" "fmt" "io" @@ -10,6 +11,7 @@ import ( "net/http" "strconv" "strings" + "time" "github.com/cespare/xxhash/v2" "github.com/gorilla/mux" @@ -20,6 +22,7 @@ import ( jsontypes "github.com/ipfs/boxo/routing/http/types/json" "github.com/ipfs/go-cid" "github.com/libp2p/go-libp2p/core/peer" + "github.com/multiformats/go-multiaddr" logging "github.com/ipfs/go-log/v2" ) @@ -37,6 +40,7 @@ const ( var logger = logging.Logger("routing/http/server") const ( + providePath = "/routing/v1/providers/" findProvidersPath = "/routing/v1/providers/{cid}" findPeersPath = "/routing/v1/peers/{peer-id}" findIPNSPath = "/routing/v1/ipns/{cid}" @@ -52,6 +56,11 @@ type ContentRouter interface { // Limit indicates the maximum amount of results to return; 0 means unbounded. FindProviders(ctx context.Context, cid cid.Cid, limit int) (iter.ResultIter[types.Record], error) + // Deprecated: protocol-agnostic provide is being worked on in [IPIP-378]: + // + // [IPIP-378]: https://github.com/ipfs/specs/pull/378 + ProvideBitswap(ctx context.Context, req *BitswapWriteProvideRequest) (time.Duration, error) + // FindPeers searches for peers who have the provided [peer.ID]. // Limit indicates the maximum amount of results to return; 0 means unbounded. FindPeers(ctx context.Context, pid peer.ID, limit int) (iter.ResultIter[types.Record], error) @@ -64,6 +73,26 @@ type ContentRouter interface { ProvideIPNS(ctx context.Context, name ipns.Name, record *ipns.Record) error } +// Deprecated: protocol-agnostic provide is being worked on in [IPIP-378]: +// +// [IPIP-378]: https://github.com/ipfs/specs/pull/378 +type BitswapWriteProvideRequest struct { + Keys []cid.Cid + Timestamp time.Time + AdvisoryTTL time.Duration + ID peer.ID + Addrs []multiaddr.Multiaddr +} + +// Deprecated: protocol-agnostic provide is being worked on in [IPIP-378]: +// +// [IPIP-378]: https://github.com/ipfs/specs/pull/378 +type WriteProvideRequest struct { + Protocol string + Schema string + Bytes []byte +} + type Option func(s *server) // WithStreamingResultsDisabled disables ndjson responses, so that the server only supports JSON responses. @@ -104,6 +133,7 @@ func Handler(svc ContentRouter, opts ...Option) http.Handler { r := mux.NewRouter() r.HandleFunc(findProvidersPath, server.findProviders).Methods(http.MethodGet) + r.HandleFunc(providePath, server.provide).Methods(http.MethodPut) r.HandleFunc(findPeersPath, server.findPeers).Methods(http.MethodGet) r.HandleFunc(findIPNSPath, server.findIPNS).Methods(http.MethodGet) r.HandleFunc(findIPNSPath, server.provideIPNS).Methods(http.MethodPut) @@ -254,6 +284,65 @@ func (s *server) findPeers(w http.ResponseWriter, r *http.Request) { handlerFunc(w, provIter) } +func (s *server) provide(w http.ResponseWriter, httpReq *http.Request) { + //lint:ignore SA1019 // ignore staticcheck + req := jsontypes.WriteProvidersRequest{} + err := json.NewDecoder(httpReq.Body).Decode(&req) + _ = httpReq.Body.Close() + if err != nil { + writeErr(w, "Provide", http.StatusBadRequest, fmt.Errorf("invalid request: %w", err)) + return + } + + //lint:ignore SA1019 // ignore staticcheck + resp := jsontypes.WriteProvidersResponse{} + + for i, prov := range req.Providers { + switch v := prov.(type) { + //lint:ignore SA1019 // ignore staticcheck + case *types.WriteBitswapRecord: + err := v.Verify() + if err != nil { + logErr("Provide", "signature verification failed", err) + writeErr(w, "Provide", http.StatusForbidden, errors.New("signature verification failed")) + return + } + + keys := make([]cid.Cid, len(v.Payload.Keys)) + for i, k := range v.Payload.Keys { + keys[i] = k.Cid + } + addrs := make([]multiaddr.Multiaddr, len(v.Payload.Addrs)) + for i, a := range v.Payload.Addrs { + addrs[i] = a.Multiaddr + } + advisoryTTL, err := s.svc.ProvideBitswap(httpReq.Context(), &BitswapWriteProvideRequest{ + Keys: keys, + Timestamp: v.Payload.Timestamp.Time, + AdvisoryTTL: v.Payload.AdvisoryTTL.Duration, + ID: *v.Payload.ID, + Addrs: addrs, + }) + if err != nil { + writeErr(w, "Provide", http.StatusInternalServerError, fmt.Errorf("delegate error: %w", err)) + return + } + resp.ProvideResults = append(resp.ProvideResults, + //lint:ignore SA1019 // ignore staticcheck + &types.WriteBitswapRecordResponse{ + Protocol: v.Protocol, + Schema: v.Schema, + AdvisoryTTL: &types.Duration{Duration: advisoryTTL}, + }, + ) + default: + writeErr(w, "Provide", http.StatusBadRequest, fmt.Errorf("provider record %d is not bitswap", i)) + return + } + } + writeJSONResult(w, "Provide", resp) +} + func (s *server) findPeersJSON(w http.ResponseWriter, peersIter iter.ResultIter[types.Record]) { defer peersIter.Close() diff --git a/routing/http/server/server_test.go b/routing/http/server/server_test.go index d23390366..ad49e09d9 100644 --- a/routing/http/server/server_test.go +++ b/routing/http/server/server_test.go @@ -368,6 +368,11 @@ func (m *mockContentRouter) FindProviders(ctx context.Context, key cid.Cid, limi return args.Get(0).(iter.ResultIter[types.Record]), args.Error(1) } +func (m *mockContentRouter) ProvideBitswap(ctx context.Context, req *BitswapWriteProvideRequest) (time.Duration, error) { + args := m.Called(ctx, req) + return args.Get(0).(time.Duration), args.Error(1) +} + func (m *mockContentRouter) FindPeers(ctx context.Context, pid peer.ID, limit int) (iter.ResultIter[types.Record], error) { args := m.Called(ctx, pid, limit) return args.Get(0).(iter.ResultIter[types.Record]), args.Error(1) diff --git a/routing/http/types/json/requests.go b/routing/http/types/json/requests.go new file mode 100644 index 000000000..4b582c3ba --- /dev/null +++ b/routing/http/types/json/requests.go @@ -0,0 +1,51 @@ +package json + +import ( + "encoding/json" + + "github.com/ipfs/boxo/routing/http/types" +) + +// Deprecated: protocol-agnostic provide is being worked on in [IPIP-378]: +// +// [IPIP-378]: https://github.com/ipfs/specs/pull/378 +type WriteProvidersRequest struct { + Providers []types.Record +} + +func (r *WriteProvidersRequest) UnmarshalJSON(b []byte) error { + type wpr struct{ Providers []json.RawMessage } + var tempWPR wpr + err := json.Unmarshal(b, &tempWPR) + if err != nil { + return err + } + + for _, provBytes := range tempWPR.Providers { + var rawProv types.UnknownRecord + err := json.Unmarshal(provBytes, &rawProv) + if err != nil { + return err + } + + switch rawProv.Schema { + //lint:ignore SA1019 // ignore staticcheck + case types.SchemaBitswap: + //lint:ignore SA1019 // ignore staticcheck + var prov types.WriteBitswapRecord + err := json.Unmarshal(rawProv.Bytes, &prov) + if err != nil { + return err + } + r.Providers = append(r.Providers, &prov) + default: + var prov types.UnknownRecord + err := json.Unmarshal(b, &prov) + if err != nil { + return err + } + r.Providers = append(r.Providers, &prov) + } + } + return nil +} diff --git a/routing/http/types/json/responses.go b/routing/http/types/json/responses.go index b7b5dcbb9..dfcfad830 100644 --- a/routing/http/types/json/responses.go +++ b/routing/http/types/json/responses.go @@ -41,6 +41,15 @@ func (r *RecordsArray) UnmarshalJSON(b []byte) error { return err } *r = append(*r, &prov) + //lint:ignore SA1019 // ignore staticcheck + case types.SchemaBitswap: + //lint:ignore SA1019 // ignore staticcheck + var prov types.BitswapRecord + err := json.Unmarshal(provBytes, &prov) + if err != nil { + return err + } + *r = append(*r, &prov) default: *r = append(*r, &readProv) } @@ -48,3 +57,42 @@ func (r *RecordsArray) UnmarshalJSON(b []byte) error { } return nil } + +// Deprecated: protocol-agnostic provide is being worked on in [IPIP-378]: +// +// [IPIP-378]: https://github.com/ipfs/specs/pull/378 +type WriteProvidersResponse struct { + ProvideResults []types.Record +} + +func (r *WriteProvidersResponse) UnmarshalJSON(b []byte) error { + var tempWPR struct{ ProvideResults []json.RawMessage } + err := json.Unmarshal(b, &tempWPR) + if err != nil { + return err + } + + for _, provBytes := range tempWPR.ProvideResults { + var rawProv types.UnknownRecord + err := json.Unmarshal(provBytes, &rawProv) + if err != nil { + return err + } + + switch rawProv.Schema { + //lint:ignore SA1019 // ignore staticcheck + case types.SchemaBitswap: + //lint:ignore SA1019 // ignore staticcheck + var prov types.WriteBitswapRecordResponse + err := json.Unmarshal(rawProv.Bytes, &prov) + if err != nil { + return err + } + r.ProvideResults = append(r.ProvideResults, &prov) + default: + r.ProvideResults = append(r.ProvideResults, &rawProv) + } + } + + return nil +} diff --git a/routing/http/types/ndjson/records.go b/routing/http/types/ndjson/records.go index 32e9199a1..d1a36b411 100644 --- a/routing/http/types/ndjson/records.go +++ b/routing/http/types/ndjson/records.go @@ -26,6 +26,16 @@ func NewRecordsIter(r io.Reader) iter.Iter[iter.Result[types.Record]] { return result } result.Val = &prov + //lint:ignore SA1019 // ignore staticcheck + case types.SchemaBitswap: + //lint:ignore SA1019 // ignore staticcheck + var prov types.BitswapRecord + err := json.Unmarshal(upr.Val.Bytes, &prov) + if err != nil { + result.Err = err + return result + } + result.Val = &prov default: result.Val = &upr.Val } diff --git a/routing/http/types/record_bitswap.go b/routing/http/types/record_bitswap.go new file mode 100644 index 000000000..5e49c8165 --- /dev/null +++ b/routing/http/types/record_bitswap.go @@ -0,0 +1,186 @@ +package types + +import ( + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + + "github.com/ipfs/boxo/routing/http/internal/drjson" + "github.com/libp2p/go-libp2p/core/crypto" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/multiformats/go-multibase" +) + +// Deprecated: use the more versatile [SchemaPeer] instead. For more information, read [IPIP-417]. +// +// [IPIP-417]: https://github.com/ipfs/specs/pull/417 +const SchemaBitswap = "bitswap" + +var ( + _ Record = &BitswapRecord{} +) + +// Deprecated: use the more versatile [PeerRecord] instead. For more information, read [IPIP-417]. +// +// [IPIP-417]: https://github.com/ipfs/specs/pull/417 +type BitswapRecord struct { + Schema string + Protocol string + ID *peer.ID + Addrs []Multiaddr +} + +func (br *BitswapRecord) GetSchema() string { + return br.Schema +} + +var _ Record = &WriteBitswapRecord{} + +// Deprecated: protocol-agnostic provide is being worked on in [IPIP-378]: +// +// [IPIP-378]: https://github.com/ipfs/specs/pull/378 +type WriteBitswapRecord struct { + Schema string + Protocol string + Signature string + + // this content must be untouched because it is signed and we need to verify it + RawPayload json.RawMessage `json:"Payload"` + Payload BitswapPayload `json:"-"` +} + +type BitswapPayload struct { + Keys []CID + Timestamp *Time + AdvisoryTTL *Duration + ID *peer.ID + Addrs []Multiaddr +} + +func (wr *WriteBitswapRecord) GetSchema() string { + return wr.Schema +} + +type tmpBWPR WriteBitswapRecord + +func (p *WriteBitswapRecord) UnmarshalJSON(b []byte) error { + var bwp tmpBWPR + err := json.Unmarshal(b, &bwp) + if err != nil { + return err + } + + p.Protocol = bwp.Protocol + p.Schema = bwp.Schema + p.Signature = bwp.Signature + p.RawPayload = bwp.RawPayload + + return json.Unmarshal(bwp.RawPayload, &p.Payload) +} + +func (p *WriteBitswapRecord) IsSigned() bool { + return p.Signature != "" +} + +func (p *WriteBitswapRecord) setRawPayload() error { + payloadBytes, err := drjson.MarshalJSONBytes(p.Payload) + if err != nil { + return fmt.Errorf("marshaling bitswap write provider payload: %w", err) + } + + p.RawPayload = payloadBytes + + return nil +} + +func (p *WriteBitswapRecord) Sign(peerID peer.ID, key crypto.PrivKey) error { + if p.IsSigned() { + return errors.New("already signed") + } + + if key == nil { + return errors.New("no key provided") + } + + sid, err := peer.IDFromPrivateKey(key) + if err != nil { + return err + } + if sid != peerID { + return errors.New("not the correct signing key") + } + + err = p.setRawPayload() + if err != nil { + return err + } + hash := sha256.Sum256([]byte(p.RawPayload)) + sig, err := key.Sign(hash[:]) + if err != nil { + return err + } + + sigStr, err := multibase.Encode(multibase.Base64, sig) + if err != nil { + return fmt.Errorf("multibase-encoding signature: %w", err) + } + + p.Signature = sigStr + return nil +} + +func (p *WriteBitswapRecord) Verify() error { + if !p.IsSigned() { + return errors.New("not signed") + } + + if p.Payload.ID == nil { + return errors.New("peer ID must be specified") + } + + // note that we only generate and set the payload if it hasn't already been set + // to allow for passing through the payload untouched if it is already provided + if p.RawPayload == nil { + err := p.setRawPayload() + if err != nil { + return err + } + } + + pk, err := p.Payload.ID.ExtractPublicKey() + if err != nil { + return fmt.Errorf("extracing public key from peer ID: %w", err) + } + + _, sigBytes, err := multibase.Decode(p.Signature) + if err != nil { + return fmt.Errorf("multibase-decoding signature to verify: %w", err) + } + + hash := sha256.Sum256([]byte(p.RawPayload)) + ok, err := pk.Verify(hash[:], sigBytes) + if err != nil { + return fmt.Errorf("verifying hash with signature: %w", err) + } + if !ok { + return errors.New("signature failed to verify") + } + + return nil +} + +var _ Record = &WriteBitswapRecordResponse{} + +// Deprecated: protocol-agnostic provide is being worked on in [IPIP-378]: +// +// [IPIP-378]: https://github.com/ipfs/specs/pull/378 +type WriteBitswapRecordResponse struct { + Schema string + Protocol string + AdvisoryTTL *Duration +} + +func (r *WriteBitswapRecordResponse) GetSchema() string { + return r.Schema +} From d019ab1dc1b2c7dfa040a61183ef6a259e3675d7 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Thu, 24 Aug 2023 11:33:24 +0200 Subject: [PATCH 10/17] docs: changelog --- CHANGELOG.md | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 997711a54..d6fbfb877 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,10 +16,26 @@ The following emojis are used to highlight certain changes: ### Added +* ✨ The `routing/http` not supports Delegated Peer Routing as per [IPIP-417](https://github.com/ipfs/specs/pull/417). + ### Changed +* 🛠 The `routing/http` package has suffered the following modifications: + * Client `GetIPNSRecord` and `PutIPNSRecord` have been renamed to `FindIPNS` and + `ProvideIPNS`, respectively. Similarly, the required function names in the server + `ContentRouter` have also been updated. + * `ReadBitswapProviderRecord` has been renamed to `BitswapRecord` and marked as deprecated. + From now on, please use the protocol-agnostic `PeerRecord` for most use cases. The new + Peer Schema has been introduced in [IPIP-417](https://github.com/ipfs/specs/pull/417). + ### Removed +* 🛠 The `routing/http` package has suffered the following removals: + * Server and client no longer support the generic `Provide` method for content routing. + `ProvideBitswap` is still usable, but marked as deprecated. A protocol-agnostic + provide mechanism is being worked on in [IPIP-378](https://github.com/ipfs/specs/pull/378). + * Server no longer exports `FindProvidersPath` and `ProvidePath`. + ### Fixed ### Security @@ -32,32 +48,16 @@ The following emojis are used to highlight certain changes: as per [IPIP-379](https://specs.ipfs.tech/ipips/ipip-0379/). * 🛠 The `verifycid` package has been updated with the new Allowlist interface as part of reducing globals efforts. -* The `blockservice` and `provider` packages has been updated to accommodate for +* The `blockservice` and `provider` packages has been updated to accommodate for changes in `verifycid`. -* ✨ The `routing/http` package has received the following additions: - * Supports Delegated IPNS as per [IPIP-379](https://specs.ipfs.tech/ipips/ipip-0379/). - * Supports Delegated Peer Routing as per [IPIP-417](https://github.com/ipfs/specs/pull/417). ### Changed * 🛠 `blockservice.New` now accepts a variadic of func options following the [Functional Options pattern](https://www.sohamkamani.com/golang/options-pattern/). -* 🛠 The `routing/http` package has suffered the following modifications: - * Client `GetIPNSRecord` and `PutIPNSRecord` have been renamed to `FindIPNS` and - `ProvideIPNS`, respectively. Similarly, the required function names in the server - `ContentRouter` have also been updated. - * `ReadBitswapProviderRecord` has been renamed to `BitswapRecord` and marked as deprecated. - From now on, please use the protocol-agnostic `PeerRecord` for most use cases. The new - Peer Schema has been introduced in [IPIP-417](https://github.com/ipfs/specs/pull/417). ### Removed -* 🛠 The `routing/http` package has suffered the following removals: - * Server and client no longer support the `Provide*` methods for content routing. - These methods did not conform to any specification, as it is still being worked - out in [IPIP-378](https://github.com/ipfs/specs/pull/378). - * Server no longer exports `FindProvidersPath` and `ProvidePath`. - ### Fixed - HTTP Gateway API: Not having a block will result in a 5xx error rather than 404 From c87c3b07ba3928900814ac14251c25131a2b8aa6 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Fri, 25 Aug 2023 10:26:43 +0200 Subject: [PATCH 11/17] chore: address feedback --- routing/http/contentrouter/contentrouter_test.go | 4 ++-- routing/http/server/server_test.go | 16 +++++++++------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/routing/http/contentrouter/contentrouter_test.go b/routing/http/contentrouter/contentrouter_test.go index b3445640f..041280773 100644 --- a/routing/http/contentrouter/contentrouter_test.go +++ b/routing/http/contentrouter/contentrouter_test.go @@ -154,7 +154,7 @@ func TestFindProvidersAsync(t *testing.T) { Protocols: []string{"transport-horse"}, }, &types.UnknownRecord{ - Schema: "UNKNOWN", + Schema: "unknown", }, } aisIter := iter.ToResultIter[types.Record](iter.FromSlice(ais)) @@ -185,7 +185,7 @@ func TestFindPeer(t *testing.T) { p1 := peer.ID("peer1") ais := []types.Record{ &types.UnknownRecord{ - Schema: "Unknown", + Schema: "unknown", }, &types.PeerRecord{ Schema: types.SchemaPeer, diff --git a/routing/http/server/server_test.go b/routing/http/server/server_test.go index ad49e09d9..0520862fb 100644 --- a/routing/http/server/server_test.go +++ b/routing/http/server/server_test.go @@ -90,11 +90,13 @@ func TestProviders(t *testing.T) { Protocols: []string{"transport-bitswap"}, Addrs: []types.Multiaddr{}, }}, - {Val: &types.PeerRecord{ - Schema: types.SchemaPeer, - ID: &pid2, - Protocols: []string{"transport-bitswap"}, - Addrs: []types.Multiaddr{}, + //lint:ignore SA1019 // ignore staticcheck + {Val: &types.BitswapRecord{ + //lint:ignore SA1019 // ignore staticcheck + Schema: types.SchemaBitswap, + ID: &pid2, + Protocol: "transport-bitswap", + Addrs: []types.Multiaddr{}, }}}, ) @@ -126,11 +128,11 @@ func TestProviders(t *testing.T) { } t.Run("JSON Response", func(t *testing.T) { - runTest(t, mediaTypeJSON, false, `{"Providers":[{"Addrs":[],"ID":"12D3KooWM8sovaEGU1bmiWGWAzvs47DEcXKZZTuJnpQyVTkRs2Vn","Protocols":["transport-bitswap"],"Schema":"peer"},{"Addrs":[],"ID":"12D3KooWM8sovaEGU1bmiWGWAzvs47DEcXKZZTuJnpQyVTkRs2Vz","Protocols":["transport-bitswap"],"Schema":"peer"}]}`) + runTest(t, mediaTypeJSON, false, `{"Providers":[{"Addrs":[],"ID":"12D3KooWM8sovaEGU1bmiWGWAzvs47DEcXKZZTuJnpQyVTkRs2Vn","Protocols":["transport-bitswap"],"Schema":"peer"},{"Schema":"bitswap","Protocol":"transport-bitswap","ID":"12D3KooWM8sovaEGU1bmiWGWAzvs47DEcXKZZTuJnpQyVTkRs2Vz","Addrs":[]}]}`) }) t.Run("NDJSON Response", func(t *testing.T) { - runTest(t, mediaTypeNDJSON, true, `{"Addrs":[],"ID":"12D3KooWM8sovaEGU1bmiWGWAzvs47DEcXKZZTuJnpQyVTkRs2Vn","Protocols":["transport-bitswap"],"Schema":"peer"}`+"\n"+`{"Addrs":[],"ID":"12D3KooWM8sovaEGU1bmiWGWAzvs47DEcXKZZTuJnpQyVTkRs2Vz","Protocols":["transport-bitswap"],"Schema":"peer"}`+"\n") + runTest(t, mediaTypeNDJSON, true, `{"Addrs":[],"ID":"12D3KooWM8sovaEGU1bmiWGWAzvs47DEcXKZZTuJnpQyVTkRs2Vn","Protocols":["transport-bitswap"],"Schema":"peer"}`+"\n"+`{"Schema":"bitswap","Protocol":"transport-bitswap","ID":"12D3KooWM8sovaEGU1bmiWGWAzvs47DEcXKZZTuJnpQyVTkRs2Vz","Addrs":[]}`+"\n") }) } From e40b7c69d04adc936cf83ca0b5e8884a31dc3226 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Fri, 25 Aug 2023 13:30:55 +0200 Subject: [PATCH 12/17] refactor: rename Find/ProvideIPNS to Get/PutIPNS --- CHANGELOG.md | 4 +- gateway/blocks_backend.go | 2 +- gateway/gateway.go | 4 +- gateway/gateway_test.go | 4 +- gateway/handler_ipns_record.go | 2 +- gateway/metrics.go | 6 +-- gateway/utilities_test.go | 2 +- routing/http/client/client.go | 4 +- routing/http/client/client_test.go | 20 ++++---- routing/http/contentrouter/contentrouter.go | 10 ++-- .../http/contentrouter/contentrouter_test.go | 8 ++-- routing/http/server/server.go | 46 +++++++++---------- routing/http/server/server_test.go | 8 ++-- 13 files changed, 60 insertions(+), 60 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d6fbfb877..4519cb209 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,8 +21,8 @@ The following emojis are used to highlight certain changes: ### Changed * 🛠 The `routing/http` package has suffered the following modifications: - * Client `GetIPNSRecord` and `PutIPNSRecord` have been renamed to `FindIPNS` and - `ProvideIPNS`, respectively. Similarly, the required function names in the server + * Client `GetIPNSRecord` and `PutIPNSRecord` have been renamed to `GetIPNS` and + `PutIPNS`, respectively. Similarly, the required function names in the server `ContentRouter` have also been updated. * `ReadBitswapProviderRecord` has been renamed to `BitswapRecord` and marked as deprecated. From now on, please use the protocol-agnostic `PeerRecord` for most use cases. The new diff --git a/gateway/blocks_backend.go b/gateway/blocks_backend.go index 3509991f6..42d97e9ec 100644 --- a/gateway/blocks_backend.go +++ b/gateway/blocks_backend.go @@ -586,7 +586,7 @@ func (bb *BlocksBackend) ResolveMutable(ctx context.Context, p ifacepath.Path) ( } } -func (bb *BlocksBackend) FindIPNS(ctx context.Context, c cid.Cid) ([]byte, error) { +func (bb *BlocksBackend) GetIPNS(ctx context.Context, c cid.Cid) ([]byte, error) { if bb.routing == nil { return nil, NewErrorStatusCode(errors.New("IPNS Record responses are not supported by this gateway"), http.StatusNotImplemented) } diff --git a/gateway/gateway.go b/gateway/gateway.go index d5d00d484..3a2c9d4ed 100644 --- a/gateway/gateway.go +++ b/gateway/gateway.go @@ -330,9 +330,9 @@ type IPFSBackend interface { // IsCached returns whether or not the path exists locally. IsCached(context.Context, path.Path) bool - // FindIPNS retrieves the best IPNS record for a given CID (libp2p-key) + // GetIPNS retrieves the best IPNS record for a given CID (libp2p-key) // from the routing system. - FindIPNS(context.Context, cid.Cid) ([]byte, error) + GetIPNS(context.Context, cid.Cid) ([]byte, error) // ResolveMutable takes a mutable path and resolves it into an immutable one. This means recursively resolving any // DNSLink or IPNS records. diff --git a/gateway/gateway_test.go b/gateway/gateway_test.go index 4e7c76998..b4d98a9dd 100644 --- a/gateway/gateway_test.go +++ b/gateway/gateway_test.go @@ -731,7 +731,7 @@ func (mb *errorMockBackend) ResolveMutable(ctx context.Context, path ipath.Path) return ImmutablePath{}, mb.err } -func (mb *errorMockBackend) FindIPNS(ctx context.Context, c cid.Cid) ([]byte, error) { +func (mb *errorMockBackend) GetIPNS(ctx context.Context, c cid.Cid) ([]byte, error) { return nil, mb.err } @@ -815,7 +815,7 @@ func (mb *panicMockBackend) ResolveMutable(ctx context.Context, p ipath.Path) (I panic("i am panicking") } -func (mb *panicMockBackend) FindIPNS(ctx context.Context, c cid.Cid) ([]byte, error) { +func (mb *panicMockBackend) GetIPNS(ctx context.Context, c cid.Cid) ([]byte, error) { panic("i am panicking") } diff --git a/gateway/handler_ipns_record.go b/gateway/handler_ipns_record.go index 41e8e6268..95563b22f 100644 --- a/gateway/handler_ipns_record.go +++ b/gateway/handler_ipns_record.go @@ -41,7 +41,7 @@ func (i *handler) serveIpnsRecord(ctx context.Context, w http.ResponseWriter, r return false } - rawRecord, err := i.backend.FindIPNS(ctx, c) + rawRecord, err := i.backend.GetIPNS(ctx, c) if err != nil { i.webError(w, r, err, http.StatusInternalServerError) return false diff --git a/gateway/metrics.go b/gateway/metrics.go index f6f71a486..b89029fd5 100644 --- a/gateway/metrics.go +++ b/gateway/metrics.go @@ -143,13 +143,13 @@ func (b *ipfsBackendWithMetrics) IsCached(ctx context.Context, path path.Path) b return bln } -func (b *ipfsBackendWithMetrics) FindIPNS(ctx context.Context, cid cid.Cid) ([]byte, error) { +func (b *ipfsBackendWithMetrics) GetIPNS(ctx context.Context, cid cid.Cid) ([]byte, error) { begin := time.Now() - name := "IPFSBackend.FindIPNS" + name := "IPFSBackend.GetIPNS" ctx, span := spanTrace(ctx, name, trace.WithAttributes(attribute.String("cid", cid.String()))) defer span.End() - r, err := b.backend.FindIPNS(ctx, cid) + r, err := b.backend.GetIPNS(ctx, cid) b.updateBackendCallMetric(name, err, begin) return r, err diff --git a/gateway/utilities_test.go b/gateway/utilities_test.go index 6c59f133d..9780c8b3d 100644 --- a/gateway/utilities_test.go +++ b/gateway/utilities_test.go @@ -157,7 +157,7 @@ func (mb *mockBackend) ResolveMutable(ctx context.Context, p ipath.Path) (Immuta return mb.gw.ResolveMutable(ctx, p) } -func (mb *mockBackend) FindIPNS(ctx context.Context, c cid.Cid) ([]byte, error) { +func (mb *mockBackend) GetIPNS(ctx context.Context, c cid.Cid) ([]byte, error) { return nil, routing.ErrNotSupported } diff --git a/routing/http/client/client.go b/routing/http/client/client.go index 750d2f93e..4a0d29b33 100644 --- a/routing/http/client/client.go +++ b/routing/http/client/client.go @@ -405,7 +405,7 @@ func (c *client) FindPeers(ctx context.Context, pid peer.ID) (peers iter.ResultI return &measuringIter[iter.Result[types.Record]]{Iter: it, ctx: ctx, m: m}, nil } -func (c *client) FindIPNS(ctx context.Context, name ipns.Name) (*ipns.Record, error) { +func (c *client) GetIPNS(ctx context.Context, name ipns.Name) (*ipns.Record, error) { url := c.baseURL + "/routing/v1/ipns/" + name.String() httpReq, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) @@ -443,7 +443,7 @@ func (c *client) FindIPNS(ctx context.Context, name ipns.Name) (*ipns.Record, er return record, nil } -func (c *client) ProvideIPNS(ctx context.Context, name ipns.Name, record *ipns.Record) error { +func (c *client) PutIPNS(ctx context.Context, name ipns.Name, record *ipns.Record) error { url := c.baseURL + "/routing/v1/ipns/" + name.String() rawRecord, err := ipns.MarshalRecord(record) diff --git a/routing/http/client/client_test.go b/routing/http/client/client_test.go index 27cd4eb04..95683bc3f 100644 --- a/routing/http/client/client_test.go +++ b/routing/http/client/client_test.go @@ -48,12 +48,12 @@ func (m *mockContentRouter) FindPeers(ctx context.Context, pid peer.ID, limit in return args.Get(0).(iter.ResultIter[types.Record]), args.Error(1) } -func (m *mockContentRouter) FindIPNS(ctx context.Context, name ipns.Name) (*ipns.Record, error) { +func (m *mockContentRouter) GetIPNS(ctx context.Context, name ipns.Name) (*ipns.Record, error) { args := m.Called(ctx, name) return args.Get(0).(*ipns.Record), args.Error(1) } -func (m *mockContentRouter) ProvideIPNS(ctx context.Context, name ipns.Name, record *ipns.Record) error { +func (m *mockContentRouter) PutIPNS(ctx context.Context, name ipns.Name, record *ipns.Record) error { args := m.Called(ctx, name, record) return args.Error(0) } @@ -648,9 +648,9 @@ func TestClient_IPNS(t *testing.T) { client := deps.client router := deps.router - router.On("FindIPNS", mock.Anything, name).Return(nil, errors.New("something wrong happened")) + router.On("GetIPNS", mock.Anything, name).Return(nil, errors.New("something wrong happened")) - receivedRecord, err := client.FindIPNS(context.Background(), name) + receivedRecord, err := client.GetIPNS(context.Background(), name) require.Error(t, err) require.Nil(t, receivedRecord) }) @@ -664,9 +664,9 @@ func TestClient_IPNS(t *testing.T) { client := deps.client router := deps.router - router.On("FindIPNS", mock.Anything, name).Return(record, nil) + router.On("GetIPNS", mock.Anything, name).Return(record, nil) - receivedRecord, err := client.FindIPNS(context.Background(), name) + receivedRecord, err := client.GetIPNS(context.Background(), name) require.NoError(t, err) require.Equal(t, record, receivedRecord) }) @@ -680,9 +680,9 @@ func TestClient_IPNS(t *testing.T) { client := deps.client router := deps.router - router.On("FindIPNS", mock.Anything, name2).Return(record, nil) + router.On("GetIPNS", mock.Anything, name2).Return(record, nil) - receivedRecord, err := client.FindIPNS(context.Background(), name2) + receivedRecord, err := client.GetIPNS(context.Background(), name2) require.Error(t, err) require.Nil(t, receivedRecord) }) @@ -695,9 +695,9 @@ func TestClient_IPNS(t *testing.T) { client := deps.client router := deps.router - router.On("ProvideIPNS", mock.Anything, name, record).Return(nil) + router.On("PutIPNS", mock.Anything, name, record).Return(nil) - err := client.ProvideIPNS(context.Background(), name, record) + err := client.PutIPNS(context.Background(), name, record) require.NoError(t, err) }) } diff --git a/routing/http/contentrouter/contentrouter.go b/routing/http/contentrouter/contentrouter.go index 8322e37ef..0301cdcc2 100644 --- a/routing/http/contentrouter/contentrouter.go +++ b/routing/http/contentrouter/contentrouter.go @@ -28,8 +28,8 @@ type Client interface { FindProviders(ctx context.Context, key cid.Cid) (iter.ResultIter[types.Record], error) ProvideBitswap(ctx context.Context, keys []cid.Cid, ttl time.Duration) (time.Duration, error) FindPeers(ctx context.Context, pid peer.ID) (peers iter.ResultIter[types.Record], err error) - FindIPNS(ctx context.Context, name ipns.Name) (*ipns.Record, error) - ProvideIPNS(ctx context.Context, name ipns.Name, record *ipns.Record) error + GetIPNS(ctx context.Context, name ipns.Name) (*ipns.Record, error) + PutIPNS(ctx context.Context, name ipns.Name, record *ipns.Record) error } type contentRouter struct { @@ -243,7 +243,7 @@ func (c *contentRouter) PutValue(ctx context.Context, key string, data []byte, o return err } - return c.client.ProvideIPNS(ctx, name, record) + return c.client.PutIPNS(ctx, name, record) } func (c *contentRouter) GetValue(ctx context.Context, key string, opts ...routing.Option) ([]byte, error) { @@ -256,7 +256,7 @@ func (c *contentRouter) GetValue(ctx context.Context, key string, opts ...routin return nil, err } - record, err := c.client.FindIPNS(ctx, name) + record, err := c.client.GetIPNS(ctx, name) if err != nil { return nil, err } @@ -277,7 +277,7 @@ func (c *contentRouter) SearchValue(ctx context.Context, key string, opts ...rou ch := make(chan []byte) go func() { - record, err := c.client.FindIPNS(ctx, name) + record, err := c.client.GetIPNS(ctx, name) if err != nil { close(ch) return diff --git a/routing/http/contentrouter/contentrouter_test.go b/routing/http/contentrouter/contentrouter_test.go index 041280773..aad34ee03 100644 --- a/routing/http/contentrouter/contentrouter_test.go +++ b/routing/http/contentrouter/contentrouter_test.go @@ -43,12 +43,12 @@ func (m *mockClient) Ready(ctx context.Context) (bool, error) { return args.Bool(0), args.Error(1) } -func (m *mockClient) FindIPNS(ctx context.Context, name ipns.Name) (*ipns.Record, error) { +func (m *mockClient) GetIPNS(ctx context.Context, name ipns.Name) (*ipns.Record, error) { args := m.Called(ctx, name) return args.Get(0).(*ipns.Record), args.Error(1) } -func (m *mockClient) ProvideIPNS(ctx context.Context, name ipns.Name, record *ipns.Record) error { +func (m *mockClient) PutIPNS(ctx context.Context, name ipns.Name, record *ipns.Record) error { args := m.Called(ctx, name, record) return args.Error(0) } @@ -249,7 +249,7 @@ func TestGetValue(t *testing.T) { t.Run("Succeeds On Valid IPNS Name", func(t *testing.T) { sk, name := makeName(t) rec, rawRec := makeIPNSRecord(t, sk) - client.On("FindIPNS", ctx, name).Return(rec, nil) + client.On("GetIPNS", ctx, name).Return(rec, nil) v, err := crc.GetValue(ctx, string(name.RoutingKey())) require.NoError(t, err) require.Equal(t, rawRec, v) @@ -280,7 +280,7 @@ func TestPutValue(t *testing.T) { }) t.Run("Succeeds On Valid IPNS Name & Record", func(t *testing.T) { - client.On("ProvideIPNS", ctx, name, mock.Anything).Return(nil) + client.On("PutIPNS", ctx, name, mock.Anything).Return(nil) err := crc.PutValue(ctx, string(name.RoutingKey()), rawRec) require.NoError(t, err) }) diff --git a/routing/http/server/server.go b/routing/http/server/server.go index d32d2d4b7..20a708835 100644 --- a/routing/http/server/server.go +++ b/routing/http/server/server.go @@ -43,7 +43,7 @@ const ( providePath = "/routing/v1/providers/" findProvidersPath = "/routing/v1/providers/{cid}" findPeersPath = "/routing/v1/peers/{peer-id}" - findIPNSPath = "/routing/v1/ipns/{cid}" + GetIPNSPath = "/routing/v1/ipns/{cid}" ) type FindProvidersAsyncResponse struct { @@ -65,12 +65,12 @@ type ContentRouter interface { // Limit indicates the maximum amount of results to return; 0 means unbounded. FindPeers(ctx context.Context, pid peer.ID, limit int) (iter.ResultIter[types.Record], error) - // FindIPNS searches for an [ipns.Record] for the given [ipns.Name]. - FindIPNS(ctx context.Context, name ipns.Name) (*ipns.Record, error) + // GetIPNS searches for an [ipns.Record] for the given [ipns.Name]. + GetIPNS(ctx context.Context, name ipns.Name) (*ipns.Record, error) - // ProvideIPNS stores the provided [ipns.Record] for the given [ipns.Name]. + // PutIPNS stores the provided [ipns.Record] for the given [ipns.Name]. // It is guaranteed that the record matches the provided name. - ProvideIPNS(ctx context.Context, name ipns.Name, record *ipns.Record) error + PutIPNS(ctx context.Context, name ipns.Name, record *ipns.Record) error } // Deprecated: protocol-agnostic provide is being worked on in [IPIP-378]: @@ -135,8 +135,8 @@ func Handler(svc ContentRouter, opts ...Option) http.Handler { r.HandleFunc(findProvidersPath, server.findProviders).Methods(http.MethodGet) r.HandleFunc(providePath, server.provide).Methods(http.MethodPut) r.HandleFunc(findPeersPath, server.findPeers).Methods(http.MethodGet) - r.HandleFunc(findIPNSPath, server.findIPNS).Methods(http.MethodGet) - r.HandleFunc(findIPNSPath, server.provideIPNS).Methods(http.MethodPut) + r.HandleFunc(GetIPNSPath, server.GetIPNS).Methods(http.MethodGet) + r.HandleFunc(GetIPNSPath, server.PutIPNS).Methods(http.MethodPut) return r } @@ -361,9 +361,9 @@ func (s *server) findPeersNDJSON(w http.ResponseWriter, peersIter iter.ResultIte writeResultsIterNDJSON(w, peersIter) } -func (s *server) findIPNS(w http.ResponseWriter, r *http.Request) { +func (s *server) GetIPNS(w http.ResponseWriter, r *http.Request) { if !strings.Contains(r.Header.Get("Accept"), mediaTypeIPNSRecord) { - writeErr(w, "FindIPNS", http.StatusNotAcceptable, errors.New("content type in 'Accept' header is missing or not supported")) + writeErr(w, "GetIPNS", http.StatusNotAcceptable, errors.New("content type in 'Accept' header is missing or not supported")) return } @@ -371,25 +371,25 @@ func (s *server) findIPNS(w http.ResponseWriter, r *http.Request) { cidStr := vars["cid"] cid, err := cid.Decode(cidStr) if err != nil { - writeErr(w, "FindIPNS", http.StatusBadRequest, fmt.Errorf("unable to parse CID: %w", err)) + writeErr(w, "GetIPNS", http.StatusBadRequest, fmt.Errorf("unable to parse CID: %w", err)) return } name, err := ipns.NameFromCid(cid) if err != nil { - writeErr(w, "FindIPNS", http.StatusBadRequest, fmt.Errorf("peer ID CID is not valid: %w", err)) + writeErr(w, "GetIPNS", http.StatusBadRequest, fmt.Errorf("peer ID CID is not valid: %w", err)) return } - record, err := s.svc.FindIPNS(r.Context(), name) + record, err := s.svc.GetIPNS(r.Context(), name) if err != nil { - writeErr(w, "FindIPNS", http.StatusInternalServerError, fmt.Errorf("delegate error: %w", err)) + writeErr(w, "GetIPNS", http.StatusInternalServerError, fmt.Errorf("delegate error: %w", err)) return } rawRecord, err := ipns.MarshalRecord(record) if err != nil { - writeErr(w, "FindIPNS", http.StatusInternalServerError, err) + writeErr(w, "GetIPNS", http.StatusInternalServerError, err) return } @@ -405,9 +405,9 @@ func (s *server) findIPNS(w http.ResponseWriter, r *http.Request) { w.Write(rawRecord) } -func (s *server) provideIPNS(w http.ResponseWriter, r *http.Request) { +func (s *server) PutIPNS(w http.ResponseWriter, r *http.Request) { if !strings.Contains(r.Header.Get("Content-Type"), mediaTypeIPNSRecord) { - writeErr(w, "ProvideIPNS", http.StatusNotAcceptable, errors.New("content type in 'Content-Type' header is missing or not supported")) + writeErr(w, "PutIPNS", http.StatusNotAcceptable, errors.New("content type in 'Content-Type' header is missing or not supported")) return } @@ -415,38 +415,38 @@ func (s *server) provideIPNS(w http.ResponseWriter, r *http.Request) { cidStr := vars["cid"] cid, err := cid.Decode(cidStr) if err != nil { - writeErr(w, "ProvideIPNS", http.StatusBadRequest, fmt.Errorf("unable to parse CID: %w", err)) + writeErr(w, "PutIPNS", http.StatusBadRequest, fmt.Errorf("unable to parse CID: %w", err)) return } name, err := ipns.NameFromCid(cid) if err != nil { - writeErr(w, "ProvideIPNS", http.StatusBadRequest, fmt.Errorf("peer ID CID is not valid: %w", err)) + writeErr(w, "PutIPNS", http.StatusBadRequest, fmt.Errorf("peer ID CID is not valid: %w", err)) return } // Limit the reader to the maximum record size. rawRecord, err := io.ReadAll(io.LimitReader(r.Body, int64(ipns.MaxRecordSize))) if err != nil { - writeErr(w, "ProvideIPNS", http.StatusBadRequest, fmt.Errorf("provided record is too long: %w", err)) + writeErr(w, "PutIPNS", http.StatusBadRequest, fmt.Errorf("provided record is too long: %w", err)) return } record, err := ipns.UnmarshalRecord(rawRecord) if err != nil { - writeErr(w, "ProvideIPNS", http.StatusBadRequest, fmt.Errorf("provided record is invalid: %w", err)) + writeErr(w, "PutIPNS", http.StatusBadRequest, fmt.Errorf("provided record is invalid: %w", err)) return } err = ipns.ValidateWithName(record, name) if err != nil { - writeErr(w, "ProvideIPNS", http.StatusBadRequest, fmt.Errorf("provided record is invalid: %w", err)) + writeErr(w, "PutIPNS", http.StatusBadRequest, fmt.Errorf("provided record is invalid: %w", err)) return } - err = s.svc.ProvideIPNS(r.Context(), name, record) + err = s.svc.PutIPNS(r.Context(), name, record) if err != nil { - writeErr(w, "ProvideIPNS", http.StatusInternalServerError, fmt.Errorf("delegate error: %w", err)) + writeErr(w, "PutIPNS", http.StatusInternalServerError, fmt.Errorf("delegate error: %w", err)) return } diff --git a/routing/http/server/server_test.go b/routing/http/server/server_test.go index 0520862fb..ced5b2e64 100644 --- a/routing/http/server/server_test.go +++ b/routing/http/server/server_test.go @@ -285,7 +285,7 @@ func TestIPNS(t *testing.T) { require.NoError(t, err) router := &mockContentRouter{} - router.On("FindIPNS", mock.Anything, name1).Return(rec, nil) + router.On("GetIPNS", mock.Anything, name1).Return(rec, nil) resp := makeRequest(t, router, "/routing/v1/ipns/"+name1.String()) require.Equal(t, 200, resp.StatusCode) @@ -318,7 +318,7 @@ func TestIPNS(t *testing.T) { t.Parallel() router := &mockContentRouter{} - router.On("ProvideIPNS", mock.Anything, name1, record1).Return(nil) + router.On("PutIPNS", mock.Anything, name1, record1).Return(nil) server := httptest.NewServer(Handler(router)) t.Cleanup(server.Close) @@ -380,12 +380,12 @@ func (m *mockContentRouter) FindPeers(ctx context.Context, pid peer.ID, limit in return args.Get(0).(iter.ResultIter[types.Record]), args.Error(1) } -func (m *mockContentRouter) FindIPNS(ctx context.Context, name ipns.Name) (*ipns.Record, error) { +func (m *mockContentRouter) GetIPNS(ctx context.Context, name ipns.Name) (*ipns.Record, error) { args := m.Called(ctx, name) return args.Get(0).(*ipns.Record), args.Error(1) } -func (m *mockContentRouter) ProvideIPNS(ctx context.Context, name ipns.Name, record *ipns.Record) error { +func (m *mockContentRouter) PutIPNS(ctx context.Context, name ipns.Name, record *ipns.Record) error { args := m.Called(ctx, name, record) return args.Error(0) } From 3b793487866e5b98c3e9b788dc009ead75afe1db Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Fri, 25 Aug 2023 13:51:23 +0200 Subject: [PATCH 13/17] feat: do not filter over transport-bitswap for FindProviders --- routing/http/contentrouter/contentrouter.go | 9 ++------- routing/http/contentrouter/contentrouter_test.go | 1 + 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/routing/http/contentrouter/contentrouter.go b/routing/http/contentrouter/contentrouter.go index 0301cdcc2..2438d4fea 100644 --- a/routing/http/contentrouter/contentrouter.go +++ b/routing/http/contentrouter/contentrouter.go @@ -17,7 +17,6 @@ import ( "github.com/libp2p/go-libp2p/core/routing" "github.com/multiformats/go-multiaddr" "github.com/multiformats/go-multihash" - "github.com/samber/lo" ) var logger = logging.Logger("routing/http/contentrouter") @@ -112,7 +111,8 @@ func (c *contentRouter) Ready() bool { return true } -// readProviderResponses reads bitswap records from the iterator into the given channel, dropping non-bitswap records. +// readProviderResponses reads peer records (and bitswap records for legacy +// compatibility) from the iterator into the given channel. func readProviderResponses(iter iter.ResultIter[types.Record], ch chan<- peer.AddrInfo) { defer close(ch) defer iter.Close() @@ -135,11 +135,6 @@ func readProviderResponses(iter iter.ResultIter[types.Record], ch chan<- peer.Ad continue } - // We only care about Bitswap records here. - if !lo.Contains(result.Protocols, "transport-bitswap") { - continue - } - var addrs []multiaddr.Multiaddr for _, a := range result.Addrs { addrs = append(addrs, a.Multiaddr) diff --git a/routing/http/contentrouter/contentrouter_test.go b/routing/http/contentrouter/contentrouter_test.go index aad34ee03..83a086997 100644 --- a/routing/http/contentrouter/contentrouter_test.go +++ b/routing/http/contentrouter/contentrouter_test.go @@ -172,6 +172,7 @@ func TestFindProvidersAsync(t *testing.T) { {ID: p1}, {ID: p2}, {ID: p3}, + {ID: p4}, } require.Equal(t, expected, actualAIs) From bdd7f8762146363dafdb94b0324ced665f3594fe Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Fri, 25 Aug 2023 14:09:17 +0200 Subject: [PATCH 14/17] gateway: revert name changes --- gateway/blocks_backend.go | 2 +- gateway/gateway.go | 4 ++-- gateway/gateway_test.go | 4 ++-- gateway/handler_ipns_record.go | 2 +- gateway/metrics.go | 6 +++--- gateway/utilities_test.go | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/gateway/blocks_backend.go b/gateway/blocks_backend.go index 42d97e9ec..208c92062 100644 --- a/gateway/blocks_backend.go +++ b/gateway/blocks_backend.go @@ -586,7 +586,7 @@ func (bb *BlocksBackend) ResolveMutable(ctx context.Context, p ifacepath.Path) ( } } -func (bb *BlocksBackend) GetIPNS(ctx context.Context, c cid.Cid) ([]byte, error) { +func (bb *BlocksBackend) GetIPNSRecord(ctx context.Context, c cid.Cid) ([]byte, error) { if bb.routing == nil { return nil, NewErrorStatusCode(errors.New("IPNS Record responses are not supported by this gateway"), http.StatusNotImplemented) } diff --git a/gateway/gateway.go b/gateway/gateway.go index 3a2c9d4ed..780691a45 100644 --- a/gateway/gateway.go +++ b/gateway/gateway.go @@ -330,9 +330,9 @@ type IPFSBackend interface { // IsCached returns whether or not the path exists locally. IsCached(context.Context, path.Path) bool - // GetIPNS retrieves the best IPNS record for a given CID (libp2p-key) + // GetIPNSRecord retrieves the best IPNS record for a given CID (libp2p-key) // from the routing system. - GetIPNS(context.Context, cid.Cid) ([]byte, error) + GetIPNSRecord(context.Context, cid.Cid) ([]byte, error) // ResolveMutable takes a mutable path and resolves it into an immutable one. This means recursively resolving any // DNSLink or IPNS records. diff --git a/gateway/gateway_test.go b/gateway/gateway_test.go index b4d98a9dd..98996acb3 100644 --- a/gateway/gateway_test.go +++ b/gateway/gateway_test.go @@ -731,7 +731,7 @@ func (mb *errorMockBackend) ResolveMutable(ctx context.Context, path ipath.Path) return ImmutablePath{}, mb.err } -func (mb *errorMockBackend) GetIPNS(ctx context.Context, c cid.Cid) ([]byte, error) { +func (mb *errorMockBackend) GetIPNSRecord(ctx context.Context, c cid.Cid) ([]byte, error) { return nil, mb.err } @@ -815,7 +815,7 @@ func (mb *panicMockBackend) ResolveMutable(ctx context.Context, p ipath.Path) (I panic("i am panicking") } -func (mb *panicMockBackend) GetIPNS(ctx context.Context, c cid.Cid) ([]byte, error) { +func (mb *panicMockBackend) GetIPNSRecord(ctx context.Context, c cid.Cid) ([]byte, error) { panic("i am panicking") } diff --git a/gateway/handler_ipns_record.go b/gateway/handler_ipns_record.go index 95563b22f..b077fa59a 100644 --- a/gateway/handler_ipns_record.go +++ b/gateway/handler_ipns_record.go @@ -41,7 +41,7 @@ func (i *handler) serveIpnsRecord(ctx context.Context, w http.ResponseWriter, r return false } - rawRecord, err := i.backend.GetIPNS(ctx, c) + rawRecord, err := i.backend.GetIPNSRecord(ctx, c) if err != nil { i.webError(w, r, err, http.StatusInternalServerError) return false diff --git a/gateway/metrics.go b/gateway/metrics.go index b89029fd5..69e81425f 100644 --- a/gateway/metrics.go +++ b/gateway/metrics.go @@ -143,13 +143,13 @@ func (b *ipfsBackendWithMetrics) IsCached(ctx context.Context, path path.Path) b return bln } -func (b *ipfsBackendWithMetrics) GetIPNS(ctx context.Context, cid cid.Cid) ([]byte, error) { +func (b *ipfsBackendWithMetrics) GetIPNSRecord(ctx context.Context, cid cid.Cid) ([]byte, error) { begin := time.Now() - name := "IPFSBackend.GetIPNS" + name := "IPFSBackend.GetIPNSRecord" ctx, span := spanTrace(ctx, name, trace.WithAttributes(attribute.String("cid", cid.String()))) defer span.End() - r, err := b.backend.GetIPNS(ctx, cid) + r, err := b.backend.GetIPNSRecord(ctx, cid) b.updateBackendCallMetric(name, err, begin) return r, err diff --git a/gateway/utilities_test.go b/gateway/utilities_test.go index 9780c8b3d..27ba43a14 100644 --- a/gateway/utilities_test.go +++ b/gateway/utilities_test.go @@ -157,7 +157,7 @@ func (mb *mockBackend) ResolveMutable(ctx context.Context, p ipath.Path) (Immuta return mb.gw.ResolveMutable(ctx, p) } -func (mb *mockBackend) GetIPNS(ctx context.Context, c cid.Cid) ([]byte, error) { +func (mb *mockBackend) GetIPNSRecord(ctx context.Context, c cid.Cid) ([]byte, error) { return nil, routing.ErrNotSupported } From 07479efc3b3bbe578afeb68c5e84c78f6a50dc3f Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Fri, 25 Aug 2023 14:26:44 +0200 Subject: [PATCH 15/17] fix: do not produce null fields in marshalled json --- routing/http/server/server_test.go | 4 ++-- routing/http/types/record_bitswap.go | 2 +- routing/http/types/record_peer.go | 13 +++++++++++-- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/routing/http/server/server_test.go b/routing/http/server/server_test.go index ced5b2e64..f6d4a3dba 100644 --- a/routing/http/server/server_test.go +++ b/routing/http/server/server_test.go @@ -128,11 +128,11 @@ func TestProviders(t *testing.T) { } t.Run("JSON Response", func(t *testing.T) { - runTest(t, mediaTypeJSON, false, `{"Providers":[{"Addrs":[],"ID":"12D3KooWM8sovaEGU1bmiWGWAzvs47DEcXKZZTuJnpQyVTkRs2Vn","Protocols":["transport-bitswap"],"Schema":"peer"},{"Schema":"bitswap","Protocol":"transport-bitswap","ID":"12D3KooWM8sovaEGU1bmiWGWAzvs47DEcXKZZTuJnpQyVTkRs2Vz","Addrs":[]}]}`) + runTest(t, mediaTypeJSON, false, `{"Providers":[{"Addrs":[],"ID":"12D3KooWM8sovaEGU1bmiWGWAzvs47DEcXKZZTuJnpQyVTkRs2Vn","Protocols":["transport-bitswap"],"Schema":"peer"},{"Schema":"bitswap","Protocol":"transport-bitswap","ID":"12D3KooWM8sovaEGU1bmiWGWAzvs47DEcXKZZTuJnpQyVTkRs2Vz"}]}`) }) t.Run("NDJSON Response", func(t *testing.T) { - runTest(t, mediaTypeNDJSON, true, `{"Addrs":[],"ID":"12D3KooWM8sovaEGU1bmiWGWAzvs47DEcXKZZTuJnpQyVTkRs2Vn","Protocols":["transport-bitswap"],"Schema":"peer"}`+"\n"+`{"Schema":"bitswap","Protocol":"transport-bitswap","ID":"12D3KooWM8sovaEGU1bmiWGWAzvs47DEcXKZZTuJnpQyVTkRs2Vz","Addrs":[]}`+"\n") + runTest(t, mediaTypeNDJSON, true, `{"Addrs":[],"ID":"12D3KooWM8sovaEGU1bmiWGWAzvs47DEcXKZZTuJnpQyVTkRs2Vn","Protocols":["transport-bitswap"],"Schema":"peer"}`+"\n"+`{"Schema":"bitswap","Protocol":"transport-bitswap","ID":"12D3KooWM8sovaEGU1bmiWGWAzvs47DEcXKZZTuJnpQyVTkRs2Vz"}`+"\n") }) } diff --git a/routing/http/types/record_bitswap.go b/routing/http/types/record_bitswap.go index 5e49c8165..0780fc3eb 100644 --- a/routing/http/types/record_bitswap.go +++ b/routing/http/types/record_bitswap.go @@ -28,7 +28,7 @@ type BitswapRecord struct { Schema string Protocol string ID *peer.ID - Addrs []Multiaddr + Addrs []Multiaddr `json:",omitempty"` } func (br *BitswapRecord) GetSchema() string { diff --git a/routing/http/types/record_peer.go b/routing/http/types/record_peer.go index ff4704eb3..76bd810e0 100644 --- a/routing/http/types/record_peer.go +++ b/routing/http/types/record_peer.go @@ -64,9 +64,18 @@ func (pr PeerRecord) MarshalJSON() ([]byte, error) { m[key] = val } } + + // Schema and ID must always be set. m["Schema"] = pr.Schema m["ID"] = pr.ID - m["Addrs"] = pr.Addrs - m["Protocols"] = pr.Protocols + + if pr.Addrs != nil { + m["Addrs"] = pr.Addrs + } + + if pr.Protocols != nil { + m["Protocols"] = pr.Protocols + } + return drjson.MarshalJSONBytes(m) } From ab6523c3423863e977e8b0e08addf856da5b588d Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Fri, 25 Aug 2023 15:10:36 +0200 Subject: [PATCH 16/17] feat: meaningful error when cannot decode peer id --- routing/http/server/server.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/routing/http/server/server.go b/routing/http/server/server.go index 20a708835..9e7d81a04 100644 --- a/routing/http/server/server.go +++ b/routing/http/server/server.go @@ -246,7 +246,11 @@ func (s *server) findPeers(w http.ResponseWriter, r *http.Request) { // [peer.Decode] because that would allow other formats to pass through. cid, err := cid.Decode(pidStr) if err != nil { - writeErr(w, "FindPeers", http.StatusBadRequest, fmt.Errorf("unable to parse peer ID: %w", err)) + if pid, err := peer.Decode(pidStr); err == nil { + writeErr(w, "FindPeers", http.StatusBadRequest, fmt.Errorf("the value is a peer ID, try using its CID representation: %s", peer.ToCid(pid).String())) + } else { + writeErr(w, "FindPeers", http.StatusBadRequest, fmt.Errorf("unable to parse peer ID: %w", err)) + } return } From 6c62606175d2a401dc398fbe007cc287f4ca41f8 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Fri, 25 Aug 2023 15:26:42 +0200 Subject: [PATCH 17/17] docs: cleanup changelog --- CHANGELOG.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4519cb209..8a4122902 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,11 +16,11 @@ The following emojis are used to highlight certain changes: ### Added -* ✨ The `routing/http` not supports Delegated Peer Routing as per [IPIP-417](https://github.com/ipfs/specs/pull/417). +* ✨ The `routing/http` implements Delegated Peer Routing introduced in [IPIP-417](https://github.com/ipfs/specs/pull/417). ### Changed -* 🛠 The `routing/http` package has suffered the following modifications: +* 🛠 The `routing/http` package received the following modifications: * Client `GetIPNSRecord` and `PutIPNSRecord` have been renamed to `GetIPNS` and `PutIPNS`, respectively. Similarly, the required function names in the server `ContentRouter` have also been updated. @@ -30,8 +30,8 @@ The following emojis are used to highlight certain changes: ### Removed -* 🛠 The `routing/http` package has suffered the following removals: - * Server and client no longer support the generic `Provide` method for content routing. +* 🛠 The `routing/http` package experienced following removals: + * Server and client no longer support the experimental `Provide` method. `ProvideBitswap` is still usable, but marked as deprecated. A protocol-agnostic provide mechanism is being worked on in [IPIP-378](https://github.com/ipfs/specs/pull/378). * Server no longer exports `FindProvidersPath` and `ProvidePath`.