Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Bitswap Retrieval, Sync version #86

Merged
merged 21 commits into from
Feb 16, 2023
Merged

Bitswap Retrieval, Sync version #86

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
ebfd862
feat(bitswap): add helpers to fit in go-bitswap
hannahhoward Feb 14, 2023
2137ea4
feat(bitswap): add bitswap retriever impl
hannahhoward Feb 15, 2023
ec93d7c
feat(retriever): add bitswap retriever test
hannahhoward Feb 15, 2023
63a177c
Update pkg/retriever/bitswaphelpers/indexerrouting.go
hannahhoward Feb 15, 2023
eb7ff13
Update pkg/retriever/bitswaphelpers/indexerrouting.go
hannahhoward Feb 15, 2023
980be5f
Update pkg/retriever/retriever.go
hannahhoward Feb 15, 2023
57b272c
refactor(bitswaphelpers): respond PR comments
hannahhoward Feb 15, 2023
cd6c923
fix(deps): get all deps in line
hannahhoward Feb 15, 2023
a858f39
fix(retriever): add atomics
hannahhoward Feb 15, 2023
da02d45
style(lint): remove unused type
hannahhoward Feb 15, 2023
17d6cf7
style(lint): fix unkeyed fields
hannahhoward Feb 15, 2023
185384b
fix: improve flakes, make test runs faster
rvagg Feb 15, 2023
531d3de
Update pkg/retriever/bitswaphelpers/multiblockstore.go
hannahhoward Feb 15, 2023
78c0f94
Merge pull request #87 from filecoin-project/rvagg/test-robust-and-fa…
hannahhoward Feb 15, 2023
b643c79
feat(itest): add integration test for bitswap
hannahhoward Feb 15, 2023
017fb14
style(lint): fix lint errors
hannahhoward Feb 15, 2023
827fcd7
fix(itest): speed up itest
hannahhoward Feb 15, 2023
288e9db
Update pkg/internal/itest/bitswapfetch_test.go
hannahhoward Feb 16, 2023
ce79d61
feat(bitswap): add timeouts, further testing
hannahhoward Feb 16, 2023
e93ce02
Merge branch 'feat/bitswap-local' into feat/bitswap
hannahhoward Feb 16, 2023
3ba9704
fix(retriever): increase timeout
hannahhoward Feb 16, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 10 additions & 1 deletion cmd/lassie/daemon.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,9 @@ package main

import (
"fmt"
"time"

"github.com/filecoin-project/lassie/pkg/lassie"
httpserver "github.com/filecoin-project/lassie/pkg/server/http"
"github.com/urfave/cli/v2"
)
Expand Down Expand Up @@ -39,7 +41,14 @@ var daemonCmd = &cli.Command{
func daemonCommand(cctx *cli.Context) error {
address := cctx.String("address")
port := cctx.Uint("port")
httpServer, err := httpserver.NewHttpServer(cctx.Context, address, port)

// create a lassie instance
lassie, err := lassie.NewLassie(cctx.Context, lassie.WithProviderTimeout(20*time.Second))
if err != nil {
return err
}

httpServer, err := httpserver.NewHttpServer(cctx.Context, lassie, address, port)
Comment on lines -42 to +51
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

👍

if err != nil {
log.Errorw("failed to create http server", "err", err)
return err
Expand Down
24 changes: 12 additions & 12 deletions cmd/lassie/fetch.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ func Fetch(c *cli.Context) error {
}

timeout := c.Duration("timeout")
timeoutOpt := lassie.WithTimeout(timeout)
timeoutOpt := lassie.WithProviderTimeout(timeout)

var opts = []lassie.LassieOption{timeoutOpt}
if fetchProviderAddrInfo != nil {
Expand Down Expand Up @@ -172,23 +172,23 @@ func (pp *progressPrinter) subscriber(event types.RetrievalEvent) {
case types.IndexerPhase:
fmt.Printf("\rQuerying indexer for %s...\n", ret.PayloadCid())
case types.QueryPhase:
fmt.Printf("\rQuerying [%s] (%s)...\n", ret.StorageProviderId(), ret.Code())
fmt.Printf("\rQuerying [%s] (%s)...\n", types.Identifier(ret), ret.Code())
case types.RetrievalPhase:
fmt.Printf("\rRetrieving from [%s] (%s)...\n", ret.StorageProviderId(), ret.Code())
fmt.Printf("\rRetrieving from [%s] (%s)...\n", types.Identifier(ret), ret.Code())
}
case events.RetrievalEventConnected:
switch ret.Phase() {
case types.QueryPhase:
fmt.Printf("\rQuerying [%s] (%s)...\n", ret.StorageProviderId(), ret.Code())
fmt.Printf("\rQuerying [%s] (%s)...\n", types.Identifier(ret), ret.Code())
case types.RetrievalPhase:
fmt.Printf("\rRetrieving from [%s] (%s)...\n", ret.StorageProviderId(), ret.Code())
fmt.Printf("\rRetrieving from [%s] (%s)...\n", types.Identifier(ret), ret.Code())
}
case events.RetrievalEventProposed:
fmt.Printf("\rRetrieving from [%s] (%s)...\n", ret.StorageProviderId(), ret.Code())
fmt.Printf("\rRetrieving from [%s] (%s)...\n", types.Identifier(ret), ret.Code())
case events.RetrievalEventAccepted:
fmt.Printf("\rRetrieving from [%s] (%s)...\n", ret.StorageProviderId(), ret.Code())
fmt.Printf("\rRetrieving from [%s] (%s)...\n", types.Identifier(ret), ret.Code())
case events.RetrievalEventFirstByte:
fmt.Printf("\rRetrieving from [%s] (%s)...\n", ret.StorageProviderId(), ret.Code())
fmt.Printf("\rRetrieving from [%s] (%s)...\n", types.Identifier(ret), ret.Code())
case events.RetrievalEventCandidatesFound:
pp.candidatesFound = len(ret.Candidates())
case events.RetrievalEventCandidatesFiltered:
Expand All @@ -204,17 +204,17 @@ func (pp *progressPrinter) subscriber(event types.RetrievalEvent) {
fmt.Printf("Using the explicitly specified storage provider, querying %s:\n", num)
}
for _, candidate := range ret.Candidates() {
fmt.Printf("\r\t%s\n", candidate.MinerPeer.ID)
fmt.Printf("\r\t%s, Protocols: %v\n", candidate.MinerPeer.ID, candidate.Metadata.Protocols())
}
case events.RetrievalEventQueryAsked:
fmt.Printf("\rGot query response from [%s] (checking): size=%s, price-per-byte=%s, unseal-price=%s, message=%s\n", ret.StorageProviderId(), humanize.IBytes(ret.QueryResponse().Size), ret.QueryResponse().MinPricePerByte, ret.QueryResponse().UnsealPrice, ret.QueryResponse().Message)
fmt.Printf("\rGot query response from [%s] (checking): size=%s, price-per-byte=%s, unseal-price=%s, message=%s\n", types.Identifier(ret), humanize.IBytes(ret.QueryResponse().Size), ret.QueryResponse().MinPricePerByte, ret.QueryResponse().UnsealPrice, ret.QueryResponse().Message)
case events.RetrievalEventQueryAskedFiltered:
fmt.Printf("\rGot query response from [%s] (filtered): size=%s, price-per-byte=%s, unseal-price=%s, message=%s\n", ret.StorageProviderId(), humanize.IBytes(ret.QueryResponse().Size), ret.QueryResponse().MinPricePerByte, ret.QueryResponse().UnsealPrice, ret.QueryResponse().Message)
fmt.Printf("\rGot query response from [%s] (filtered): size=%s, price-per-byte=%s, unseal-price=%s, message=%s\n", types.Identifier(ret), humanize.IBytes(ret.QueryResponse().Size), ret.QueryResponse().MinPricePerByte, ret.QueryResponse().UnsealPrice, ret.QueryResponse().Message)
case events.RetrievalEventFailed:
if ret.Phase() == types.IndexerPhase {
fmt.Printf("\rRetrieval failure from indexer: %s\n", ret.ErrorMessage())
} else {
fmt.Printf("\rRetrieval failure for [%s]: %s\n", ret.StorageProviderId(), ret.ErrorMessage())
fmt.Printf("\rRetrieval failure for [%s]: %s\n", types.Identifier(ret), ret.ErrorMessage())
}
case events.RetrievalEventSuccess:
// noop, handled at return from Retrieve()
Expand Down
2 changes: 2 additions & 0 deletions cmd/lassie/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,8 @@ func before(cctx *cli.Context) error {
subsystems := []string{
"lassie",
"lassie/httpserver",
"indexerlookup",
"lassie/bitswap",
}

level := "WARN"
Expand Down
78 changes: 48 additions & 30 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ require (
github.com/benbjohnson/clock v1.3.0
github.com/dustin/go-humanize v1.0.0
github.com/filecoin-project/go-address v1.1.0
github.com/filecoin-project/go-data-transfer/v2 v2.0.0-rc2
github.com/filecoin-project/go-data-transfer/v2 v2.0.0-rc2.0.20230214154556-9cac4a18a101
github.com/filecoin-project/go-fil-markets v1.25.3-0.20230107010325-143abaddd0f3
github.com/filecoin-project/go-state-types v0.9.9
github.com/filecoin-project/index-provider v0.9.2
Expand All @@ -18,12 +18,17 @@ require (
github.com/ipfs/go-cid v0.3.2
github.com/ipfs/go-datastore v0.6.0
github.com/ipfs/go-graphsync v0.14.0
github.com/ipfs/go-ipfs-blockstore v1.2.0
github.com/ipfs/go-ipfs-blocksutil v0.0.1
github.com/ipfs/go-ipfs-delay v0.0.1
github.com/ipfs/go-ipld-format v0.4.0
github.com/ipfs/go-libipfs v0.5.0
github.com/ipfs/go-log/v2 v2.5.1
github.com/ipld/go-car/v2 v2.7.0
github.com/ipld/go-ipld-prime v0.19.1-0.20230210001044-7b00b1490f0b
github.com/ipld/go-ipld-prime v0.20.0
github.com/ipni/storetheindex v0.5.4
github.com/libp2p/go-libp2p v0.23.4
github.com/libp2p/go-libp2p v0.25.1
github.com/libp2p/go-libp2p-testing v0.12.0
github.com/multiformats/go-multiaddr v0.8.0
github.com/multiformats/go-multicodec v0.8.0
github.com/multiformats/go-multihash v0.2.1
Expand All @@ -38,13 +43,19 @@ require (
golang.org/x/net v0.5.0
)

require (
github.com/ipfs/go-ipfs-chunker v0.0.5 // indirect
github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect
)

require (
github.com/beorn7/perks v1.0.1 // indirect
github.com/bep/debounce v1.2.1 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/containerd/cgroups v1.0.4 // indirect
github.com/coreos/go-systemd/v22 v22.4.0 // indirect
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
github.com/cskr/pubsub v1.0.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect
Expand All @@ -60,7 +71,6 @@ require (
github.com/filecoin-project/go-statestore v0.2.0 // indirect
github.com/flynn/noise v1.0.0 // indirect
github.com/francoispqt/gojay v1.2.13 // indirect
github.com/fsnotify/fsnotify v1.5.4 // indirect
github.com/go-kit/log v0.2.1 // indirect
github.com/go-logfmt/logfmt v0.5.1 // indirect
github.com/go-logr/logr v1.2.3 // indirect
Expand All @@ -72,32 +82,35 @@ require (
github.com/golang/mock v1.6.0 // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/google/gopacket v1.1.19 // indirect
github.com/gorilla/websocket v1.5.0 // indirect
github.com/hannahhoward/cbor-gen-for v0.0.0-20200817222906-ea96cece81f1 // indirect
github.com/google/pprof v0.0.0-20221203041831-ce31453925ec // indirect
github.com/hannahhoward/cbor-gen-for v0.0.0-20230214144701-5d17c9d5243c // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/golang-lru v0.5.4 // indirect
github.com/huin/goupnp v1.0.3 // indirect
github.com/ipfs/bbloom v0.0.4 // indirect
github.com/ipfs/go-bitfield v1.1.0 // indirect
github.com/ipfs/go-block-format v0.1.1 // indirect
github.com/ipfs/go-blockservice v0.5.0 // indirect
github.com/ipfs/go-ipfs-blockstore v1.2.0 // indirect
github.com/ipfs/go-blockservice v0.5.0
github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect
github.com/ipfs/go-ipfs-exchange-interface v0.2.0 // indirect
github.com/ipfs/go-ipfs-exchange-interface v0.2.0
github.com/ipfs/go-ipfs-pq v0.0.2 // indirect
github.com/ipfs/go-ipfs-util v0.0.2 // indirect
github.com/ipfs/go-ipld-cbor v0.0.6 // indirect
github.com/ipfs/go-ipld-legacy v0.1.1 // indirect
github.com/ipfs/go-libipfs v0.4.0 // indirect
github.com/ipfs/go-log v1.0.5 // indirect
github.com/ipfs/go-merkledag v0.9.0 // indirect
github.com/ipfs/go-metrics-interface v0.0.1 // indirect
github.com/ipfs/go-peertaskqueue v0.8.0 // indirect
github.com/ipfs/go-unixfsnode v1.5.2
github.com/ipfs/go-verifcid v0.0.2 // indirect
github.com/ipld/go-codec-dagpb v1.5.0 // indirect
github.com/ipld/go-codec-dagpb v1.5.0
github.com/jackpal/go-nat-pmp v1.0.2 // indirect
github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c // indirect
github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect
github.com/jbenet/goprocess v0.1.4 // indirect
github.com/jpillora/backoff v1.0.0 // indirect
github.com/klauspost/compress v1.15.10 // indirect
github.com/klauspost/compress v1.15.12 // indirect
github.com/klauspost/cpuid/v2 v2.2.3 // indirect
github.com/koron/go-ssdp v0.0.3 // indirect
github.com/kr/pretty v0.3.1 // indirect
Expand All @@ -106,19 +119,16 @@ require (
github.com/libp2p/go-cidranger v1.1.0 // indirect
github.com/libp2p/go-flow-metrics v0.1.0 // indirect
github.com/libp2p/go-libp2p-asn-util v0.2.0 // indirect
github.com/libp2p/go-msgio v0.2.0 // indirect
github.com/libp2p/go-libp2p-record v0.2.0 // indirect
github.com/libp2p/go-libp2p-routing-helpers v0.6.1
github.com/libp2p/go-msgio v0.3.0 // indirect
github.com/libp2p/go-nat v0.1.0 // indirect
github.com/libp2p/go-netroute v0.2.0 // indirect
github.com/libp2p/go-openssl v0.1.0 // indirect
github.com/libp2p/go-netroute v0.2.1 // indirect
github.com/libp2p/go-reuseport v0.2.0 // indirect
github.com/libp2p/go-yamux/v4 v4.0.0 // indirect
github.com/lucas-clemente/quic-go v0.29.1 // indirect
github.com/marten-seemann/qtls-go1-18 v0.1.2 // indirect
github.com/marten-seemann/qtls-go1-19 v0.1.0 // indirect
github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect
github.com/mattn/go-isatty v0.0.17 // indirect
github.com/mattn/go-pointer v0.0.1 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/miekg/dns v1.1.50 // indirect
github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect
github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect
Expand All @@ -130,10 +140,9 @@ require (
github.com/multiformats/go-multiaddr-dns v0.3.1 // indirect
github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect
github.com/multiformats/go-multibase v0.1.1 // indirect
github.com/multiformats/go-multistream v0.3.3 // indirect
github.com/multiformats/go-multistream v0.4.1 // indirect
github.com/multiformats/go-varint v0.0.7 // indirect
github.com/nxadm/tail v1.4.8 // indirect
github.com/onsi/ginkgo v1.16.5 // indirect
github.com/onsi/ginkgo/v2 v2.5.1 // indirect
github.com/opencontainers/runtime-spec v1.0.2 // indirect
github.com/opentracing/opentracing-go v1.2.0 // indirect
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
Expand All @@ -145,26 +154,35 @@ require (
github.com/prometheus/common v0.37.0 // indirect
github.com/prometheus/procfs v0.8.0 // indirect
github.com/prometheus/statsd_exporter v0.22.7 // indirect
github.com/quic-go/qpack v0.4.0 // indirect
github.com/quic-go/qtls-go1-18 v0.2.0 // indirect
github.com/quic-go/qtls-go1-19 v0.2.0 // indirect
github.com/quic-go/qtls-go1-20 v0.1.0 // indirect
github.com/quic-go/quic-go v0.32.0 // indirect
github.com/quic-go/webtransport-go v0.5.1 // indirect
github.com/raulk/go-watchdog v1.3.0 // indirect
github.com/rogpeppe/go-internal v1.9.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 // indirect
github.com/spaolacci/murmur3 v1.1.0 // indirect
github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 // indirect
github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa // indirect
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
go.opentelemetry.io/otel/sdk v1.10.0 // indirect
go.uber.org/atomic v1.10.0 // indirect
go.uber.org/dig v1.15.0 // indirect
go.uber.org/fx v1.18.2 // indirect
go.uber.org/zap v1.24.0 // indirect
golang.org/x/crypto v0.5.0 // indirect
golang.org/x/exp v0.0.0-20230129154200-a960b3787bd2 // indirect
golang.org/x/mod v0.6.0 // indirect
golang.org/x/sync v0.0.0-20220907140024-f12130a52804 // indirect
golang.org/x/mod v0.7.0 // indirect
golang.org/x/sync v0.1.0 // indirect
golang.org/x/sys v0.4.0 // indirect
golang.org/x/tools v0.2.0 // indirect
golang.org/x/text v0.6.0 // indirect
golang.org/x/tools v0.3.0 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
google.golang.org/protobuf v1.28.1 // indirect
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
lukechampine.com/blake3 v1.1.7 // indirect
nhooyr.io/websocket v1.8.7 // indirect
)
Loading