Skip to content

Commit

Permalink
split tests into selector / no-selector
Browse files Browse the repository at this point in the history
  • Loading branch information
@nonsense
nonsense committed Dec 4, 2023
1 parent 40fdb81 commit a98b0d7
Showing 1 changed file with 166 additions and 48 deletions.
214 changes: 166 additions & 48 deletions itests/graphsync_retrieval_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ package itests

import (
"context"
"math"
"path/filepath"
"testing"
"time"
Expand All @@ -14,14 +13,13 @@ import (
"github.com/filecoin-project/lotus/itests/kit"
"github.com/google/uuid"
"github.com/ipfs/go-cid"
"github.com/ipfs/go-unixfsnode"
"github.com/ipld/go-ipld-prime/datamodel"
"github.com/ipld/go-ipld-prime/node/basicnode"
"github.com/ipld/go-ipld-prime/traversal/selector/builder"
"github.com/stretchr/testify/require"
)

func TestDealRetrieval(t *testing.T) {
func TestDealRetrievalFull(t *testing.T) {
ctx := context.Background()
log := framework.Log

Expand Down Expand Up @@ -114,62 +112,182 @@ func TestDealRetrieval(t *testing.T) {
// Deal is stored and sealed, attempt different retrieval forms

retrievalCases := []struct {
name string
selector datamodel.Node
matcherFrom, matcherTo int64
expectCids []cid.Cid
name string
expectCids []cid.Cid
}{
{
name: "full file, explore-all",
selector: unixfsnode.UnixFSPathSelectorBuilder("", unixfsnode.ExploreAllRecursivelySelector, false),
expectCids: append([]cid.Cid{root}, leaves...),
},
}

for _, tc := range retrievalCases {
t.Run(tc.name, func(t *testing.T) {
log.Debugw("deal is sealed, starting retrieval", "cid", dealCid.String(), "root", root)
outPath := f.Retrieve(ctx, t, tempdir, root, res.DealParams.ClientDealProposal.Proposal.PieceCID, false, nil)

// Inspect what we got
gotCids, err := testutil.CidsInCar(outPath)
require.NoError(t, err)

toStr := func(c []cid.Cid) []string {
// for nicer debugging given that we know the CIDs we expect
out := make([]string, len(c))
for i, v := range c {
out[i] = v.String()
}
return out
}

require.Equal(t, toStr(tc.expectCids), toStr(gotCids))
})
}
}

func TestDealRetrievalSelector(t *testing.T) {
ctx := context.Background()
log := framework.Log

kit.QuietMiningLogs()
framework.SetLogLevel()
var opts []framework.FrameworkOpts
opts = append(opts, framework.EnableLegacyDeals(true))
opts = append(opts, framework.WithMaxStagingDealsBytes(10000000))
f := framework.NewTestFramework(ctx, t, opts...)
err := f.Start()
require.NoError(t, err)
defer f.Stop()

err = f.AddClientProviderBalance(abi.NewTokenAmount(1e15))
require.NoError(t, err)

// Create a CAR file
tempdir := t.TempDir()
log.Debugw("using tempdir", "dir", tempdir)
rseed := 0
size := 7 << 20 // 7MiB file

inPath, dirEnt := testutil.CreateRandomUnixfsFileInCar(t, tempdir, rseed, size)
root := dirEnt.Root
leaves := dirEnt.SelfCids[:len(dirEnt.SelfCids)-1]
/*
For a 7MiB file with seed 0, we expect to have this structure which we
can perform range selections on. The root is dirEnt.Root, the raw leaf
blocks are dirEnt.SelfCid, in order, minus the last one which is also
the root.
bafybeiet25r42mzboo5osbxmze5gq45zb4peqksscavjb4ixycwjkbwioa | File | /[0:7340031] (7340032 B)
bafkreifrdfke6yoyrjhhqbwkddacapg42vuzcbvwyau6ol24fbjxgfrsha | RawLeaf | ↳ /0[0:256143] (256144 B)
bafkreigmdnzqhuaaqrkzhwo2cqa7heshji5neoygkqku2vrubuvzz4twti | RawLeaf | /0[256144:512287] (256144 B)
bafkreidoafwiruc3tqk4fsvql6jfjhsi36pvmr334qwvfjzupkjnws2c7a | RawLeaf | /0[512288:768431] (256144 B)
bafkreibcmiojncftk5kbjo4xdv4onlxacg6dfi3d3objfl4y7dqhld4a3e | RawLeaf | /0[768432:1024575] (256144 B)
bafkreidpvdovcyfj4ypodcw47xgodpkwlq7yokmofzpxto5v3fjcwtaerm | RawLeaf | /0[1024576:1280719] (256144 B)
bafkreicm6tt6ahmimqwuoxuc7m2jdlmlexiofxmsigfsvi4nsuda45kfpu | RawLeaf | /0[1280720:1536863] (256144 B)
bafkreibnq24344buagftasgfpeomzjf6bbnhku4sw4vv3lnisx7wmvv7de | RawLeaf | /0[1536864:1793007] (256144 B)
bafkreieeqrvyz7z3ezshaqlp24zd5jfnmlzb2gnsuie3rltyhxlyb3f46m | RawLeaf | /0[1793008:2049151] (256144 B)
bafkreiaee3232qc3f3dsv5hezroy27va3cmcf35evzkw426liggu4qqguq | RawLeaf | /0[2049152:2305295] (256144 B)
bafkreiccfzitpdvr3d2tyo2ly6e3qjqcc6vmnr4jfir3ys3zxsvaviaqmy | RawLeaf | /0[2305296:2561439] (256144 B)
bafkreiek37qlpud6koj4jybkrwudic7bvdkraion37xk2x7hxmwyrqxzdi | RawLeaf | /0[2561440:2817583] (256144 B)
bafkreifdib756bpqzucggrqp3tbv3c2mvf3kl6ir5erscbsmxt35lv3qf4 | RawLeaf | /0[2817584:3073727] (256144 B)
bafkreiaflhrfx65ukovmuacvktlopj3j5d6iacvmxgi5vyyelqe77laxda | RawLeaf | /0[3073728:3329871] (256144 B)
bafkreia43b3mtjeo5ycrur5dmj72wpktnt5r2tt6u26djl3syuwt2wirmq | RawLeaf | /0[3329872:3586015] (256144 B)
bafkreic5vfpokxgadhzrx5s2yjoqtif6px6hlkqndvgscd3kgxf22omogy | RawLeaf | /0[3586016:3842159] (256144 B)
bafkreibejeisigfddgxjq3gqirbsjcaikpm5ukp6hyelcdarim3mw4gusm | RawLeaf | /0[3842160:4098303] (256144 B)
bafkreibua7gdtdndlwdpntzdzoleomfy4cuawbou2ibqcwdiqcd5c2cktq | RawLeaf | /0[4098304:4354447] (256144 B)
bafkreihce3ojxvomz2afkl4sr67qlje3vvxqcnuugb2mekjocuf2m3axh4 | RawLeaf | /0[4354448:4610591] (256144 B)
bafkreih6gxdebrtqukp7dgpz6uq4mcljkk42aont4f2fmu7yw3qvst5s44 | RawLeaf | /0[4610592:4866735] (256144 B)
bafkreihsyox3ebuboypzrzdnja7mypcq7lugd7yajnyxswaphoqrfj3mvi | RawLeaf | /0[4866736:5122879] (256144 B)
bafkreihqf3wqz5tv3vqlk7s2no7hchw2327ptwowfak773uvddarif2lzi | RawLeaf | /0[5122880:5379023] (256144 B)
bafkreig6hrykcrosz7jwljalgb2bhdh3r2e43gatvy2q3zn7nbck77dnou | RawLeaf | /0[5379024:5635167] (256144 B)
bafkreihblmdcjlmwp5p4esqn6e2h6ko5oytxilpb3msosjjdw5eaarakye | RawLeaf | /0[5635168:5891311] (256144 B)
bafkreihfmufz2fss7oqxqg2gsz4ow5l7i3byhzuqmxev73bu5fi4vdklc4 | RawLeaf | /0[5891312:6147455] (256144 B)
bafkreid3smar6sdcmi7io6e3ewdv3b3gdcyq7yrdznounhsrkswjdmptoa | RawLeaf | /0[6147456:6403599] (256144 B)
bafkreif54hyudjfodmtdphrhass4rxrrle36xvzj5osfcuue52ck4i7pr4 | RawLeaf | /0[6403600:6659743] (256144 B)
bafkreideyfbj2ljnows5gxnsibvpziyhr26azpae6ilesnhhaissbx5ori | RawLeaf | /0[6659744:6915887] (256144 B)
bafkreighy3cp6pvp3pluq4ragvgrycom4s5upargqkawcyu3rw4flturky | RawLeaf | /0[6915888:7172031] (256144 B)
bafkreifokzy5zcluf3hj23nkrvr7tx6sivpshkd4be5tpfibk6vm2mzlxy | RawLeaf | /0[7172032:7340031] (168000 B)
*/

// Start a web server to serve the car files
log.Debug("starting webserver")
server, err := testutil.HttpTestFileServer(t, tempdir)
require.NoError(t, err)
defer server.Close()

// Create a new dummy deal
log.Debug("creating dummy deal")
dealUuid := uuid.New()

// Make a deal
res, err := f.MakeDummyDeal(dealUuid, inPath, root, server.URL+"/"+filepath.Base(inPath), false)
require.NoError(t, err)
require.True(t, res.Result.Accepted)
log.Debugw("got response from MarketDummyDeal", "res", spew.Sdump(res))
dealCid, err := res.DealParams.ClientDealProposal.Proposal.Cid()
require.NoError(t, err)

time.Sleep(2 * time.Second)

log.Debugw("got deal proposal cid", "cid", dealCid.String())

err = f.WaitForDealAddedToSector(res.DealParams.DealUUID)
require.NoError(t, err)

// Deal is stored and sealed, attempt different retrieval forms

retrievalCases := []struct {
name string
selector datamodel.Node
matcherFrom, matcherTo int64
expectCids []cid.Cid
}{
{
name: "slice: 0 to 7MiB",
matcherFrom: 0,
matcherTo: 7 << 20,
expectCids: append([]cid.Cid{root}, leaves...),
},
{
name: "slice: 1MiB to 2MiB",
matcherFrom: 1 << 20,
matcherTo: 2 << 20,
expectCids: append([]cid.Cid{root}, leaves[4:9]...),
},
{
name: "slice: first byte",
matcherFrom: 0,
matcherTo: 1,
expectCids: append([]cid.Cid{root}, leaves[0]),
},
{
name: "slice: last byte",
matcherFrom: 7340031,
matcherTo: 7340032,
expectCids: append([]cid.Cid{root}, leaves[len(leaves)-1]),
},
{
name: "slice: last two blocks, negative range, boundary",
matcherFrom: -168000 - 1,
matcherTo: math.MaxInt64,
expectCids: append([]cid.Cid{root}, leaves[len(leaves)-2:]...),
},
{
name: "slice: last block, negative range, boundary",
matcherFrom: -168000,
matcherTo: math.MaxInt64,
expectCids: append([]cid.Cid{root}, leaves[len(leaves)-1]),
},
{
// In this case we are attempting to traverse beyond the file to a
// path that doesn't exist; we expect to only match the root and
// return that. This is not strictly an error case, it's up to the
// consumer of this data to verify the path doesn't resolve in the
// data they get back.
name: "path beyond file",
selector: unixfsnode.UnixFSPathSelectorBuilder("not/a/path", unixfsnode.ExploreAllRecursivelySelector, false),
expectCids: []cid.Cid{root},
},
//{
//name: "slice: 1MiB to 2MiB",
//matcherFrom: 1 << 20,
//matcherTo: 2 << 20,
//expectCids: append([]cid.Cid{root}, leaves[4:9]...),
//},
//{
//name: "slice: first byte",
//matcherFrom: 0,
//matcherTo: 1,
//expectCids: append([]cid.Cid{root}, leaves[0]),
//},
//{
//name: "slice: last byte",
//matcherFrom: 7340031,
//matcherTo: 7340032,
//expectCids: append([]cid.Cid{root}, leaves[len(leaves)-1]),
//},
//{
//name: "slice: last two blocks, negative range, boundary",
//matcherFrom: -168000 - 1,
//matcherTo: math.MaxInt64,
//expectCids: append([]cid.Cid{root}, leaves[len(leaves)-2:]...),
//},
//{
//name: "slice: last block, negative range, boundary",
//matcherFrom: -168000,
//matcherTo: math.MaxInt64,
//expectCids: append([]cid.Cid{root}, leaves[len(leaves)-1]),
//},
//{
//// In this case we are attempting to traverse beyond the file to a
//// path that doesn't exist; we expect to only match the root and
//// return that. This is not strictly an error case, it's up to the
//// consumer of this data to verify the path doesn't resolve in the
//// data they get back.
//name: "path beyond file",
//selector: unixfsnode.UnixFSPathSelectorBuilder("not/a/path", unixfsnode.ExploreAllRecursivelySelector, false),
//expectCids: []cid.Cid{root},
//},
}

for _, tc := range retrievalCases {
Expand Down

0 comments on commit a98b0d7

Please sign in to comment.