direct data onboarding - disabled (#1627)

* add dummy direct deals provider * add initial `import-direct` cmd * updated make gen and make docsgen * direct deals provider * steps to onboard direct data * sql table; direct data entry state; ORM * scaffolding up to Filecoin.SectorAddPieceToAny * rename DirectData to DirectDeal * remove docker-tc * fix AllocationId type to uint64 * remove VerifiedDeal field in DirectDeal table * update ifaces to add fast-retrieval and skip-ipni-announce * fast-retrieval becomes remove-unsealed-copy * add checkpoints to direct deal model * initial state into local db * add extra fields * feat: add `boost allocate` and `boost get-allocations` command (#1632) * add allocate & get-allocations command * comment out fullNode dependent lines * clarify error * update lotus, uncomment check code * go mod tidy * bump FFI, fix EthCall * docsgen * direct data onboarding: refactor/extract commp, use checkpoints for retries, retries on startup (#1650) * extract and refactor commp for direct deals * split Import into Import and Process, and Process all deals on startup * update Process with checkpoints check * handle commp size * devnet support for filplus (#1655) * comment * add signers and notaries * fixup * fixup * get allocation * lock lotus version * lock lotus to latest PR version * fill PieceDealInfo * bump up datacap * check for nil allocation; set ClientAddress and ProviderAddress in sqlitedb * take actor ID for a wallet from state * fixup * add start and end epoch * add startEpochSealingBuffer and compare startEpoch against chain head * comments and fixup * commpcalc; listActive * fix: doc gen * Web UI: direct deals list (#1665) * feat: ui - direct deals list * feat: direct deals ui - direct deal detail page (#1669) * fix: test compilation * fix: cbor gen * refactor: simplify db code * feat: ui - add direct deals link to menu * fix: direct deals menu item * feat: add more deal logs * feat: user defined allocation policy (#1668) * user defined allocation policy * refactor flags * fix typo * fix devnet LID, fix flag names * refactor: extract piece reading code into common function * feat: add deal to sector with retry * feat: add retryability to direct deals (#1670) * fix nil ptr exception due to missing datacap * fix: allocation bug, refactor func()s (#1673) * fix allocation bug, refactor func()s * fix lint err * feat: make retry / fail resolvers work with direct deals (#1675) * create DDProvider config (#1679) * add itest for direct data onboarding (#1684) * feat: add itest for direct data onboarding * refactor: move datacap code from ddo test function into itest framework * Watch sealing state until finalized (#1687) * feat: watch sealing state until finalized * fix: dont check deal id when watching sealing state for direct deals * fix: checking sealing sector state * refactor: use common commp throttle for direct and regular deals (#1695) * feat: watch sealing state until finalized * fix: dont check deal id when watching sealing state for direct deals * fix: checking sealing sector state * refactor: use common commp throttle for direct and regular deals * feat: add tests for direct deals db (#1700) * feat: index and announce direct deals (#1696) * index and announce * add IsDirectDeal to LID * fix new column type * add keyspace to cql * rm keyspace from add column * Use UUID for idxProv context * apply suggestions * add column in migration * fix lint err * update ipni announce log line * revert to lotus version without ddo * go.mod from main * comment out ddo-related code; revert itest framework * comment out itest for direct deals * cleanup * Update node/impl/boost.go Co-authored-by: LexLuthr <[email protected]> * fix commented out direct deals component * comment out direct deals cmd * fix cbor-gen --------- Co-authored-by: LexLuthr <[email protected]> Co-authored-by: Dirk McCormick <[email protected]> Co-authored-by: LexLuthr <[email protected]>
filecoin-project · Oct 5, 2023 · 06d02cc · 06d02cc
1 parent fc59a5d
commit 06d02cc
Show file tree

Hide file tree

Showing 63 changed files with 3,986 additions and 367 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -343,6 +343,11 @@ workflows:
           suite: itest-disabled_markets_v1_deal
           target: "./itests/disabled_markets_v1_deal_test.go"
 
+      #- test:
+          #name: test-itest-direct_deal
+          #suite: itest-direct_deal
+          #target: "./itests/ddo_test.go"
+
       - test:
           name: test-all
           suite: all

diff --git a/api/api.go b/api/api.go
@@ -50,6 +50,7 @@ type Boost interface {
 	BoostDagstorePiecesContainingMultihash(ctx context.Context, mh multihash.Multihash) ([]cid.Cid, error)                                      //perm:read
 	BoostDagstoreListShards(ctx context.Context) ([]DagstoreShardInfo, error)                                                                   //perm:admin
 	BoostMakeDeal(context.Context, smtypes.DealParams) (*ProviderDealRejectionInfo, error)                                                      //perm:write
+	BoostDirectDeal(ctx context.Context, params smtypes.DirectDealParams) (*ProviderDealRejectionInfo, error)                                   //perm:admin
 
 	// MethodGroup: Blockstore
 	BlockstoreGet(ctx context.Context, c cid.Cid) ([]byte, error)  //perm:read

diff --git a/api/proxy_gen.go b/api/proxy_gen.go
diff --git a/build/openrpc/boost.json.gz b/build/openrpc/boost.json.gz
diff --git a/cmd/boost/direct_deal.god b/cmd/boost/direct_deal.god
@@ -0,0 +1,287 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	bcli "github.com/filecoin-project/boost/cli"
+	clinode "github.com/filecoin-project/boost/cli/node"
+	"github.com/filecoin-project/boost/cmd"
+	"github.com/filecoin-project/boost/cmd/boost/util"
+	"github.com/filecoin-project/boost/cmd/lib"
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-state-types/abi"
+	verifregst "github.com/filecoin-project/go-state-types/builtin/v9/verifreg"
+	lapi "github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/chain/actors/builtin/verifreg"
+	"github.com/filecoin-project/lotus/chain/types"
+	lcli "github.com/filecoin-project/lotus/cli"
+	"github.com/filecoin-project/lotus/lib/tablewriter"
+	"github.com/urfave/cli/v2"
+)
+
+var directDealAllocate = &cli.Command{
+	Name:  "allocate",
+	Usage: "Create new allocation[s] for verified deals",
+	Flags: []cli.Flag{
+		&cli.StringSliceFlag{
+			Name:     "miner",
+			Usage:    "storage provider address[es]",
+			Required: true,
+			Aliases:  []string{"m", "provider", "p"},
+		},
+		&cli.StringSliceFlag{
+			Name:     "piece-info",
+			Usage:    "data piece-info[s] to create the allocation. The format must be --piece-info pieceCid1=pieceSize1 --piece-info pieceCid2=pieceSize2",
+			Required: true,
+			Aliases:  []string{"pi"},
+		},
+		&cli.StringFlag{
+			Name:  "wallet",
+			Usage: "the wallet address that will used create the allocation",
+		},
+		&cli.BoolFlag{
+			Name:  "quiet",
+			Usage: "do not print the allocation list",
+			Value: false,
+		},
+		&cli.Int64Flag{
+			Name: "term-min",
+			Usage: "The minimum duration which the provider must commit to storing the piece to avoid early-termination penalties (epochs).\n" +
+				"Default is 180 days.",
+			Aliases: []string{"tmin"},
+			Value:   verifregst.MinimumVerifiedAllocationTerm,
+		},
+		&cli.Int64Flag{
+			Name: "term-max",
+			Usage: "The maximum period for which a provider can earn quality-adjusted power for the piece (epochs).\n" +
+				"Default is 5 years.",
+			Aliases: []string{"tmax"},
+			Value:   verifregst.MaximumVerifiedAllocationTerm,
+		},
+		&cli.Int64Flag{
+			Name: "expiration",
+			Usage: "The latest epoch by which a provider must commit data before the allocation expires (epochs).\n" +
+				"Default is 60 days.",
+			Value: verifregst.MaximumVerifiedAllocationExpiration,
+		},
+	},
+	Before: before,
+	Action: func(cctx *cli.Context) error {
+		ctx := bcli.ReqContext(cctx)
+
+		n, err := clinode.Setup(cctx.String(cmd.FlagRepo.Name))
+		if err != nil {
+			return err
+		}
+
+		gapi, closer, err := lcli.GetGatewayAPI(cctx)
+		if err != nil {
+			return fmt.Errorf("can't setup gateway connection: %w", err)
+		}
+		defer closer()
+
+		// Get wallet address from input
+		walletAddr, err := n.GetProvidedOrDefaultWallet(ctx, cctx.String("wallet"))
+		if err != nil {
+			return err
+		}
+
+		log.Debugw("selected wallet", "wallet", walletAddr)
+
+		msg, err := util.CreateAllocationMsg(ctx, gapi, cctx.StringSlice("piece-info"), cctx.StringSlice("miner"), walletAddr, abi.ChainEpoch(cctx.Int64("term-min")), abi.ChainEpoch(cctx.Int64("term-max")), abi.ChainEpoch(cctx.Int64("expiration")))
+
+		if err != nil {
+			return err
+		}
+
+		oldallocations, err := gapi.StateGetAllocations(ctx, walletAddr, types.EmptyTSK)
+		if err != nil {
+			return fmt.Errorf("failed to get allocations: %w", err)
+		}
+
+		mcid, sent, err := lib.SignAndPushToMpool(cctx, ctx, gapi, n, msg)
+		if err != nil {
+			return err
+		}
+		if !sent {
+			return nil
+		}
+
+		log.Infow("submitted data cap allocation message", "cid", mcid.String())
+		log.Info("waiting for message to be included in a block")
+
+		res, err := gapi.StateWaitMsg(ctx, mcid, 1, lapi.LookbackNoLimit, true)
+		if err != nil {
+			return fmt.Errorf("waiting for message to be included in a block: %w", err)
+		}
+
+		if !res.Receipt.ExitCode.IsSuccess() {
+			return fmt.Errorf("failed to execute the message with error: %s", res.Receipt.ExitCode.Error())
+		}
+
+		// Return early of quiet flag is set
+		if cctx.Bool("quiet") {
+			return nil
+		}
+
+		newallocations, err := gapi.StateGetAllocations(ctx, walletAddr, types.EmptyTSK)
+		if err != nil {
+			return fmt.Errorf("failed to get allocations: %w", err)
+		}
+
+		// Generate a diff to find new allocations
+		for i := range newallocations {
+			_, ok := oldallocations[i]
+			if ok {
+				delete(newallocations, i)
+			}
+		}
+
+		return printAllocation(newallocations, cctx.Bool("json"))
+	},
+}
+
+var directDealGetAllocations = &cli.Command{
+	Name:  "list-allocations",
+	Usage: "Lists all allocations for a client address(wallet)",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:    "miner",
+			Usage:   "Storage provider address. If provided, only allocations against this minerID will be printed",
+			Aliases: []string{"m", "provider", "p"},
+		},
+		&cli.StringFlag{
+			Name:  "wallet",
+			Usage: "the wallet address that will used create the allocation",
+		},
+	},
+	Before: before,
+	Action: func(cctx *cli.Context) error {
+		ctx := bcli.ReqContext(cctx)
+
+		n, err := clinode.Setup(cctx.String(cmd.FlagRepo.Name))
+		if err != nil {
+			return err
+		}
+
+		gapi, closer, err := lcli.GetGatewayAPI(cctx)
+		if err != nil {
+			return fmt.Errorf("cant setup gateway connection: %w", err)
+		}
+		defer closer()
+
+		// Get wallet address from input
+		walletAddr, err := n.GetProvidedOrDefaultWallet(ctx, cctx.String("wallet"))
+		if err != nil {
+			return err
+		}
+
+		log.Debugw("selected wallet", "wallet", walletAddr)
+
+		allocations, err := gapi.StateGetAllocations(ctx, walletAddr, types.EmptyTSK)
+		if err != nil {
+			return fmt.Errorf("failed to get allocations: %w", err)
+		}
+
+		if cctx.String("miner") != "" {
+			// Get all minerIDs from input
+			minerId := cctx.String("miner")
+			maddr, err := address.NewFromString(minerId)
+			if err != nil {
+				return err
+			}
+
+			// Verify that minerID exists
+			_, err = gapi.StateMinerInfo(ctx, maddr, types.EmptyTSK)
+			if err != nil {
+				return err
+			}
+
+			mid, err := address.IDFromAddress(maddr)
+			if err != nil {
+				return err
+			}
+
+			for i, v := range allocations {
+				if v.Provider != abi.ActorID(mid) {
+					delete(allocations, i)
+				}
+			}
+		}
+
+		return printAllocation(allocations, cctx.Bool("json"))
+	},
+}
+
+func printAllocation(allocations map[verifreg.AllocationId]verifreg.Allocation, json bool) error {
+	// Map Keys. Corresponds to the standard tablewriter output
+	allocationID := "AllocationID"
+	client := "Client"
+	provider := "Miner"
+	pieceCid := "PieceCid"
+	pieceSize := "PieceSize"
+	tMin := "TermMin"
+	tMax := "TermMax"
+	expr := "Expiration"
+
+	// One-to-one mapping between tablewriter keys and JSON keys
+	tableKeysToJsonKeys := map[string]string{
+		allocationID: strings.ToLower(allocationID),
+		client:       strings.ToLower(client),
+		provider:     strings.ToLower(provider),
+		pieceCid:     strings.ToLower(pieceCid),
+		pieceSize:    strings.ToLower(pieceSize),
+		tMin:         strings.ToLower(tMin),
+		tMax:         strings.ToLower(tMax),
+		expr:         strings.ToLower(expr),
+	}
+
+	var allocs []map[string]interface{}
+
+	for key, val := range allocations {
+		alloc := map[string]interface{}{
+			allocationID: key,
+			client:       val.Client,
+			provider:     val.Provider,
+			pieceCid:     val.Data,
+			pieceSize:    val.Size,
+			tMin:         val.TermMin,
+			tMax:         val.TermMax,
+			expr:         val.Expiration,
+		}
+		allocs = append(allocs, alloc)
+	}
+
+	if json {
+		// get a new list of wallets with json keys instead of tablewriter keys
+		var jsonAllocs []map[string]interface{}
+		for _, alloc := range allocs {
+			jsonAlloc := make(map[string]interface{})
+			for k, v := range alloc {
+				jsonAlloc[tableKeysToJsonKeys[k]] = v
+			}
+			jsonAllocs = append(jsonAllocs, jsonAlloc)
+		}
+		// then return this!
+		return cmd.PrintJson(jsonAllocs)
+	} else {
+		// Init the tablewriter's columns
+		tw := tablewriter.New(
+			tablewriter.Col(allocationID),
+			tablewriter.Col(client),
+			tablewriter.Col(provider),
+			tablewriter.Col(pieceCid),
+			tablewriter.Col(pieceSize),
+			tablewriter.Col(tMin),
+			tablewriter.Col(tMax),
+			tablewriter.NewLineCol(expr))
+		// populate it with content
+		for _, alloc := range allocs {
+			tw.Write(alloc)
+		}
+		// return the corresponding string
+		return tw.Flush(os.Stdout)
+	}
+}
diff --git a/cmd/boost/main.go b/cmd/boost/main.go
@@ -4,12 +4,11 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	llog "log"
 	"os"
 
 	"github.com/filecoin-project/boost/cmd"
 
-	llog "log"
-
 	"github.com/filecoin-project/boost/build"
 	"github.com/filecoin-project/boostd-data/shared/cliutil"
 	logging "github.com/ipfs/go-log/v2"
@@ -41,6 +40,9 @@ func main() {
 			offlineDealCmd,
 			providerCmd,
 			walletCmd,
+			//TODO: enable when DDO ends up in a network upgrade
+			//directDealAllocate,
+			//directDealGetAllocations,
 		},
 	}
 	app.Setup()