curio migration command

Author: LexLuthr

.github/image/Dockerfile

@@ -18,7 +18,7 @@ RUN git submodule update --init
 RUN go mod download
 
 # Download Go dependencies for Curio
-WORKDIR /app/boost
+WORKDIR /app/curio
 RUN git submodule update --init
 RUN go mod download
 

.gitignore

@@ -8,6 +8,7 @@
 /booster-bitswap
 /docgen-md
 /docgen-openrpc
+/migrate-curio
 extern/filecoin-ffi/rust/target
 extern/boostd-data/boostd-data
 **/*.a

Makefile

@@ -133,6 +133,11 @@ boostci: $(BUILD_DEPS)
 	$(GOCC) build $(GOFLAGS) -o boostci ./cmd/boostci
 .PHONY: boostci
 
+migrate-curio: $(BUILD_DEPS)
+	rm -f migrate-curio
+	$(GOCC) build $(GOFLAGS) -o migrate-curio ./cmd/migrate-curio
+.PHONY: boostci
+
 react: validate-node-version
 	npm_config_legacy_peer_deps=yes npm ci --no-audit --prefix react
 	npm run --prefix react build
@@ -143,7 +148,7 @@ update-react: validate-node-version
 	npm run --prefix react build
 .PHONY: react
 
-build-go: boost boostd boostx boostd-data booster-http booster-bitswap devnet migrate-lid
+build-go: boost boostd boostx boostd-data booster-http booster-bitswap devnet migrate-lid migrate-curio
 .PHONY: build-go
 
 build: react build-go
@@ -162,6 +167,7 @@ install-boost:
 	install -C ./booster-http /usr/local/bin/booster-http
 	install -C ./booster-bitswap /usr/local/bin/booster-bitswap
 	install -C ./migrate-lid /usr/local/bin/migrate-lid
+	install -C ./migrate-curio /usr/local/bin/migrate-curio
 
 install-devnet:
 	install -C ./devnet /usr/local/bin/devnet

api/api.go

@@ -26,6 +26,7 @@ type Boost interface {
 	Net
 
 	// MethodGroup: Boost
+	BoostIndexerRemoveAll(ctx context.Context) ([]cid.Cid, error)                                                                               //perm:admin
 	BoostIndexerAnnounceAllDeals(ctx context.Context) error                                                                                     //perm:admin
 	BoostIndexerListMultihashes(ctx context.Context, contextID []byte) ([]multihash.Multihash, error)                                           //perm:admin
 	BoostIndexerAnnounceLatest(ctx context.Context) (cid.Cid, error)                                                                            //perm:admin

api/proxy_gen.go

@@ -22,6 +22,7 @@ var indexProvCmd = &cli.Command{
 		indexProvAnnounceLatestHttp,
 		indexProvAnnounceDealRemovalAd,
 		indexProvAnnounceDeal,
+		indexProvRemoveAllCmd,
 	},
 }
 
@@ -303,3 +304,29 @@ var indexProvAnnounceDeal = &cli.Command{
 		return nil
 	},
 }
+
+var indexProvRemoveAllCmd = &cli.Command{
+	Name:  "remove-all",
+	Usage: "Announce all removal ad for all contextIDs",
+	Flags: []cli.Flag{},
+	Action: func(cctx *cli.Context) error {
+		ctx := lcli.ReqContext(cctx)
+
+		// get boost api
+		napi, closer, err := bcli.GetBoostAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		// announce markets and boost deals
+		cids, err := napi.BoostIndexerRemoveAll(ctx)
+		if err != nil {
+			return err
+		}
+		for _, c := range cids {
+			fmt.Println("Published the removal ad with CID", c.String())
+		}
+		return nil
+	},
+}

build/openrpc/boost.json.gz

@@ -13,6 +13,7 @@ import (
 	"github.com/ipld/go-ipld-prime"
 	cidlink "github.com/ipld/go-ipld-prime/linking/cid"
 	"github.com/ipld/go-ipld-prime/traversal"
+	nilrouting "github.com/libp2p/go-libp2p-routing-helpers"
 	"github.com/libp2p/go-libp2p/core/host"
 	mh "github.com/multiformats/go-multihash"
 
@@ -21,7 +22,6 @@ import (
 	lcli "github.com/filecoin-project/lotus/cli"
 	"github.com/ipfs/boxo/bitswap/client"
 	bsnetwork "github.com/ipfs/boxo/bitswap/network"
-	nilrouting "github.com/ipfs/boxo/routing/none"
 	blocks "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
 	ipldlegacy "github.com/ipfs/go-ipld-legacy"
@@ -96,10 +96,7 @@ var fetchCmd = &cli.Command{
 		}
 
 		// Create a bitswap client
-		nilRouter, err := nilrouting.ConstructNilRouting(ctx, nil, nil, nil)
-		if err != nil {
-			return err
-		}
+		nilRouter := nilrouting.Null{}
 		net := bsnetwork.NewFromIpfsHost(host, nilRouter)
 		bs, err := blockstore.OpenReadWrite(outputCarPath, []cid.Cid{rootCid}, blockstore.UseWholeCIDs(true))
 		if err != nil {

cmd/boostd/index.go

@@ -8,9 +8,9 @@ import (
 	"github.com/filecoin-project/boost/protocolproxy"
 	bsnetwork "github.com/ipfs/boxo/bitswap/network"
 	"github.com/ipfs/boxo/bitswap/server"
-	blockstore "github.com/ipfs/boxo/blockstore"
-	nilrouting "github.com/ipfs/boxo/routing/none"
+	"github.com/ipfs/boxo/blockstore"
 	"github.com/ipfs/go-cid"
+	nilrouting "github.com/libp2p/go-libp2p-routing-helpers"
 	"github.com/libp2p/go-libp2p/core/host"
 	"github.com/libp2p/go-libp2p/core/peer"
 )
@@ -66,10 +66,7 @@ func (s *BitswapServer) Start(ctx context.Context, proxy *peer.AddrInfo, opts *B
 	}
 
 	// Start a bitswap server on the provider
-	nilRouter, err := nilrouting.ConstructNilRouting(s.ctx, nil, nil, nil)
-	if err != nil {
-		return err
-	}
+	nilRouter := nilrouting.Null{}
 	bsopts := []server.Option{
 		server.EngineBlockstoreWorkerCount(opts.EngineBlockstoreWorkerCount),
 		server.EngineTaskWorkerCount(opts.EngineTaskWorkerCount),

cmd/booster-bitswap/client.go

@@ -0,0 +1,19 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/urfave/cli/v2"
+)
+
+var cleanupLevelDBCmd = &cli.Command{
+	Name:        "leveldb",
+	Description: "Removes the indexes and other metadata leveldb based LID store",
+	Usage:       "migrate-curio cleanup leveldb",
+	Before:      before,
+	Action: func(cctx *cli.Context) error {
+		fmt.Println("Please remove the directory called 'LID' in the boost repo path to remove leveldb based LID")
+		fmt.Println("This directory can also be present outside of Boost repo if 'boostd-data' was running with a custom repo path")
+		return nil
+	},
+}