fix: refactor LID clean up (#1886)

* refactor pdcleaner

* move cleanup test to itest

* increase blocktime

* use evemtually and errgroup

* fix lint errs

* bump container lotus version

* bump image builder go version

* bump docker node rust versions

Author: LexLuthr

.circleci/config.yml

@@ -363,4 +363,9 @@ workflows:
           suite: booster-bitswap
           target: "./cmd/booster-bitswap"
 
+      - test:
+          name: test-itest-lid-cleanup
+          suite: itest-lid-cleanup
+          target: "./itests/lid_cleanup_test.go"
+
       - lid-docker-compose

.github/workflows/container-build.yml

@@ -16,7 +16,7 @@ jobs:
       contents: read
       packages: write
     env:
-      LOTUS_VERSION: 'v1.23.3'
+      LOTUS_VERSION: 'v1.26.0-rc1'
       LOTUS_SOURCE_IMAGE: 'ghcr.io/filecoin-shipyard/lotus-containers:lotus'
       NETWORK_NAME: 'devnet'
       FFI_BUILD_FROM_SOURCE: '0'

api/api.go

@@ -27,7 +27,7 @@ type Boost interface {
 
 	// MethodGroup: Boost
 	BoostIndexerAnnounceAllDeals(ctx context.Context) error                                                                                     //perm:admin
-	BoostIndexerListMultihashes(ctx context.Context, proposalCid cid.Cid) ([]multihash.Multihash, error)                                        //perm:admin
+	BoostIndexerListMultihashes(ctx context.Context, contextID []byte) ([]multihash.Multihash, error)                                           //perm:admin
 	BoostIndexerAnnounceLatest(ctx context.Context) (cid.Cid, error)                                                                            //perm:admin
 	BoostIndexerAnnounceLatestHttp(ctx context.Context, urls []string) (cid.Cid, error)                                                         //perm:admin
 	BoostOfflineDealWithData(ctx context.Context, dealUuid uuid.UUID, filePath string, delAfterImport bool) (*ProviderDealRejectionInfo, error) //perm:admin
@@ -48,6 +48,7 @@ type Boost interface {
 	// MethodGroup: PieceDirectory
 	PdBuildIndexForPieceCid(ctx context.Context, piececid cid.Cid) error             //perm:admin
 	PdRemoveDealForPiece(ctx context.Context, piececid cid.Cid, dealID string) error //perm:admin
+	PdCleanup(ctx context.Context) error                                             //perm:admin
 
 	// MethodGroup: Misc
 	OnlineBackup(context.Context, string) error //perm:admin

api/proxy_gen.go

@@ -8,6 +8,7 @@ import (
 	lcli "github.com/filecoin-project/lotus/cli"
 	"github.com/google/uuid"
 	"github.com/ipfs/go-cid"
+	"github.com/multiformats/go-multihash"
 	"github.com/urfave/cli/v2"
 )
 
@@ -45,16 +46,11 @@ var indexProvAnnounceAllCmd = &cli.Command{
 
 var indexProvListMultihashesCmd = &cli.Command{
 	Name:      "list-multihashes",
-	Usage:     "list-multihashes <proposal cid>",
-	UsageText: "List multihashes for a deal by proposal cid",
+	Usage:     "list-multihashes <proposal cid / deal UUID>",
+	UsageText: "List multihashes for a deal by proposal cid or deal UUID",
 	Action: func(cctx *cli.Context) error {
 		if cctx.NArg() != 1 {
-			return fmt.Errorf("must supply proposal cid")
-		}
-
-		propCid, err := cid.Parse(cctx.Args().First())
-		if err != nil {
-			return fmt.Errorf("parsing proposal cid %s: %w", cctx.Args().First(), err)
+			return fmt.Errorf("must supply a proposal cid or deal UUID")
 		}
 
 		ctx := lcli.ReqContext(cctx)
@@ -66,13 +62,46 @@ var indexProvListMultihashesCmd = &cli.Command{
 		}
 		defer closer()
 
-		// get list of multihashes
-		mhs, err := napi.BoostIndexerListMultihashes(ctx, propCid)
+		if cctx.Args().Len() != 1 {
+			return fmt.Errorf("must specify only one proposal CID / deal UUID")
+		}
+
+		id := cctx.Args().Get(0)
+
+		var proposalCid cid.Cid
+		var mhs []multihash.Multihash
+		dealUuid, err := uuid.Parse(id)
+		if err != nil {
+			propCid, err := cid.Decode(id)
+			if err != nil {
+				return fmt.Errorf("could not parse '%s' as deal uuid or proposal cid", id)
+			}
+			proposalCid = propCid
+		}
+
+		if !proposalCid.Defined() {
+			contextID, err := dealUuid.MarshalBinary()
+			if err != nil {
+				return fmt.Errorf("parsing UUID to bytes: %w", err)
+			}
+			mhs, err = napi.BoostIndexerListMultihashes(ctx, contextID)
+			if err != nil {
+				return err
+			}
+			fmt.Printf("Found %d multihashes for deal with ID %s:\n", len(mhs), id)
+			for _, mh := range mhs {
+				fmt.Println("  " + mh.String())
+			}
+
+			return nil
+		}
+
+		mhs, err = napi.BoostIndexerListMultihashes(ctx, proposalCid.Bytes())
 		if err != nil {
 			return err
 		}
 
-		fmt.Printf("Found %d multihashes for deal with proposal cid %s:\n", len(mhs), propCid)
+		fmt.Printf("Found %d multihashes for deal with ID %s:\n", len(mhs), id)
 		for _, mh := range mhs {
 			fmt.Println("  " + mh.String())
 		}

build/openrpc/boost.json.gz

@@ -19,6 +19,7 @@ var pieceDirCmd = &cli.Command{
 		pdIndexGenerate,
 		recoverCmd,
 		removeDealCmd,
+		lidCleanupCmd,
 	},
 }
 
@@ -109,3 +110,24 @@ var removeDealCmd = &cli.Command{
 
 	},
 }
+
+var lidCleanupCmd = &cli.Command{
+	Name:  "cleanup",
+	Usage: "Triggers a cleanup for LID. Command will wait for existing cleanup jobs to finish if there are any",
+	Action: func(cctx *cli.Context) error {
+		ctx := lcli.ReqContext(cctx)
+
+		napi, closer, err := bcli.GetBoostAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		err = napi.PdCleanup(ctx)
+		if err != nil {
+			return fmt.Errorf("clean up failed: %w", err)
+		}
+		fmt.Println("LID clean up complete")
+		return nil
+	},
+}

cmd/boostd/index.go

@@ -4,7 +4,7 @@
 ARG LOTUS_TEST_IMAGE=filecoin/lotus-test:latest
 FROM ${LOTUS_TEST_IMAGE} as lotus-test
 #########################################################################################
-FROM node:16.16-alpine3.15 AS react-builder
+FROM node:20.11.1-alpine3.19 AS react-builder
 
 WORKDIR /src
 COPY react /src/react
@@ -13,7 +13,7 @@ COPY gql /src/gql
 RUN npm_config_legacy_peer_deps=yes npm ci --no-audit --prefix react&& \
       npm run --prefix react build
 #########################################################################################
-FROM golang:1.20-bullseye as builder
+FROM golang:1.21-bullseye as builder
 
 RUN apt update && apt install -y \
       build-essential \
@@ -31,16 +31,18 @@ RUN apt update && apt install -y \
 ENV RUSTUP_HOME=/usr/local/rustup \
     CARGO_HOME=/usr/local/cargo \
     PATH=/usr/local/cargo/bin:$PATH \
-    RUST_VERSION=1.63.0
+    RUST_VERSION=1.76.0
 
 RUN set -eux; \
     dpkgArch="$(dpkg --print-architecture)"; \
     case "${dpkgArch##*-}" in \
-        amd64) rustArch='x86_64-unknown-linux-gnu'; rustupSha256='5cc9ffd1026e82e7fb2eec2121ad71f4b0f044e88bca39207b3f6b769aaa799c' ;; \
-        arm64) rustArch='aarch64-unknown-linux-gnu'; rustupSha256='e189948e396d47254103a49c987e7fb0e5dd8e34b200aa4481ecc4b8e41fb929' ;; \
+        amd64) rustArch='x86_64-unknown-linux-gnu'; rustupSha256='0b2f6c8f85a3d02fde2efc0ced4657869d73fccfce59defb4e8d29233116e6db' ;; \
+        armhf) rustArch='armv7-unknown-linux-gnueabihf'; rustupSha256='f21c44b01678c645d8fbba1e55e4180a01ac5af2d38bcbd14aa665e0d96ed69a' ;; \
+        arm64) rustArch='aarch64-unknown-linux-gnu'; rustupSha256='673e336c81c65e6b16dcdede33f4cc9ed0f08bde1dbe7a935f113605292dc800' ;; \
+        i386) rustArch='i686-unknown-linux-gnu'; rustupSha256='e7b0f47557c1afcd86939b118cbcf7fb95a5d1d917bdd355157b63ca00fc4333' ;; \
         *) echo >&2 "unsupported architecture: ${dpkgArch}"; exit 1 ;; \
     esac; \
-    url="https://static.rust-lang.org/rustup/archive/1.25.1/${rustArch}/rustup-init"; \
+    url="https://static.rust-lang.org/rustup/archive/1.26.0/${rustArch}/rustup-init"; \
     wget "$url"; \
     echo "${rustupSha256} *rustup-init" | sha256sum -c -; \
     chmod +x rustup-init; \

cmd/boostd/piecedir.go

@@ -55,6 +55,7 @@
   * [OnlineBackup](#onlinebackup)
 * [Pd](#pd)
   * [PdBuildIndexForPieceCid](#pdbuildindexforpiececid)
+  * [PdCleanup](#pdcleanup)
   * [PdRemoveDealForPiece](#pdremovedealforpiece)
 ## 
 
@@ -554,9 +555,7 @@ Perms: admin
 Inputs:
 ```json
 [
-  {
-    "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4"
-  }
+  "Ynl0ZSBhcnJheQ=="
 ]
 ```
 
@@ -1245,6 +1244,15 @@ Inputs:
 
 Response: `{}`
 
+### PdCleanup
+
+
+Perms: admin
+
+Inputs: `null`
+
+Response: `{}`
+
 ### PdRemoveDealForPiece
 
 

docker/devnet/Dockerfile.source

@@ -59,7 +59,7 @@ func TestDirectDeal(t *testing.T) {
 	require.NoError(t, err)
 	defer f.Stop()
 
-	// Send funs to PSD wallet as it is being used for POST
+	// Send funds to PSD wallet as it is being used for POST
 	info, err := f.FullNode.StateMinerInfo(ctx, f.MinerAddr, types.EmptyTSK)
 	require.NoError(t, err)
 	addresses := []address.Address{info.Owner, info.Worker}