From e419bcffcc427c31a906926455f5bf6f8201df36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 30 Aug 2024 13:45:04 +0200 Subject: [PATCH] Move storiface into Curio (#183) * temp: replace lotus to local * move storiface to Curio * fix build * don't refer to storiface2 * make gen * make remote path test happy * cleanup storiface --- api/api_curio.go | 3 +- api/proxy_gen.go | 3 +- cmd/curio/guidedsetup/cbor_gen.go | 3 +- cmd/curio/guidedsetup/shared.go | 2 +- cmd/curio/rpc/rpc.go | 2 +- cmd/curio/storage.go | 4 +- cmd/curio/tasks/tasks.go | 2 +- deps/deps.go | 2 +- go.mod | 29 +- go.sum | 111 ++----- itests/curio_test.go | 2 +- lib/dealdata/dealdata.go | 2 +- lib/ffi/piece_funcs.go | 3 +- lib/ffi/sdr_funcs.go | 3 +- lib/ffi/sdr_funcs_test.go | 2 +- lib/ffi/snap_funcs.go | 2 +- lib/ffi/task_storage.go | 3 +- lib/ffiselect/ffidirect/ffi-direct.go | 2 +- lib/ffiselect/ffiselect.go | 3 +- lib/partialfile/partialfile.go | 346 +++++++++++++++++++ lib/paths/db_index.go | 2 +- lib/paths/http_handler.go | 5 +- lib/paths/http_handler_test.go | 5 +- lib/paths/index.go | 3 +- lib/paths/index_locks.go | 2 +- lib/paths/index_locks_test.go | 2 +- lib/paths/interface.go | 5 +- lib/paths/local.go | 2 +- lib/paths/local_test.go | 2 +- lib/paths/localstorage.go | 86 ++++- lib/paths/localstorage_cached.go | 3 +- lib/paths/mocks/index.go | 2 +- lib/paths/mocks/pf.go | 4 +- lib/paths/mocks/store.go | 3 +- lib/paths/remote.go | 5 +- lib/paths/remote_prove.go | 3 +- lib/paths/remote_test.go | 26 +- lib/storiface/cbor_gen.go | 343 +++++++++++++++++++ lib/storiface/ffi.go | 41 +++ lib/storiface/filetype.go | 460 ++++++++++++++++++++++++++ lib/storiface/index.go | 134 ++++++++ lib/storiface/paths.go | 39 +++ lib/storiface/storage.go | 154 +++++++++ lib/storiface/worker.go | 81 +++++ market/fakelm/iface.go | 3 +- market/fakelm/lmimpl.go | 2 +- market/lmrpc/lmrpc.go | 81 ++++- scripts/docgen/docgen.go | 20 +- tasks/gc/storage_endpoint_gc.go | 2 +- tasks/gc/storage_gc_mark.go | 2 +- tasks/gc/storage_gc_sweep.go | 3 +- tasks/piece/task_cleanup_piece.go | 3 +- tasks/piece/task_park_piece.go | 3 +- tasks/seal/task_finalize.go | 3 +- tasks/seal/task_movestorage.go | 3 +- tasks/seal/task_porep.go | 2 +- tasks/seal/task_sdr.go | 2 +- tasks/seal/task_synth_proofs.go | 3 +- tasks/seal/task_treed.go | 3 +- tasks/seal/task_treerc.go | 3 +- tasks/sealsupra/task_supraseal.go | 2 +- tasks/snap/task_encode.go | 3 +- tasks/snap/task_movestorage.go | 3 +- tasks/snap/task_prove.go | 3 +- tasks/window/compute_do.go | 9 +- tasks/window/compute_task.go | 29 +- tasks/window/faults_simple.go | 3 +- tasks/window/recover_task.go | 5 +- tasks/winning/winning_task.go | 2 +- web/api/sector/sector.go | 2 +- web/api/webrpc/sector.go | 2 +- web/api/webrpc/storage_stats.go | 3 +- 72 files changed, 1887 insertions(+), 258 deletions(-) create mode 100644 lib/partialfile/partialfile.go create mode 100644 lib/storiface/cbor_gen.go create mode 100644 lib/storiface/ffi.go create mode 100644 lib/storiface/filetype.go create mode 100644 lib/storiface/index.go create mode 100644 lib/storiface/paths.go create mode 100644 lib/storiface/storage.go create mode 100644 lib/storiface/worker.go diff --git a/api/api_curio.go b/api/api_curio.go index a96b9a956..f5d07059a 100644 --- a/api/api_curio.go +++ b/api/api_curio.go @@ -8,10 +8,11 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" + storiface "github.com/filecoin-project/curio/lib/storiface" + "github.com/filecoin-project/lotus/api" lpiece "github.com/filecoin-project/lotus/storage/pipeline/piece" "github.com/filecoin-project/lotus/storage/sealer/fsutil" - "github.com/filecoin-project/lotus/storage/sealer/storiface" ) type Curio interface { diff --git a/api/proxy_gen.go b/api/proxy_gen.go index 4ddba3ce8..bcbc39d82 100644 --- a/api/proxy_gen.go +++ b/api/proxy_gen.go @@ -23,13 +23,14 @@ import ( "github.com/filecoin-project/go-state-types/dline" "github.com/filecoin-project/go-state-types/network" + storiface "github.com/filecoin-project/curio/lib/storiface" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/node/modules/dtypes" lpiece "github.com/filecoin-project/lotus/storage/pipeline/piece" "github.com/filecoin-project/lotus/storage/sealer/fsutil" - "github.com/filecoin-project/lotus/storage/sealer/storiface" ) var _ = reflect.TypeOf([]byte(nil)) diff --git a/cmd/curio/guidedsetup/cbor_gen.go b/cmd/curio/guidedsetup/cbor_gen.go index 9a2beae43..5d26246f1 100644 --- a/cmd/curio/guidedsetup/cbor_gen.go +++ b/cmd/curio/guidedsetup/cbor_gen.go @@ -14,9 +14,8 @@ import ( abi "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/curio/lib/storiface" sector "github.com/filecoin-project/curio/lib/types/sector" - - storiface "github.com/filecoin-project/lotus/storage/sealer/storiface" ) var _ = xerrors.Errorf diff --git a/cmd/curio/guidedsetup/shared.go b/cmd/curio/guidedsetup/shared.go index 51fec2d14..11327873e 100644 --- a/cmd/curio/guidedsetup/shared.go +++ b/cmd/curio/guidedsetup/shared.go @@ -26,11 +26,11 @@ import ( "github.com/filecoin-project/curio/deps" "github.com/filecoin-project/curio/deps/config" "github.com/filecoin-project/curio/harmony/harmonydb" + "github.com/filecoin-project/curio/lib/storiface" "github.com/filecoin-project/curio/lib/types/sector" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/node/repo" - "github.com/filecoin-project/lotus/storage/sealer/storiface" ) const ( diff --git a/cmd/curio/rpc/rpc.go b/cmd/curio/rpc/rpc.go index ddd1c6e00..11d97050a 100644 --- a/cmd/curio/rpc/rpc.go +++ b/cmd/curio/rpc/rpc.go @@ -33,6 +33,7 @@ import ( "github.com/filecoin-project/curio/lib/metrics" "github.com/filecoin-project/curio/lib/paths" "github.com/filecoin-project/curio/lib/repo" + storiface "github.com/filecoin-project/curio/lib/storiface" "github.com/filecoin-project/curio/web" lapi "github.com/filecoin-project/lotus/api" @@ -42,7 +43,6 @@ import ( "github.com/filecoin-project/lotus/metrics/proxy" "github.com/filecoin-project/lotus/storage/pipeline/piece" "github.com/filecoin-project/lotus/storage/sealer/fsutil" - "github.com/filecoin-project/lotus/storage/sealer/storiface" ) const metaFile = "sectorstore.json" diff --git a/cmd/curio/storage.go b/cmd/curio/storage.go index 09fb57a6c..ca15f0ebb 100644 --- a/cmd/curio/storage.go +++ b/cmd/curio/storage.go @@ -20,10 +20,10 @@ import ( "github.com/filecoin-project/curio/cmd/curio/rpc" "github.com/filecoin-project/curio/lib/reqcontext" + storiface "github.com/filecoin-project/curio/lib/storiface" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/storage/sealer/fsutil" - "github.com/filecoin-project/lotus/storage/sealer/storiface" ) var storageCmd = &cli.Command{ @@ -438,7 +438,7 @@ var storageFindCmd = &cli.Command{ } sectorTypes := []storiface.SectorFileType{ - storiface.FTUnsealed, storiface.FTSealed, storiface.FTCache, storiface.FTUpdate, storiface.FTUpdateCache, + storiface.FTUnsealed, storiface.FTSealed, storiface.FTCache, storiface.FTUpdate, storiface.FTUpdateCache, storiface.FTPiece, } byId := make(map[storiface.ID]*storedSector) diff --git a/cmd/curio/tasks/tasks.go b/cmd/curio/tasks/tasks.go index 94592c447..03c07ba2b 100644 --- a/cmd/curio/tasks/tasks.go +++ b/cmd/curio/tasks/tasks.go @@ -30,6 +30,7 @@ import ( "github.com/filecoin-project/curio/lib/multictladdr" "github.com/filecoin-project/curio/lib/paths" "github.com/filecoin-project/curio/lib/slotmgr" + "github.com/filecoin-project/curio/lib/storiface" "github.com/filecoin-project/curio/tasks/gc" "github.com/filecoin-project/curio/tasks/message" "github.com/filecoin-project/curio/tasks/metadata" @@ -44,7 +45,6 @@ import ( "github.com/filecoin-project/lotus/lib/lazy" "github.com/filecoin-project/lotus/lib/result" "github.com/filecoin-project/lotus/node/modules/dtypes" - "github.com/filecoin-project/lotus/storage/sealer/storiface" ) var log = logging.Logger("curio/deps") diff --git a/deps/deps.go b/deps/deps.go index 50b60d5eb..466cfbb1b 100644 --- a/deps/deps.go +++ b/deps/deps.go @@ -38,6 +38,7 @@ import ( "github.com/filecoin-project/curio/lib/multictladdr" "github.com/filecoin-project/curio/lib/paths" "github.com/filecoin-project/curio/lib/repo" + "github.com/filecoin-project/curio/lib/storiface" lapi "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" @@ -46,7 +47,6 @@ import ( lrepo "github.com/filecoin-project/lotus/node/repo" "github.com/filecoin-project/lotus/storage/sealer" "github.com/filecoin-project/lotus/storage/sealer/ffiwrapper" - "github.com/filecoin-project/lotus/storage/sealer/storiface" ) var log = logging.Logger("curio/deps") diff --git a/go.mod b/go.mod index 0ec897a05..41ac3822e 100644 --- a/go.mod +++ b/go.mod @@ -19,8 +19,8 @@ require ( github.com/filecoin-project/go-address v1.1.0 github.com/filecoin-project/go-bitfield v0.2.4 github.com/filecoin-project/go-cbor-util v0.0.1 - github.com/filecoin-project/go-commp-utils v0.1.3 - github.com/filecoin-project/go-commp-utils/nonffi v0.0.0-20220905160352-62059082a837 + github.com/filecoin-project/go-commp-utils v0.1.4 + github.com/filecoin-project/go-commp-utils/nonffi v0.0.0-20240802040721-2a04ffc8ffe8 github.com/filecoin-project/go-fil-commcid v0.1.0 github.com/filecoin-project/go-jsonrpc v0.6.1-0.20240820160949-2cfe810e5d2f github.com/filecoin-project/go-padreader v0.0.1 @@ -66,7 +66,7 @@ require ( github.com/snadrus/must v0.0.0-20240605044437-98cedd57f8eb github.com/stretchr/testify v1.9.0 github.com/urfave/cli/v2 v2.25.5 - github.com/whyrusleeping/cbor-gen v0.1.1 + github.com/whyrusleeping/cbor-gen v0.1.2 github.com/yugabyte/pgx/v5 v5.5.3-yb-2 go.opencensus.io v0.24.0 go.uber.org/multierr v1.11.0 @@ -74,10 +74,10 @@ require ( golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 golang.org/x/net v0.26.0 golang.org/x/sync v0.7.0 - golang.org/x/sys v0.21.0 + golang.org/x/sys v0.23.0 golang.org/x/text v0.16.0 golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d - golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 + golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9 ) require ( @@ -124,11 +124,11 @@ require ( github.com/filecoin-project/go-amt-ipld/v3 v3.1.0 // indirect github.com/filecoin-project/go-amt-ipld/v4 v4.3.0 // indirect github.com/filecoin-project/go-clock v0.1.0 // indirect - github.com/filecoin-project/go-crypto v0.0.1 // indirect - github.com/filecoin-project/go-f3 v0.0.7 // indirect + github.com/filecoin-project/go-crypto v0.1.0 // indirect + github.com/filecoin-project/go-f3 v0.2.0 // indirect github.com/filecoin-project/go-hamt-ipld v0.1.5 // indirect github.com/filecoin-project/go-hamt-ipld/v2 v2.0.0 // indirect - github.com/filecoin-project/go-hamt-ipld/v3 v3.1.0 // indirect + github.com/filecoin-project/go-hamt-ipld/v3 v3.4.0 // indirect github.com/filecoin-project/go-paramfetch v0.0.4 // indirect github.com/filecoin-project/go-statemachine v1.0.3 // indirect github.com/filecoin-project/go-storedcounter v0.1.0 // indirect @@ -190,7 +190,6 @@ require ( github.com/ipld/go-codec-dagpb v1.6.0 // indirect github.com/ipld/go-ipld-prime v0.21.0 // indirect github.com/ipni/go-libipni v0.0.8 // indirect - github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 // indirect github.com/jackc/puddle/v2 v2.2.1 // indirect @@ -201,9 +200,9 @@ require ( github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/jpillora/backoff v1.0.0 // indirect - github.com/kilic/bls12-381 v0.1.0 // indirect - github.com/klauspost/compress v1.17.8 // indirect - github.com/klauspost/cpuid/v2 v2.2.7 // indirect + github.com/kilic/bls12-381 v0.1.1-0.20220929213557-ca162e8a70f4 // indirect + github.com/klauspost/compress v1.17.9 // indirect + github.com/klauspost/cpuid/v2 v2.2.8 // indirect github.com/koron/go-ssdp v0.0.4 // indirect github.com/libp2p/go-cidranger v1.1.0 // indirect github.com/libp2p/go-flow-metrics v0.1.0 // indirect @@ -295,6 +294,8 @@ require ( github.com/zondax/ledger-filecoin-go v0.11.1 // indirect github.com/zondax/ledger-go v0.14.3 // indirect github.com/zyedidia/generic v1.2.1 // indirect + gitlab.com/yawning/secp256k1-voi v0.0.0-20230925100816-f2616030848b // indirect + gitlab.com/yawning/tuplehash v0.0.0-20230713102510-df83abbf9a02 // indirect go.opentelemetry.io/otel v1.28.0 // indirect go.opentelemetry.io/otel/bridge/opencensus v1.28.0 // indirect go.opentelemetry.io/otel/exporters/jaeger v1.14.0 // indirect @@ -308,9 +309,9 @@ require ( go.uber.org/fx v1.22.1 // indirect go.uber.org/mock v0.4.0 // indirect go4.org v0.0.0-20230225012048-214862532bf5 // indirect - golang.org/x/crypto v0.24.0 // indirect + golang.org/x/crypto v0.25.0 // indirect golang.org/x/mod v0.17.0 // indirect - golang.org/x/term v0.21.0 // indirect + golang.org/x/term v0.22.0 // indirect golang.org/x/time v0.5.0 // indirect gonum.org/v1/gonum v0.15.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 // indirect diff --git a/go.sum b/go.sum index 7e5c89848..1fe11f3ee 100644 --- a/go.sum +++ b/go.sum @@ -179,7 +179,6 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:ma github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cskr/pubsub v1.0.2 h1:vlOzMhl6PFn60gRlTQQsIfVwaPB/B/8MziK8FhEPt/0= github.com/cskr/pubsub v1.0.2/go.mod h1:/8MzYXk/NJAz782G8RPkFzXTZVu63VotefPnR9TIRis= github.com/daaku/go.zipexe v1.0.2 h1:Zg55YLYTr7M9wjKn8SY/WcpuuEi+kR2u4E8RhvpyXmk= @@ -255,7 +254,6 @@ github.com/filecoin-project/go-amt-ipld/v3 v3.0.0/go.mod h1:Qa95YNAbtoVCTSVtX38a github.com/filecoin-project/go-amt-ipld/v3 v3.1.0 h1:ZNJ9tEG5bE72vBWYiuh5bkxJVM3ViHNOmQ7qew9n6RE= github.com/filecoin-project/go-amt-ipld/v3 v3.1.0/go.mod h1:UjM2QhDFrrjD5s1CdnkJkat4ga+LqZBZgTMniypABRo= github.com/filecoin-project/go-amt-ipld/v4 v4.0.0/go.mod h1:gF053YQ4BIpzTNDoEwHZas7U3oAwncDVGvOHyY8oDpE= -github.com/filecoin-project/go-amt-ipld/v4 v4.2.0/go.mod h1:0eDVF7pROvxrsxvLJx+SJZXqRaXXcEPUcgb/rG0zGU4= github.com/filecoin-project/go-amt-ipld/v4 v4.3.0 h1:bY42N1gR0DqrLMCKUPzX1VhYVgXaETQm0Um4ohvyEP8= github.com/filecoin-project/go-amt-ipld/v4 v4.3.0/go.mod h1:39Ep/yBbF6xN94WevLG9qSbglBJepHa5zeEbAE1pYsc= github.com/filecoin-project/go-bitfield v0.2.0/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM= @@ -267,16 +265,15 @@ github.com/filecoin-project/go-cbor-util v0.0.1 h1:E1LYZYTtjfAQwCReho0VXvbu8t3CY github.com/filecoin-project/go-cbor-util v0.0.1/go.mod h1:pqTiPHobNkOVM5thSRsHYjyQfq7O5QSCMhvuu9JoDlg= github.com/filecoin-project/go-clock v0.1.0 h1:SFbYIM75M8NnFm1yMHhN9Ahy3W5bEZV9gd6MPfXbKVU= github.com/filecoin-project/go-clock v0.1.0/go.mod h1:4uB/O4PvOjlx1VCMdZ9MyDZXRm//gkj1ELEbxfI1AZs= -github.com/filecoin-project/go-commp-utils v0.1.3 h1:rTxbkNXZU7FLgdkBk8RsQIEOuPONHykEoX3xGk41Fkw= -github.com/filecoin-project/go-commp-utils v0.1.3/go.mod h1:3ENlD1pZySaUout0p9ANQrY3fDFoXdqyX04J+dWpK30= -github.com/filecoin-project/go-commp-utils/nonffi v0.0.0-20220905160352-62059082a837 h1:4cITW0pwgvqLs86Q9bWQa34+jBfR1V687bDkmv2DgnA= -github.com/filecoin-project/go-commp-utils/nonffi v0.0.0-20220905160352-62059082a837/go.mod h1:e2YBjSblNVoBckkbv3PPqsq71q98oFkFqL7s1etViGo= +github.com/filecoin-project/go-commp-utils v0.1.4 h1:/WSsrAb0xupo+aRWRyD80lRUXAXJvYoTgDQS1pYZ1Mk= +github.com/filecoin-project/go-commp-utils v0.1.4/go.mod h1:Sekocu5q9b4ECAUFu853GFUbm8I7upAluummHFe2kFo= +github.com/filecoin-project/go-commp-utils/nonffi v0.0.0-20240802040721-2a04ffc8ffe8 h1:jAG2g1Fs/qoDSSaI8JaP/KmqR+QQ8IVQ6k9xKONa72M= +github.com/filecoin-project/go-commp-utils/nonffi v0.0.0-20240802040721-2a04ffc8ffe8/go.mod h1:kU2KuSPLB+Xz4FEbVE0abzSN4l6irZ8tqgcYWPVDftU= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= -github.com/filecoin-project/go-crypto v0.0.1 h1:AcvpSGGCgjaY8y1az6AMfKQWreF/pWO2JJGLl6gCq6o= -github.com/filecoin-project/go-crypto v0.0.1/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= -github.com/filecoin-project/go-f3 v0.0.7 h1:dqmxtQXfX1r3hhFZvCszqryg80MZJmfcPFL3nhyHCVA= -github.com/filecoin-project/go-f3 v0.0.7/go.mod h1:ihW5IGLBEuW8pVc9t5MQiAhdzv95EBBfnnrGfMfEbTY= -github.com/filecoin-project/go-fil-commcid v0.0.0-20201016201715-d41df56b4f6a/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ= +github.com/filecoin-project/go-crypto v0.1.0 h1:Pob2MphoipMbe/ksxZOMcQvmBHAd3sI/WEqcbpIsGI0= +github.com/filecoin-project/go-crypto v0.1.0/go.mod h1:K9UFXvvoyAVvB+0Le7oGlKiT9mgA5FHOJdYQXEE8IhI= +github.com/filecoin-project/go-f3 v0.2.0 h1:Gis44+hOrDjSUEw3IDmU7CudNILi5e+bb1pgZgp680k= +github.com/filecoin-project/go-f3 v0.2.0/go.mod h1:43fBLX0iX0+Nnw4Z91wSrdfDYAd6YEDexy7GcLnIJtk= github.com/filecoin-project/go-fil-commcid v0.1.0 h1:3R4ds1A9r6cr8mvZBfMYxTS88OqLYEo6roi+GiIeOh8= github.com/filecoin-project/go-fil-commcid v0.1.0/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ= github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM= @@ -284,11 +281,11 @@ github.com/filecoin-project/go-hamt-ipld v0.1.5/go.mod h1:6Is+ONR5Cd5R6XZoCse1CW github.com/filecoin-project/go-hamt-ipld/v2 v2.0.0 h1:b3UDemBYN2HNfk3KOXNuxgTTxlWi3xVvbQP0IT38fvM= github.com/filecoin-project/go-hamt-ipld/v2 v2.0.0/go.mod h1:7aWZdaQ1b16BVoQUYR+eEvrDCGJoPLxFpDynFjYfBjI= github.com/filecoin-project/go-hamt-ipld/v3 v3.0.1/go.mod h1:gXpNmr3oQx8l3o7qkGyDjJjYSRX7hp/FGOStdqrWyDI= -github.com/filecoin-project/go-hamt-ipld/v3 v3.1.0 h1:rVVNq0x6RGQIzCo1iiJlGFm9AGIZzeifggxtKMU7zmI= github.com/filecoin-project/go-hamt-ipld/v3 v3.1.0/go.mod h1:bxmzgT8tmeVQA1/gvBwFmYdT8SOFUwB3ovSUfG1Ux0g= +github.com/filecoin-project/go-hamt-ipld/v3 v3.4.0 h1:nYs6OPUF8KbZ3E8o9p9HJnQaE8iugjHR5WYVMcicDJc= +github.com/filecoin-project/go-hamt-ipld/v3 v3.4.0/go.mod h1:s0qiHRhFyrgW0SvdQMSJFQxNa4xEIG5XvqCBZUEgcbc= github.com/filecoin-project/go-jsonrpc v0.6.1-0.20240820160949-2cfe810e5d2f h1:0FMH/uwBH7RinWrE+TkiOotYoqxSM54teKx/olJ/cWs= github.com/filecoin-project/go-jsonrpc v0.6.1-0.20240820160949-2cfe810e5d2f/go.mod h1:/n/niXcS4ZQua6i37LcVbY1TmlJR0UIK9mDFQq2ICek= -github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20/go.mod h1:mPn+LRRd5gEKNAtc+r3ScpW2JRU/pj4NBKdADYWHiak= github.com/filecoin-project/go-padreader v0.0.1 h1:8h2tVy5HpoNbr2gBRr+WD6zV6VD6XHig+ynSGJg8ZOs= github.com/filecoin-project/go-padreader v0.0.1/go.mod h1:VYVPJqwpsfmtoHnAmPx6MUwmrK6HIcDqZJiuZhtmfLQ= github.com/filecoin-project/go-paramfetch v0.0.4 h1:H+Me8EL8T5+79z/KHYQQcT8NVOzYVqXIi7nhb48tdm8= @@ -298,8 +295,6 @@ github.com/filecoin-project/go-state-types v0.0.0-20200928172055-2df22083d8ab/go github.com/filecoin-project/go-state-types v0.0.0-20201102161440-c8033295a1fc/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= github.com/filecoin-project/go-state-types v0.1.0/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= github.com/filecoin-project/go-state-types v0.1.6/go.mod h1:UwGVoMsULoCK+bWjEdd/xLCvLAQFBC7EDT477SKml+Q= -github.com/filecoin-project/go-state-types v0.1.10/go.mod h1:UwGVoMsULoCK+bWjEdd/xLCvLAQFBC7EDT477SKml+Q= -github.com/filecoin-project/go-state-types v0.14.0-rc1/go.mod h1:cHpOPup9H1g2T29dKHAjC2sc7/Ef5ypjuW9A3I+e9yY= github.com/filecoin-project/go-state-types v0.14.0 h1:JFw8r/LA0/Hvu865Yn2Gz3R5e2woItKeHTgbT4VsXoU= github.com/filecoin-project/go-state-types v0.14.0/go.mod h1:cDbxwjbmVtV+uNi5D/cFtxKlsRqibnQNlz7xQA1EqYg= github.com/filecoin-project/go-statemachine v1.0.3 h1:N07o6alys+V1tNoSTi4WuuoeNC4erS/6jE74+NsgQuk= @@ -336,7 +331,6 @@ github.com/flynn/noise v1.1.0 h1:KjPQoQCEFdZDiP03phOvGi11+SVVhBG2wOWAorLsstg= github.com/flynn/noise v1.1.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag= github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk= github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY= -github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -469,9 +463,7 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= @@ -531,7 +523,6 @@ github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+l github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru/arc/v2 v2.0.7 h1:QxkVTxwColcduO+LP7eJO56r2hFiG8zEbfAAzRv52KQ= @@ -583,8 +574,6 @@ github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67Fexh github.com/ipfs/go-cid v0.0.6-0.20200501230655-7c82f3b81c00/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog= github.com/ipfs/go-cid v0.0.6/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= -github.com/ipfs/go-cid v0.2.0/go.mod h1:P+HXFDF4CVhaVayiEb4wkAy7zBHxBwsJyt0Y5U6MLro= -github.com/ipfs/go-cid v0.3.2/go.mod h1:gQ8pKqT/sUxGY+tIwy1RPpAojYu7jAyCp5Tz1svoupw= github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= github.com/ipfs/go-cid v0.4.1/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk= github.com/ipfs/go-datastore v0.0.1/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= @@ -610,7 +599,6 @@ github.com/ipfs/go-fs-lock v0.0.6/go.mod h1:OTR+Rj9sHiRubJh3dRhD15Juhd/+w6VPOY28 github.com/ipfs/go-fs-lock v0.0.7 h1:6BR3dajORFrFTkb5EpCUFIAypsoxpGpDSVUdFwzgL9U= github.com/ipfs/go-fs-lock v0.0.7/go.mod h1:Js8ka+FNYmgQRLrRXzU3CB/+Csr1BwrRilEcvYrHhhc= github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08= -github.com/ipfs/go-ipfs-blockstore v1.2.0/go.mod h1:eh8eTFLiINYNSNawfZOC7HOxNTxpB1PFuA5E1m/7exE= github.com/ipfs/go-ipfs-blockstore v1.3.1 h1:cEI9ci7V0sRNivqaOr0elDsamxXFxJMMMy7PTTDQNsQ= github.com/ipfs/go-ipfs-blockstore v1.3.1/go.mod h1:KgtZyc9fq+P2xJUiCAzbRdhhqJHvsw8u2Dlqy2MyRTE= github.com/ipfs/go-ipfs-blocksutil v0.0.1 h1:Eh/H4pc1hsvhzsQoMEP3Bke/aW5P5rVM1IWFJMcGIPQ= @@ -622,7 +610,6 @@ github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1Y github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ= github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-ds-help v0.0.1/go.mod h1:gtP9xRaZXqIQRh1HRpp595KbBEdgqWFxefeVKOV8sxo= -github.com/ipfs/go-ipfs-ds-help v1.1.0/go.mod h1:YR5+6EaebOhfcqVCyqemItCLthrpVNot+rsOU/5IatU= github.com/ipfs/go-ipfs-ds-help v1.1.1 h1:B5UJOH52IbcfS56+Ul+sv8jnIV10lbjLF5eOO0C66Nw= github.com/ipfs/go-ipfs-ds-help v1.1.1/go.mod h1:75vrVCkSdSFidJscs8n4W+77AtTpCIAdDGAwjitJMIo= github.com/ipfs/go-ipfs-exchange-interface v0.0.1/go.mod h1:c8MwfHjtQjPoDyiy9cFquVtVHkO9b9Ob3FG91qJnWCM= @@ -647,15 +634,12 @@ github.com/ipfs/go-ipfs-util v0.0.3/go.mod h1:LHzG1a0Ig4G+iZ26UUOMjHd+lfM84LZCrn github.com/ipfs/go-ipld-cbor v0.0.2/go.mod h1:wTBtrQZA3SoFKMVkp6cn6HMRteIB1VsmHA0AQFOn7Nc= github.com/ipfs/go-ipld-cbor v0.0.3/go.mod h1:wTBtrQZA3SoFKMVkp6cn6HMRteIB1VsmHA0AQFOn7Nc= github.com/ipfs/go-ipld-cbor v0.0.4/go.mod h1:BkCduEx3XBCO6t2Sfo5BaHzuok7hbhdMm9Oh8B2Ftq4= -github.com/ipfs/go-ipld-cbor v0.0.5/go.mod h1:BkCduEx3XBCO6t2Sfo5BaHzuok7hbhdMm9Oh8B2Ftq4= github.com/ipfs/go-ipld-cbor v0.0.6-0.20211211231443-5d9b9e1f6fa8/go.mod h1:ssdxxaLJPXH7OjF5V4NSjBbcfh+evoR4ukuru0oPXMA= github.com/ipfs/go-ipld-cbor v0.0.6/go.mod h1:ssdxxaLJPXH7OjF5V4NSjBbcfh+evoR4ukuru0oPXMA= github.com/ipfs/go-ipld-cbor v0.1.0 h1:dx0nS0kILVivGhfWuB6dUpMa/LAwElHPw1yOGYopoYs= github.com/ipfs/go-ipld-cbor v0.1.0/go.mod h1:U2aYlmVrJr2wsUBU67K4KgepApSZddGRDWBYR0H4sCk= github.com/ipfs/go-ipld-format v0.0.1/go.mod h1:kyJtbkDALmFHv3QR6et67i35QzO3S0dCDnkOJhcZkms= github.com/ipfs/go-ipld-format v0.0.2/go.mod h1:4B6+FM2u9OJ9zCV+kSbgFAZlOrv1Hqbf0INGQgiKf9k= -github.com/ipfs/go-ipld-format v0.3.0/go.mod h1:co/SdBE8h99968X0hViiw1MNlh6fvxxnHpvVLnH7jSM= -github.com/ipfs/go-ipld-format v0.4.0/go.mod h1:co/SdBE8h99968X0hViiw1MNlh6fvxxnHpvVLnH7jSM= github.com/ipfs/go-ipld-format v0.6.0 h1:VEJlA2kQ3LqFSIm5Vu6eIlSxD/Ze90xtc4Meten1F5U= github.com/ipfs/go-ipld-format v0.6.0/go.mod h1:g4QVMTn3marU3qXchwjpKPKgJv+zF+OlaKMyhJ4LHPg= github.com/ipfs/go-ipld-legacy v0.2.1 h1:mDFtrBpmU7b//LzLSypVrXsD8QxkEWxu5qVxN99/+tk= @@ -667,7 +651,6 @@ github.com/ipfs/go-log v1.0.5 h1:2dOuUCB1Z7uoczMWgAyDck5JLb72zHzrMnGnCNNbvY8= github.com/ipfs/go-log v1.0.5/go.mod h1:j0b8ZoR+7+R99LD9jZ6+AJsrzkPbSXbZfGakb5JPtIo= github.com/ipfs/go-log/v2 v2.0.1/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBWFaa9+0= github.com/ipfs/go-log/v2 v2.0.5/go.mod h1:eZs4Xt4ZUJQFM3DlanGhy7TkwwawCZcSByscwkWG+dw= -github.com/ipfs/go-log/v2 v2.1.2-0.20200626104915-0016c0b4b3e4/go.mod h1:2v2nsGfZsvvAJz13SyFzf9ObaqwHiHxsPLEHntrv9KM= github.com/ipfs/go-log/v2 v2.1.2/go.mod h1:2v2nsGfZsvvAJz13SyFzf9ObaqwHiHxsPLEHntrv9KM= github.com/ipfs/go-log/v2 v2.1.3/go.mod h1:/8d0SH3Su5Ooc31QlL1WysJhvyOTDCjcCZ9Axpmri6g= github.com/ipfs/go-log/v2 v2.3.0/go.mod h1:QqGoj30OTpnKaG/LKTGTxoP2mmQtjVMEnK72gynbe/g= @@ -697,7 +680,6 @@ github.com/ipld/go-car/v2 v2.13.1/go.mod h1:QkdjjFNGit2GIkpQ953KBwowuoukoM75nP/J github.com/ipld/go-codec-dagpb v1.6.0 h1:9nYazfyu9B1p3NAgfVdpRco3Fs2nFC72DqVsMj6rOcc= github.com/ipld/go-codec-dagpb v1.6.0/go.mod h1:ANzFhfP2uMJxRBr8CE+WQWs5UsNa0pYtmKZ+agnUw9s= github.com/ipld/go-ipld-prime v0.0.2-0.20191108012745-28a82f04c785/go.mod h1:bDDSvVz7vaK12FNvMeRYnpRFkSUPNQOiCYQezMD/P3w= -github.com/ipld/go-ipld-prime v0.19.0/go.mod h1:Q9j3BaVXwaA3o5JUDNvptDDr/x8+F7FG6XJ8WI3ILg4= github.com/ipld/go-ipld-prime v0.21.0 h1:n4JmcpOlPDIxBcY037SVfpd1G+Sj1nKZah0m6QH9C2E= github.com/ipld/go-ipld-prime v0.21.0/go.mod h1:3RLqy//ERg/y5oShXXdx5YIp50cFGOanyMctpPjsvxQ= github.com/ipld/go-ipld-prime-proto v0.0.0-20191113031812-e32bd156a1e5/go.mod h1:gcvzoEDBjwycpXt3LBE061wT9f46szXGHAmj9uoP6fU= @@ -760,8 +742,8 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= -github.com/kilic/bls12-381 v0.1.0 h1:encrdjqKMEvabVQ7qYOKu1OvhqpK4s47wDYtNiPtlp4= -github.com/kilic/bls12-381 v0.1.0/go.mod h1:vDTTHJONJ6G+P2R74EhnyotQDTliQDnFEwhdmfzw1ig= +github.com/kilic/bls12-381 v0.1.1-0.20220929213557-ca162e8a70f4 h1:xWK4TZ4bRL05WQUU/3x6TG1l+IYAqdXpAeSLt/zZJc4= +github.com/kilic/bls12-381 v0.1.1-0.20220929213557-ca162e8a70f4/go.mod h1:tlkavyke+Ac7h8R3gZIjI5LKBcvMlSWnXNMgT3vZXo8= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -769,13 +751,10 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23 h1:FOOIBWrEkLgmlgGfMuZT83xIwfPDxEI2OHu6xUmJMFE= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= -github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= -github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= -github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.3/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= -github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= -github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= +github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/koron/go-ssdp v0.0.0-20180514024734-4a0ed625a78b/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= @@ -785,7 +764,6 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -937,7 +915,6 @@ github.com/minio/sha256-simd v0.0.0-20190328051042-05b4dd3047e5/go.mod h1:2FMWW+ github.com/minio/sha256-simd v0.1.0/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= -github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= @@ -959,7 +936,6 @@ github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKt github.com/muesli/termenv v0.15.2 h1:GohcuySI0QmI3wN8Ok9PtKGkgkFIk7y6Vpb5PvrY+Wo= github.com/muesli/termenv v0.15.2/go.mod h1:Epx+iuz8sNs7mNKhxzH4fWXGNpZwUaJKRS1noLXviQ8= github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA= -github.com/multiformats/go-base32 v0.0.4/go.mod h1:jNLFzjPZtp3aIARHbJRZIaPuspdH0J6q39uUM5pnABM= github.com/multiformats/go-base32 v0.1.0 h1:pVx9xoSPqEIQG8o+UbAe7DNi51oej1NtK+aGkbLYxPE= github.com/multiformats/go-base32 v0.1.0/go.mod h1:Kj3tFY6zNr+ABYMqeUNeGvkIC/UYgtWibDcT0rExnbI= github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoRdUUOENyW/Vv6sM= @@ -985,7 +961,6 @@ github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/g github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= github.com/multiformats/go-multibase v0.2.0 h1:isdYCVLvksgWlMW9OZRYJEa9pZETFivncJHmHnnd87g= github.com/multiformats/go-multibase v0.2.0/go.mod h1:bFBZX4lKCA/2lyOFSAoKH5SS6oPyjtnzK/XTFDPkNuk= -github.com/multiformats/go-multicodec v0.6.0/go.mod h1:GUC8upxSBE4oG+q3kWZRw/+6yC1BqO550bjhWsJbZlw= github.com/multiformats/go-multicodec v0.9.0 h1:pb/dlPnzee/Sxv/j4PmkDRxCOi3hXTz3IbPKOXWJkmg= github.com/multiformats/go-multicodec v0.9.0/go.mod h1:L3QTQvMIaVBkXOXXtVmYE+LI16i14xuaojr/H7Ai54k= github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= @@ -995,8 +970,6 @@ github.com/multiformats/go-multihash v0.0.9/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa github.com/multiformats/go-multihash v0.0.10/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= github.com/multiformats/go-multihash v0.0.14/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= -github.com/multiformats/go-multihash v0.0.15/go.mod h1:D6aZrWNLFTV/ynMpKsNtB40mJzmCl4jb1alC0OvHiHg= -github.com/multiformats/go-multihash v0.2.1/go.mod h1:WxoMcYG85AZVQUyRyo9s4wULvW5qrI9vb2Lt6evduFc= github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U= github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM= github.com/multiformats/go-multistream v0.1.0/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg= @@ -1004,7 +977,6 @@ github.com/multiformats/go-multistream v0.5.0 h1:5htLSLl7lvJk3xx3qT/8Zm9J4K8vEOf github.com/multiformats/go-multistream v0.5.0/go.mod h1:n6tMZiwiP2wUsR8DgfDWw1dydlEqV3l6N3/GBsX6ILA= github.com/multiformats/go-varint v0.0.1/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= -github.com/multiformats/go-varint v0.0.6/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8= github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= @@ -1109,7 +1081,6 @@ github.com/polydawn/refmt v0.0.0-20190221155625-df39d6c2d992/go.mod h1:uIp+gprXx github.com/polydawn/refmt v0.0.0-20190408063855-01bf1e26dd14/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= github.com/polydawn/refmt v0.0.0-20190807091052-3d65705ee9f1/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= -github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= github.com/polydawn/refmt v0.89.0 h1:ADJTApkvkeBZsN0tBTx8QjpD9JkmxbKp0cxfr9qszm4= github.com/polydawn/refmt v0.89.0/go.mod h1:/zvteZs/GwLtCgZ4BL6CBsk9IKIlexP43ObX9AxTqTw= github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -1166,7 +1137,6 @@ github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJ github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= @@ -1219,7 +1189,6 @@ github.com/smartystreets/assertions v1.13.0/go.mod h1:wDmR7qL282YbGsPy6H/yAsesrx github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa/go.mod h1:2RVY1rIf+2J2o/IM9+vPq9RzmHDSseB7FoXiSNIUsoU= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hgR6gDIPg= github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM= github.com/snadrus/must v0.0.0-20240605044437-98cedd57f8eb h1:78YgPq3NbWnO4xyNhLsn2zitc7NiZpjQZ560rsxVLm4= @@ -1286,12 +1255,10 @@ github.com/valyala/fasttemplate v1.0.1 h1:tY9CJiPnMXf1ERmG2EyK7gNUd+c6RKGD0IfU8W github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= -github.com/warpfork/go-testmark v0.10.0/go.mod h1:jhEf8FVxd+F17juRubpmut64NEG6I2rgkUhlcqqXwE0= github.com/warpfork/go-testmark v0.12.1 h1:rMgCpJfwy1sJ50x0M0NgyphxYYPMOODIJHhsXyEHU0s= github.com/warpfork/go-testmark v0.12.1/go.mod h1:kHwy7wfvGSPh1rQJYKayD4AbtNaeyZdcGi9tNJTaa5Y= github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/warpfork/go-wish v0.0.0-20190328234359-8b3e70f8e830/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= -github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0 h1:GDDkbFiaK8jsSDJfjId/PEGEShv6ugrt4kYsC5UIDaQ= github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/weaveworks/common v0.0.0-20230531151736-e2613bee6b73 h1:CMM9+/AgM77vaMXMQedzqPRMuNwjbI0EcdofPqxc9F8= @@ -1312,11 +1279,8 @@ github.com/whyrusleeping/cbor-gen v0.0.0-20200806213330-63aa96ca5488/go.mod h1:f github.com/whyrusleeping/cbor-gen v0.0.0-20200810223238-211df3b9e24c/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/cbor-gen v0.0.0-20200812213548-958ddffe352c/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/cbor-gen v0.0.0-20210118024343-169e9d70c0c2/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= -github.com/whyrusleeping/cbor-gen v0.0.0-20210303213153-67a261a1d291/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= -github.com/whyrusleeping/cbor-gen v0.0.0-20220323183124-98fa8256a799/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= -github.com/whyrusleeping/cbor-gen v0.1.0/go.mod h1:pM99HXyEbSQHcosHc0iW7YFmwnscr+t9Te4ibko05so= -github.com/whyrusleeping/cbor-gen v0.1.1 h1:eKfcJIoxivjMtwfCfmJAqSF56MHcWqyIScXwaC1VBgw= -github.com/whyrusleeping/cbor-gen v0.1.1/go.mod h1:pM99HXyEbSQHcosHc0iW7YFmwnscr+t9Te4ibko05so= +github.com/whyrusleeping/cbor-gen v0.1.2 h1:WQFlrPhpcQl+M2/3dP5cvlTLWPVsL6LGBb9jJt6l/cA= +github.com/whyrusleeping/cbor-gen v0.1.2/go.mod h1:pM99HXyEbSQHcosHc0iW7YFmwnscr+t9Te4ibko05so= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f/go.mod h1:p9UJB6dDgdPgMJZs7UjUOdulKyRr9fqkS+6JKAInPy8= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 h1:EKhdznlJHPMoKr0XTrX+IlJs1LH3lyx2nfr1dOlZ79k= @@ -1351,6 +1315,10 @@ github.com/zondax/ledger-go v0.14.3 h1:wEpJt2CEcBJ428md/5MgSLsXLBos98sBOyxNmCjfU github.com/zondax/ledger-go v0.14.3/go.mod h1:IKKaoxupuB43g4NxeQmbLXv7T9AlQyie1UpHb342ycI= github.com/zyedidia/generic v1.2.1 h1:Zv5KS/N2m0XZZiuLS82qheRG4X1o5gsWreGb0hR7XDc= github.com/zyedidia/generic v1.2.1/go.mod h1:ly2RBz4mnz1yeuVbQA/VFwGjK3mnHGRj1JuoG336Bis= +gitlab.com/yawning/secp256k1-voi v0.0.0-20230925100816-f2616030848b h1:CzigHMRySiX3drau9C6Q5CAbNIApmLdat5jPMqChvDA= +gitlab.com/yawning/secp256k1-voi v0.0.0-20230925100816-f2616030848b/go.mod h1:/y/V339mxv2sZmYYR64O07VuCpdNZqCTwO8ZcouTMI8= +gitlab.com/yawning/tuplehash v0.0.0-20230713102510-df83abbf9a02 h1:qwDnMxjkyLmAFgcfgTnfJrmYKWhHnci3GjDqcZp1M3Q= +gitlab.com/yawning/tuplehash v0.0.0-20230713102510-df83abbf9a02/go.mod h1:JTnUj0mpYiAsuZLmKjTx/ex3AtMowcCgnE7YNyCEP0I= go.dedis.ch/fixbuf v1.0.3 h1:hGcV9Cd/znUxlusJ64eAlExS+5cJDIyTyEG+otu5wQs= go.dedis.ch/fixbuf v1.0.3/go.mod h1:yzJMt34Wa5xD37V5RTdmp38cz3QhMagdGoem9anUalw= go.dedis.ch/protobuf v1.0.11 h1:FTYVIEzY/bfl37lu3pR4lIj+F9Vp1jE8oh91VmxKgLo= @@ -1433,21 +1401,16 @@ golang.org/x/crypto v0.0.0-20190927123631-a832865fa7ad/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200602180216-279210d13fed/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= -golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= -golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= -golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= -golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= +golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= +golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1532,11 +1495,9 @@ golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= @@ -1632,11 +1593,9 @@ golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201101102859-da207088b7d1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1648,13 +1607,11 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220708085239-5a0f0661e09d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1662,28 +1619,24 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= -golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM= +golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= -golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= -golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= -golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= +golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk= +golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1691,7 +1644,6 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= @@ -1765,8 +1717,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= -golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= +golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9 h1:LLhsEBxRTBLuKlQxFBYUOU8xyFgXv6cOTp2HASDlsDk= +golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= gonum.org/v1/gonum v0.15.0 h1:2lYxjRbTYyxkJxlhC+LvJIx3SsANPdRybu1tGj9/OrQ= gonum.org/v1/gonum v0.15.0/go.mod h1:xzZVBJBtS+Mz4q0Yl2LJTk+OxOg4jiXZ7qBoM0uISGo= google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= @@ -1892,7 +1844,6 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= @@ -1908,8 +1859,6 @@ honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= howett.net/plist v0.0.0-20181124034731-591f970eefbb h1:jhnBjNi9UFpfpl8YZhA9CrOqpnJdvzuiHsl/dnxl11M= howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0= -lukechampine.com/blake3 v1.1.6/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= -lukechampine.com/blake3 v1.1.7/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= lukechampine.com/blake3 v1.3.0 h1:sJ3XhFINmHSrYCgl958hscfIa3bw8x4DqMP3u1YvoYE= lukechampine.com/blake3 v1.3.0/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1LM6k= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= diff --git a/itests/curio_test.go b/itests/curio_test.go index 003853a17..12e9ed3ca 100644 --- a/itests/curio_test.go +++ b/itests/curio_test.go @@ -31,6 +31,7 @@ import ( "github.com/filecoin-project/curio/deps/config" "github.com/filecoin-project/curio/harmony/harmonydb" "github.com/filecoin-project/curio/lib/ffiselect" + storiface "github.com/filecoin-project/curio/lib/storiface" "github.com/filecoin-project/curio/market/lmrpc" "github.com/filecoin-project/curio/tasks/seal" @@ -41,7 +42,6 @@ import ( "github.com/filecoin-project/lotus/cli/spcli/createminer" "github.com/filecoin-project/lotus/itests/kit" "github.com/filecoin-project/lotus/node" - "github.com/filecoin-project/lotus/storage/sealer/storiface" ) func TestCurioNewActor(t *testing.T) { diff --git a/lib/dealdata/dealdata.go b/lib/dealdata/dealdata.go index 8b00c37c4..c3c658a0e 100644 --- a/lib/dealdata/dealdata.go +++ b/lib/dealdata/dealdata.go @@ -18,10 +18,10 @@ import ( "github.com/filecoin-project/curio/harmony/harmonydb" "github.com/filecoin-project/curio/lib/ffi" "github.com/filecoin-project/curio/lib/filler" + "github.com/filecoin-project/curio/lib/storiface" "github.com/filecoin-project/lotus/storage/pipeline/lib/nullreader" "github.com/filecoin-project/lotus/storage/sealer/ffiwrapper" - "github.com/filecoin-project/lotus/storage/sealer/storiface" ) var log = logging.Logger("dealdata") diff --git a/lib/ffi/piece_funcs.go b/lib/ffi/piece_funcs.go index 78cc8a101..b7a582882 100644 --- a/lib/ffi/piece_funcs.go +++ b/lib/ffi/piece_funcs.go @@ -9,8 +9,7 @@ import ( "golang.org/x/xerrors" "github.com/filecoin-project/curio/harmony/harmonytask" - - "github.com/filecoin-project/lotus/storage/sealer/storiface" + storiface "github.com/filecoin-project/curio/lib/storiface" ) func (sb *SealCalls) WritePiece(ctx context.Context, taskID *harmonytask.TaskID, pieceID storiface.PieceNumber, size int64, data io.Reader) error { diff --git a/lib/ffi/sdr_funcs.go b/lib/ffi/sdr_funcs.go index ebe26be86..78dbfd656 100644 --- a/lib/ffi/sdr_funcs.go +++ b/lib/ffi/sdr_funcs.go @@ -19,6 +19,7 @@ import ( "github.com/filecoin-project/curio/harmony/harmonytask" "github.com/filecoin-project/curio/lib/ffiselect" "github.com/filecoin-project/curio/lib/proof" + storiface "github.com/filecoin-project/curio/lib/storiface" // TODO everywhere here that we call this we should call our proxy instead. ffi "github.com/filecoin-project/filecoin-ffi" @@ -28,8 +29,6 @@ import ( "github.com/filecoin-project/curio/lib/paths" "github.com/filecoin-project/curio/lib/proofpaths" - - "github.com/filecoin-project/lotus/storage/sealer/storiface" ) const C1CheckNumber = 3 diff --git a/lib/ffi/sdr_funcs_test.go b/lib/ffi/sdr_funcs_test.go index c4db815e7..6a3edcf2e 100644 --- a/lib/ffi/sdr_funcs_test.go +++ b/lib/ffi/sdr_funcs_test.go @@ -4,7 +4,7 @@ import ( "path/filepath" "testing" - "github.com/filecoin-project/lotus/storage/sealer/storiface" + "github.com/filecoin-project/curio/lib/storiface" ) func TestChangePathType(t *testing.T) { diff --git a/lib/ffi/snap_funcs.go b/lib/ffi/snap_funcs.go index 571e04cc9..7cbfc707d 100644 --- a/lib/ffi/snap_funcs.go +++ b/lib/ffi/snap_funcs.go @@ -21,10 +21,10 @@ import ( "github.com/filecoin-project/curio/lib/ffiselect" paths2 "github.com/filecoin-project/curio/lib/paths" "github.com/filecoin-project/curio/lib/proof" + storiface "github.com/filecoin-project/curio/lib/storiface" "github.com/filecoin-project/curio/lib/tarutil" "github.com/filecoin-project/lotus/storage/sealer/fr32" - "github.com/filecoin-project/lotus/storage/sealer/storiface" ) func (sb *SealCalls) EncodeUpdate( diff --git a/lib/ffi/task_storage.go b/lib/ffi/task_storage.go index 669836352..8924eea87 100644 --- a/lib/ffi/task_storage.go +++ b/lib/ffi/task_storage.go @@ -12,8 +12,7 @@ import ( "github.com/filecoin-project/curio/harmony/harmonytask" "github.com/filecoin-project/curio/harmony/resources" storagePaths "github.com/filecoin-project/curio/lib/paths" - - "github.com/filecoin-project/lotus/storage/sealer/storiface" + storiface "github.com/filecoin-project/curio/lib/storiface" ) type SectorRef struct { diff --git a/lib/ffiselect/ffidirect/ffi-direct.go b/lib/ffiselect/ffidirect/ffi-direct.go index 48dde6c58..3dbb99e9a 100644 --- a/lib/ffiselect/ffidirect/ffi-direct.go +++ b/lib/ffiselect/ffidirect/ffi-direct.go @@ -11,7 +11,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/proof" - "github.com/filecoin-project/lotus/storage/sealer/storiface" + "github.com/filecoin-project/curio/lib/storiface" ) // This allow reflection access to the FFI functions. diff --git a/lib/ffiselect/ffiselect.go b/lib/ffiselect/ffiselect.go index fcb5ee33d..5122c8453 100644 --- a/lib/ffiselect/ffiselect.go +++ b/lib/ffiselect/ffiselect.go @@ -19,8 +19,7 @@ import ( "github.com/filecoin-project/go-state-types/proof" "github.com/filecoin-project/curio/build" - - "github.com/filecoin-project/lotus/storage/sealer/storiface" + "github.com/filecoin-project/curio/lib/storiface" ) type logCtxKt struct{} diff --git a/lib/partialfile/partialfile.go b/lib/partialfile/partialfile.go new file mode 100644 index 000000000..ac02aafbf --- /dev/null +++ b/lib/partialfile/partialfile.go @@ -0,0 +1,346 @@ +package partialfile + +import ( + "encoding/binary" + "io" + "os" + "syscall" + + "github.com/detailyang/go-fallocate" + logging "github.com/ipfs/go-log/v2" + "golang.org/x/xerrors" + + rlepluslazy "github.com/filecoin-project/go-bitfield/rle" + "github.com/filecoin-project/go-state-types/abi" + + "github.com/filecoin-project/curio/lib/storiface" + + "github.com/filecoin-project/lotus/lib/readerutil" + "github.com/filecoin-project/lotus/storage/sealer/fsutil" +) + +var log = logging.Logger("partialfile") + +const veryLargeRle = 1 << 20 + +// Sectors can be partially unsealed. We support this by appending a small +// trailer to each unsealed sector file containing an RLE+ marking which bytes +// in a sector are unsealed, and which are not (holes) + +// unsealed sector files internally have this structure +// [unpadded (raw) data][rle+][4B LE length fo the rle+ field] + +type PartialFile struct { + maxPiece abi.PaddedPieceSize + + path string + allocated rlepluslazy.RLE + + file *os.File +} + +func writeTrailer(maxPieceSize int64, w *os.File, r rlepluslazy.RunIterator) error { + trailer, err := rlepluslazy.EncodeRuns(r, nil) + if err != nil { + return xerrors.Errorf("encoding trailer: %w", err) + } + + // maxPieceSize == unpadded(sectorSize) == trailer start + if _, err := w.Seek(maxPieceSize, io.SeekStart); err != nil { + return xerrors.Errorf("seek to trailer start: %w", err) + } + + rb, err := w.Write(trailer) + if err != nil { + return xerrors.Errorf("writing trailer data: %w", err) + } + + if err := binary.Write(w, binary.LittleEndian, uint32(len(trailer))); err != nil { + return xerrors.Errorf("writing trailer length: %w", err) + } + + return w.Truncate(maxPieceSize + int64(rb) + 4) +} + +func CreatePartialFile(maxPieceSize abi.PaddedPieceSize, path string) (*PartialFile, error) { + f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0644) // nolint + if err != nil { + return nil, xerrors.Errorf("opening partial file '%s': %w", path, err) + } + + err = func() error { + err := fallocate.Fallocate(f, 0, int64(maxPieceSize)) + if errno, ok := err.(syscall.Errno); ok { + if errno == syscall.EOPNOTSUPP || errno == syscall.ENOSYS { + log.Warnf("could not allocate space, ignoring: %v", errno) + err = nil // log and ignore + } + } + if err != nil { + return xerrors.Errorf("fallocate '%s': %w", path, err) + } + + if err := writeTrailer(int64(maxPieceSize), f, &rlepluslazy.RunSliceIterator{}); err != nil { + return xerrors.Errorf("writing trailer: %w", err) + } + + return nil + }() + if err != nil { + _ = f.Close() + return nil, err + } + if err := f.Close(); err != nil { + return nil, xerrors.Errorf("close empty partial file: %w", err) + } + + return OpenPartialFile(maxPieceSize, path) +} + +func OpenPartialFile(maxPieceSize abi.PaddedPieceSize, path string) (*PartialFile, error) { + f, err := os.OpenFile(path, os.O_RDWR, 0644) // nolint + if err != nil { + return nil, xerrors.Errorf("opening partial file '%s': %w", path, err) + } + + st, err := f.Stat() + if err != nil { + return nil, xerrors.Errorf("stat '%s': %w", path, err) + } + if st.Size() < int64(maxPieceSize) { + return nil, xerrors.Errorf("sector file '%s' was smaller than the sector size %d < %d", path, st.Size(), maxPieceSize) + } + if st.Size() == int64(maxPieceSize) { + log.Debugw("no partial file trailer, assuming fully allocated", "path", path) + + allAlloc := &rlepluslazy.RunSliceIterator{Runs: []rlepluslazy.Run{{Val: true, Len: uint64(maxPieceSize)}}} + enc, err := rlepluslazy.EncodeRuns(allAlloc, []byte{}) + if err != nil { + return nil, xerrors.Errorf("encoding full allocation: %w", err) + } + + rle, err := rlepluslazy.FromBuf(enc) + if err != nil { + return nil, xerrors.Errorf("decoding full allocation: %w", err) + } + + return &PartialFile{ + maxPiece: maxPieceSize, + path: path, + allocated: rle, + file: f, + }, nil + } + + var rle rlepluslazy.RLE + err = func() error { + // read trailer + var tlen [4]byte + _, err = f.ReadAt(tlen[:], st.Size()-int64(len(tlen))) + if err != nil { + return xerrors.Errorf("reading trailer length: %w", err) + } + + // sanity-check the length + trailerLen := binary.LittleEndian.Uint32(tlen[:]) + expectLen := int64(trailerLen) + int64(len(tlen)) + int64(maxPieceSize) + if expectLen != st.Size() { + return xerrors.Errorf("file '%s' has inconsistent length; has %d bytes; expected %d (%d trailer, %d sector data)", path, st.Size(), expectLen, int64(trailerLen)+int64(len(tlen)), maxPieceSize) + } + if trailerLen > veryLargeRle { + log.Warnf("Partial file '%s' has a VERY large trailer with %d bytes", path, trailerLen) + } + + trailerStart := st.Size() - int64(len(tlen)) - int64(trailerLen) + if trailerStart != int64(maxPieceSize) { + return xerrors.Errorf("expected sector size to equal trailer start index") + } + + trailerBytes := make([]byte, trailerLen) + _, err = f.ReadAt(trailerBytes, trailerStart) + if err != nil { + return xerrors.Errorf("reading trailer: %w", err) + } + + rle, err = rlepluslazy.FromBuf(trailerBytes) + if err != nil { + return xerrors.Errorf("decoding trailer: %w", err) + } + + it, err := rle.RunIterator() + if err != nil { + return xerrors.Errorf("getting trailer run iterator: %w", err) + } + + f, err := rlepluslazy.Fill(it) + if err != nil { + return xerrors.Errorf("filling bitfield: %w", err) + } + lastSet, err := rlepluslazy.Count(f) + if err != nil { + return xerrors.Errorf("finding last set byte index: %w", err) + } + + if lastSet > uint64(maxPieceSize) { + return xerrors.Errorf("last set byte at index higher than sector size: %d > %d", lastSet, maxPieceSize) + } + + return nil + }() + if err != nil { + _ = f.Close() + return nil, err + } + + return &PartialFile{ + maxPiece: maxPieceSize, + path: path, + allocated: rle, + file: f, + }, nil +} + +func (pf *PartialFile) Close() error { + return pf.file.Close() +} + +func (pf *PartialFile) Writer(offset storiface.PaddedByteIndex, size abi.PaddedPieceSize) (io.Writer, error) { + if _, err := pf.file.Seek(int64(offset), io.SeekStart); err != nil { + return nil, xerrors.Errorf("seek piece start: %w", err) + } + + { + have, err := pf.allocated.RunIterator() + if err != nil { + return nil, err + } + + and, err := rlepluslazy.And(have, PieceRun(offset, size)) + if err != nil { + return nil, err + } + + c, err := rlepluslazy.Count(and) + if err != nil { + return nil, err + } + + if c > 0 { + log.Warnf("getting partial file writer overwriting %d allocated bytes", c) + } + } + + return pf.file, nil +} + +func (pf *PartialFile) MarkAllocated(offset storiface.PaddedByteIndex, size abi.PaddedPieceSize) error { + have, err := pf.allocated.RunIterator() + if err != nil { + return err + } + + ored, err := rlepluslazy.Or(have, PieceRun(offset, size)) + if err != nil { + return err + } + + if err := writeTrailer(int64(pf.maxPiece), pf.file, ored); err != nil { + return xerrors.Errorf("writing trailer: %w", err) + } + + return nil +} + +func (pf *PartialFile) Free(offset storiface.PaddedByteIndex, size abi.PaddedPieceSize) error { + have, err := pf.allocated.RunIterator() + if err != nil { + return err + } + + if err := fsutil.Deallocate(pf.file, int64(offset), int64(size)); err != nil { + return xerrors.Errorf("deallocating: %w", err) + } + + s, err := rlepluslazy.Subtract(have, PieceRun(offset, size)) + if err != nil { + return err + } + + if err := writeTrailer(int64(pf.maxPiece), pf.file, s); err != nil { + return xerrors.Errorf("writing trailer: %w", err) + } + + return nil +} + +// Reader forks off a new reader from the underlying file, and returns a reader +// starting at the given offset and reading the given size. Safe for concurrent +// use. +func (pf *PartialFile) Reader(offset storiface.PaddedByteIndex, size abi.PaddedPieceSize) (io.Reader, error) { + if _, err := pf.file.Seek(int64(offset), io.SeekStart); err != nil { + return nil, xerrors.Errorf("seek piece start: %w", err) + } + + { + have, err := pf.allocated.RunIterator() + if err != nil { + return nil, err + } + + and, err := rlepluslazy.And(have, PieceRun(offset, size)) + if err != nil { + return nil, err + } + + c, err := rlepluslazy.Count(and) + if err != nil { + return nil, err + } + + if c != uint64(size) { + log.Warnf("getting partial file reader reading %d unallocated bytes", uint64(size)-c) + } + } + + return io.LimitReader(readerutil.NewReadSeekerFromReaderAt(pf.file, int64(offset)), int64(size)), nil +} + +func (pf *PartialFile) Allocated() (rlepluslazy.RunIterator, error) { + return pf.allocated.RunIterator() +} + +func (pf *PartialFile) HasAllocated(offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize) (bool, error) { + have, err := pf.Allocated() + if err != nil { + return false, err + } + + u, err := rlepluslazy.And(have, PieceRun(offset.Padded(), size.Padded())) + if err != nil { + return false, err + } + + uc, err := rlepluslazy.Count(u) + if err != nil { + return false, err + } + + return abi.PaddedPieceSize(uc) == size.Padded(), nil +} + +func PieceRun(offset storiface.PaddedByteIndex, size abi.PaddedPieceSize) rlepluslazy.RunIterator { + var runs []rlepluslazy.Run + if offset > 0 { + runs = append(runs, rlepluslazy.Run{ + Val: false, + Len: uint64(offset), + }) + } + + runs = append(runs, rlepluslazy.Run{ + Val: true, + Len: uint64(size), + }) + + return &rlepluslazy.RunSliceIterator{Runs: runs} +} diff --git a/lib/paths/db_index.go b/lib/paths/db_index.go index 54d42f1ec..e30ffbb76 100644 --- a/lib/paths/db_index.go +++ b/lib/paths/db_index.go @@ -21,10 +21,10 @@ import ( "github.com/filecoin-project/curio/harmony/harmonydb" "github.com/filecoin-project/curio/lib/paths/alertinginterface" + storiface "github.com/filecoin-project/curio/lib/storiface" "github.com/filecoin-project/lotus/metrics" "github.com/filecoin-project/lotus/storage/sealer/fsutil" - "github.com/filecoin-project/lotus/storage/sealer/storiface" ) const NoMinerFilter = abi.ActorID(0) diff --git a/lib/paths/http_handler.go b/lib/paths/http_handler.go index 936fffdda..c308dedf5 100644 --- a/lib/paths/http_handler.go +++ b/lib/paths/http_handler.go @@ -15,10 +15,9 @@ import ( "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/curio/lib/partialfile" + "github.com/filecoin-project/curio/lib/storiface" "github.com/filecoin-project/curio/lib/tarutil" - - "github.com/filecoin-project/lotus/storage/sealer/partialfile" - "github.com/filecoin-project/lotus/storage/sealer/storiface" ) var log = logging.Logger("stores") diff --git a/lib/paths/http_handler_test.go b/lib/paths/http_handler_test.go index 55ee45929..e5d067a35 100644 --- a/lib/paths/http_handler_test.go +++ b/lib/paths/http_handler_test.go @@ -15,11 +15,10 @@ import ( "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/curio/lib/partialfile" "github.com/filecoin-project/curio/lib/paths" "github.com/filecoin-project/curio/lib/paths/mocks" - - "github.com/filecoin-project/lotus/storage/sealer/partialfile" - "github.com/filecoin-project/lotus/storage/sealer/storiface" + storiface "github.com/filecoin-project/curio/lib/storiface" ) func TestRemoteGetAllocated(t *testing.T) { diff --git a/lib/paths/index.go b/lib/paths/index.go index c217518dc..35e667fe1 100644 --- a/lib/paths/index.go +++ b/lib/paths/index.go @@ -9,8 +9,9 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" + storiface "github.com/filecoin-project/curio/lib/storiface" + "github.com/filecoin-project/lotus/storage/sealer/fsutil" - "github.com/filecoin-project/lotus/storage/sealer/storiface" ) var HeartbeatInterval = 10 * time.Second diff --git a/lib/paths/index_locks.go b/lib/paths/index_locks.go index ab95cb4a7..c5c2e8f87 100644 --- a/lib/paths/index_locks.go +++ b/lib/paths/index_locks.go @@ -9,7 +9,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/lotus/storage/sealer/storiface" + storiface "github.com/filecoin-project/curio/lib/storiface" ) type sectorLock struct { diff --git a/lib/paths/index_locks_test.go b/lib/paths/index_locks_test.go index d3134055b..d53e4da80 100644 --- a/lib/paths/index_locks_test.go +++ b/lib/paths/index_locks_test.go @@ -9,7 +9,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/lotus/storage/sealer/storiface" + "github.com/filecoin-project/curio/lib/storiface" ) var aSector = abi.SectorID{ diff --git a/lib/paths/interface.go b/lib/paths/interface.go index f376ef284..3b04225ce 100644 --- a/lib/paths/interface.go +++ b/lib/paths/interface.go @@ -8,9 +8,10 @@ import ( "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/curio/lib/partialfile" + "github.com/filecoin-project/curio/lib/storiface" + "github.com/filecoin-project/lotus/storage/sealer/fsutil" - "github.com/filecoin-project/lotus/storage/sealer/partialfile" - "github.com/filecoin-project/lotus/storage/sealer/storiface" ) //go:generate go run github.com/golang/mock/mockgen -destination=mocks/pf.go -package=mocks . PartialFileHandler diff --git a/lib/paths/local.go b/lib/paths/local.go index f2e94f96a..9510d7704 100644 --- a/lib/paths/local.go +++ b/lib/paths/local.go @@ -22,11 +22,11 @@ import ( "github.com/filecoin-project/go-state-types/proof" cuproof "github.com/filecoin-project/curio/lib/proof" + "github.com/filecoin-project/curio/lib/storiface" "github.com/filecoin-project/curio/lib/supraffi" "github.com/filecoin-project/lotus/lib/result" "github.com/filecoin-project/lotus/storage/sealer/fsutil" - "github.com/filecoin-project/lotus/storage/sealer/storiface" ) type LocalStorage interface { diff --git a/lib/paths/local_test.go b/lib/paths/local_test.go index f9b5e24a3..92c9b5074 100644 --- a/lib/paths/local_test.go +++ b/lib/paths/local_test.go @@ -11,9 +11,9 @@ import ( "github.com/stretchr/testify/require" "github.com/filecoin-project/curio/harmony/harmonydb" + storiface "github.com/filecoin-project/curio/lib/storiface" "github.com/filecoin-project/lotus/storage/sealer/fsutil" - "github.com/filecoin-project/lotus/storage/sealer/storiface" ) const pathSize = 16 << 20 diff --git a/lib/paths/localstorage.go b/lib/paths/localstorage.go index 6f67a18f8..d3050a776 100644 --- a/lib/paths/localstorage.go +++ b/lib/paths/localstorage.go @@ -1,9 +1,19 @@ package paths import ( - "github.com/filecoin-project/lotus/node/config" + "encoding/json" + "errors" + "io" + "io/fs" + "os" + gopath "path" + + "github.com/mitchellh/go-homedir" + "golang.org/x/xerrors" + + "github.com/filecoin-project/curio/lib/storiface" + "github.com/filecoin-project/lotus/storage/sealer/fsutil" - "github.com/filecoin-project/lotus/storage/sealer/storiface" ) type BasicLocalStorage struct { @@ -14,7 +24,7 @@ var _ LocalStorage = &BasicLocalStorage{} func (ls *BasicLocalStorage) GetStorage() (storiface.StorageConfig, error) { var def storiface.StorageConfig - c, err := config.StorageFromFile(ls.PathToJSON, &def) + c, err := StorageFromFile(ls.PathToJSON, &def) if err != nil { return storiface.StorageConfig{}, err } @@ -23,12 +33,12 @@ func (ls *BasicLocalStorage) GetStorage() (storiface.StorageConfig, error) { func (ls *BasicLocalStorage) SetStorage(f func(*storiface.StorageConfig)) error { var def storiface.StorageConfig - c, err := config.StorageFromFile(ls.PathToJSON, &def) + c, err := StorageFromFile(ls.PathToJSON, &def) if err != nil { return err } f(c) - return config.WriteStorageFile(ls.PathToJSON, *c) + return WriteStorageFile(ls.PathToJSON, *c) } func (ls *BasicLocalStorage) Stat(path string) (fsutil.FsStat, error) { @@ -42,3 +52,69 @@ func (ls *BasicLocalStorage) DiskUsage(path string) (int64, error) { } return si.OnDisk, nil } + +func StorageFromFile(path string, def *storiface.StorageConfig) (*storiface.StorageConfig, error) { + path, err := homedir.Expand(path) + if err != nil { + return nil, xerrors.Errorf("expanding storage config path: %w", err) + } + + file, err := os.Open(path) + switch { + case os.IsNotExist(err): + if def == nil { + return nil, xerrors.Errorf("couldn't load storage config: %w", err) + } + return def, nil + case err != nil: + return nil, err + } + + defer file.Close() //nolint:errcheck // The file is RO + return StorageFromReader(file) +} + +func StorageFromReader(reader io.Reader) (*storiface.StorageConfig, error) { + var cfg storiface.StorageConfig + err := json.NewDecoder(reader).Decode(&cfg) + if err != nil { + return nil, err + } + + return &cfg, nil +} + +func WriteStorageFile(filePath string, config storiface.StorageConfig) error { + filePath, err := homedir.Expand(filePath) + if err != nil { + return xerrors.Errorf("expanding storage config path: %w", err) + } + + b, err := json.MarshalIndent(config, "", " ") + if err != nil { + return xerrors.Errorf("marshaling storage config: %w", err) + } + + info, err := os.Stat(filePath) + if err != nil { + if !errors.Is(err, fs.ErrNotExist) { + return xerrors.Errorf("statting storage config (%s): %w", filePath, err) + } + if gopath.Base(filePath) == "." { + filePath = gopath.Join(filePath, "storage.json") + } + } else { + if info.IsDir() || gopath.Base(filePath) == "." { + filePath = gopath.Join(filePath, "storage.json") + } + } + + if err := os.MkdirAll(gopath.Dir(filePath), 0755); err != nil { + return xerrors.Errorf("making storage config parent directory: %w", err) + } + if err := os.WriteFile(filePath, b, 0644); err != nil { + return xerrors.Errorf("persisting storage config (%s): %w", filePath, err) + } + + return nil +} diff --git a/lib/paths/localstorage_cached.go b/lib/paths/localstorage_cached.go index 9289852b4..fd7560cf2 100644 --- a/lib/paths/localstorage_cached.go +++ b/lib/paths/localstorage_cached.go @@ -7,8 +7,9 @@ import ( lru "github.com/hashicorp/golang-lru/v2" + "github.com/filecoin-project/curio/lib/storiface" + "github.com/filecoin-project/lotus/storage/sealer/fsutil" - "github.com/filecoin-project/lotus/storage/sealer/storiface" ) var StatTimeout = 5 * time.Second diff --git a/lib/paths/mocks/index.go b/lib/paths/mocks/index.go index f64b80dea..082d22908 100644 --- a/lib/paths/mocks/index.go +++ b/lib/paths/mocks/index.go @@ -13,9 +13,9 @@ import ( abi "github.com/filecoin-project/go-state-types/abi" paths "github.com/filecoin-project/curio/lib/paths" + storiface "github.com/filecoin-project/curio/lib/storiface" fsutil "github.com/filecoin-project/lotus/storage/sealer/fsutil" - storiface "github.com/filecoin-project/lotus/storage/sealer/storiface" ) // MockSectorIndex is a mock of SectorIndex interface. diff --git a/lib/paths/mocks/pf.go b/lib/paths/mocks/pf.go index e1604be79..072e47b2e 100644 --- a/lib/paths/mocks/pf.go +++ b/lib/paths/mocks/pf.go @@ -12,8 +12,8 @@ import ( abi "github.com/filecoin-project/go-state-types/abi" - partialfile "github.com/filecoin-project/lotus/storage/sealer/partialfile" - storiface "github.com/filecoin-project/lotus/storage/sealer/storiface" + partialfile "github.com/filecoin-project/curio/lib/partialfile" + storiface "github.com/filecoin-project/curio/lib/storiface" ) // MockPartialFileHandler is a mock of PartialFileHandler interface. diff --git a/lib/paths/mocks/store.go b/lib/paths/mocks/store.go index 2ee11327a..e7c3c184c 100644 --- a/lib/paths/mocks/store.go +++ b/lib/paths/mocks/store.go @@ -13,8 +13,9 @@ import ( abi "github.com/filecoin-project/go-state-types/abi" + storiface "github.com/filecoin-project/curio/lib/storiface" + fsutil "github.com/filecoin-project/lotus/storage/sealer/fsutil" - storiface "github.com/filecoin-project/lotus/storage/sealer/storiface" ) // MockStore is a mock of Store interface. diff --git a/lib/paths/remote.go b/lib/paths/remote.go index ed696bf76..c0ff6322c 100644 --- a/lib/paths/remote.go +++ b/lib/paths/remote.go @@ -20,9 +20,10 @@ import ( "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/curio/lib/partialfile" + storiface "github.com/filecoin-project/curio/lib/storiface" + "github.com/filecoin-project/lotus/storage/sealer/fsutil" - "github.com/filecoin-project/lotus/storage/sealer/partialfile" - "github.com/filecoin-project/lotus/storage/sealer/storiface" ) var FetchTempSubdir = "fetching" diff --git a/lib/paths/remote_prove.go b/lib/paths/remote_prove.go index 5185f0c6d..edb83b9aa 100644 --- a/lib/paths/remote_prove.go +++ b/lib/paths/remote_prove.go @@ -16,9 +16,8 @@ import ( "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/curio/lib/storiface" "github.com/filecoin-project/curio/lib/tarutil" - - "github.com/filecoin-project/lotus/storage/sealer/storiface" ) // ReadMinCacheInto reads finalized-like (few MiB) cache files into the target dir diff --git a/lib/paths/remote_test.go b/lib/paths/remote_test.go index 9c070f2f2..7602eb116 100644 --- a/lib/paths/remote_test.go +++ b/lib/paths/remote_test.go @@ -22,12 +22,10 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/curio/harmony/harmonydb" + "github.com/filecoin-project/curio/lib/partialfile" "github.com/filecoin-project/curio/lib/paths" "github.com/filecoin-project/curio/lib/paths/mocks" - - "github.com/filecoin-project/lotus/node/repo" - "github.com/filecoin-project/lotus/storage/sealer/partialfile" - "github.com/filecoin-project/lotus/storage/sealer/storiface" + storiface "github.com/filecoin-project/curio/lib/storiface" ) const metaFile = "sectorstore.json" @@ -70,23 +68,9 @@ func TestMoveShared(t *testing.T) { dir := t.TempDir() - openRepo := func(dir string) repo.LockedRepo { - r, err := repo.NewFS(dir) - require.NoError(t, err) - require.NoError(t, r.Init(repo.Worker)) - lr, err := r.Lock(repo.Worker) - require.NoError(t, err) - - t.Cleanup(func() { - _ = lr.Close() - }) - - err = lr.SetStorage(func(config *storiface.StorageConfig) { - *config = storiface.StorageConfig{} - }) - require.NoError(t, err) - - return lr + openRepo := func(dir string) paths.LocalStorage { + bls := &paths.BasicLocalStorage{PathToJSON: filepath.Join(t.TempDir(), "storage.json")} + return bls } // setup two repos with two storage paths: diff --git a/lib/storiface/cbor_gen.go b/lib/storiface/cbor_gen.go new file mode 100644 index 000000000..e1d5f6467 --- /dev/null +++ b/lib/storiface/cbor_gen.go @@ -0,0 +1,343 @@ +// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. + +package storiface + +import ( + "fmt" + "io" + "math" + "sort" + + cid "github.com/ipfs/go-cid" + cbg "github.com/whyrusleeping/cbor-gen" + xerrors "golang.org/x/xerrors" +) + +var _ = xerrors.Errorf +var _ = cid.Undef +var _ = math.E +var _ = sort.Sort + +func (t *SecDataHttpHeader) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{162}); err != nil { + return err + } + + // t.Key (string) (string) + if len("Key") > 8192 { + return xerrors.Errorf("Value in field \"Key\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Key"))); err != nil { + return err + } + if _, err := cw.WriteString(string("Key")); err != nil { + return err + } + + if len(t.Key) > 8192 { + return xerrors.Errorf("Value in field t.Key was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Key))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.Key)); err != nil { + return err + } + + // t.Value (string) (string) + if len("Value") > 8192 { + return xerrors.Errorf("Value in field \"Value\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Value"))); err != nil { + return err + } + if _, err := cw.WriteString(string("Value")); err != nil { + return err + } + + if len(t.Value) > 8192 { + return xerrors.Errorf("Value in field t.Value was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Value))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.Value)); err != nil { + return err + } + return nil +} + +func (t *SecDataHttpHeader) UnmarshalCBOR(r io.Reader) (err error) { + *t = SecDataHttpHeader{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("SecDataHttpHeader: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadStringWithMax(cr, 8192) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Key (string) (string) + case "Key": + + { + sval, err := cbg.ReadStringWithMax(cr, 8192) + if err != nil { + return err + } + + t.Key = string(sval) + } + // t.Value (string) (string) + case "Value": + + { + sval, err := cbg.ReadStringWithMax(cr, 8192) + if err != nil { + return err + } + + t.Value = string(sval) + } + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} +func (t *SectorLocation) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{163}); err != nil { + return err + } + + // t.URL (string) (string) + if len("URL") > 8192 { + return xerrors.Errorf("Value in field \"URL\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("URL"))); err != nil { + return err + } + if _, err := cw.WriteString(string("URL")); err != nil { + return err + } + + if len(t.URL) > 8192 { + return xerrors.Errorf("Value in field t.URL was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.URL))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.URL)); err != nil { + return err + } + + // t.Local (bool) (bool) + if len("Local") > 8192 { + return xerrors.Errorf("Value in field \"Local\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Local"))); err != nil { + return err + } + if _, err := cw.WriteString(string("Local")); err != nil { + return err + } + + if err := cbg.WriteBool(w, t.Local); err != nil { + return err + } + + // t.Headers ([]storiface.SecDataHttpHeader) (slice) + if len("Headers") > 8192 { + return xerrors.Errorf("Value in field \"Headers\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Headers"))); err != nil { + return err + } + if _, err := cw.WriteString(string("Headers")); err != nil { + return err + } + + if len(t.Headers) > 8192 { + return xerrors.Errorf("Slice value in field t.Headers was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Headers))); err != nil { + return err + } + for _, v := range t.Headers { + if err := v.MarshalCBOR(cw); err != nil { + return err + } + + } + return nil +} + +func (t *SectorLocation) UnmarshalCBOR(r io.Reader) (err error) { + *t = SectorLocation{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("SectorLocation: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadStringWithMax(cr, 8192) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.URL (string) (string) + case "URL": + + { + sval, err := cbg.ReadStringWithMax(cr, 8192) + if err != nil { + return err + } + + t.URL = string(sval) + } + // t.Local (bool) (bool) + case "Local": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + if maj != cbg.MajOther { + return fmt.Errorf("booleans must be major type 7") + } + switch extra { + case 20: + t.Local = false + case 21: + t.Local = true + default: + return fmt.Errorf("booleans are either major type 7, value 20 or 21 (got %d)", extra) + } + // t.Headers ([]storiface.SecDataHttpHeader) (slice) + case "Headers": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > 8192 { + return fmt.Errorf("t.Headers: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + + if extra > 0 { + t.Headers = make([]SecDataHttpHeader, extra) + } + + for i := 0; i < int(extra); i++ { + { + var maj byte + var extra uint64 + var err error + _ = maj + _ = extra + _ = err + + { + + if err := t.Headers[i].UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Headers[i]: %w", err) + } + + } + + } + } + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} diff --git a/lib/storiface/ffi.go b/lib/storiface/ffi.go new file mode 100644 index 000000000..4a9f832b8 --- /dev/null +++ b/lib/storiface/ffi.go @@ -0,0 +1,41 @@ +package storiface + +import ( + "context" + "errors" + + "github.com/ipfs/go-cid" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-state-types/abi" + + "github.com/filecoin-project/lotus/storage/sealer/fr32" +) + +var ErrSectorNotFound = errors.New("sector not found") + +type UnpaddedByteIndex uint64 + +func (i UnpaddedByteIndex) Padded() PaddedByteIndex { + return PaddedByteIndex(abi.UnpaddedPieceSize(i).Padded()) +} + +func (i UnpaddedByteIndex) Valid() error { + if i%127 != 0 { + return xerrors.Errorf("unpadded byte index must be a multiple of 127") + } + + return nil +} + +func UnpaddedFloor(n uint64) UnpaddedByteIndex { + return UnpaddedByteIndex(n / uint64(fr32.UnpaddedFr32Chunk) * uint64(fr32.UnpaddedFr32Chunk)) +} + +func UnpaddedCeil(n uint64) UnpaddedByteIndex { + return UnpaddedByteIndex((n + uint64(fr32.UnpaddedFr32Chunk-1)) / uint64(fr32.UnpaddedFr32Chunk) * uint64(fr32.UnpaddedFr32Chunk)) +} + +type PaddedByteIndex uint64 + +type RGetter func(ctx context.Context, id abi.SectorID) (sealed cid.Cid, update bool, err error) diff --git a/lib/storiface/filetype.go b/lib/storiface/filetype.go new file mode 100644 index 000000000..56983a25d --- /dev/null +++ b/lib/storiface/filetype.go @@ -0,0 +1,460 @@ +package storiface + +import ( + "fmt" + + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-state-types/abi" +) + +// FTUnsealed represents an unsealed sector file. +// FTSealed represents a sealed sector file. +// FTCache represents a cache sector file. +// FTUpdate represents an update sector file. +// FTUpdateCache represents an update cache sector file. +// FTPiece represents a Piece Park sector file. +// FileTypes represents the total number of file. +// +// The SectorFileType type is an integer type that represents different sector file. +// It has several methods to manipulate and query the file. +// The String method returns a string representation of the file. +// The Strings method returns a slice of string representations of all the file that are set in the receiver object. +// The AllSet method returns a slice of all the file that are set in the receiver object. +// The Has method checks whether a specific file type is set in the receiver object. +// The SealSpaceUse method calculates the space used by the receiver object in sealing a sector of a given size. +// The SubAllowed method removes selected file from the receiver object based on a list of allowed and denied file. +// The Unset method removes selected file from the receiver object. +// The AnyAllowed method checks whether any file in the receiver object are allowed based on a list of allowed and denied file. +// The Allowed method checks whether all file in the receiver object are allowed based on a list of allowed and denied file. +// The StoreSpaceUse method calculates the space used by the receiver object in storing a sector of a given size. +// The All method returns an array that represents which file are set in the receiver object. +// The IsNone method checks whether the receiver object represents no file. +const ( + // "regular" sectors + FTUnsealed SectorFileType = 1 << iota + FTSealed + FTCache + + // snap + FTUpdate + FTUpdateCache + + // Piece Park + FTPiece + + FileTypes = iota +) + +// PathTypes is a slice of SectorFileType that represents different types of sector file paths. +// It contains the following types of sector file paths: +// - FTUnsealed: represents regular unsealed sectors +// - FTSealed: represents sealed sectors +// - FTCache: represents cache sectors +// - FTUpdate: represents snap sectors +// - FTUpdateCache: represents snap cache sectors +// - FTPiece: represents Piece Park sectors +var PathTypes = []SectorFileType{FTUnsealed, FTSealed, FTCache, FTUpdate, FTUpdateCache, FTPiece} + +// FTNone represents a sector file type of none. This constant is used in the StorageLock method to specify that a sector should not have any file locked. +// Example usage: +// err := m.index.StorageLock(ctx, sector.ID, storiface.FTNone, storiface.FTSealed|storiface.FTUnsealed|storiface.FTCache) +const ( + FTNone SectorFileType = 0 +) + +// FTAll represents the combination of all available sector file. +// It is a variable of type SectorFileType. +// The value of FTAll is calculated by iterating over the PathTypes slice and using the |= operator to perform a bitwise OR operation on each path type. +// The result is assigned to the variable out and returned. +// FTAll is immediately invoked as a function using the anonymous function syntax, so the result is returned as soon as it is calculated. +var FTAll = func() (out SectorFileType) { + for _, pathType := range PathTypes { + out |= pathType + } + return out +}() + +// FSOverheadDen represents the constant value 10, which is used to calculate the overhead in various storage space utilization calculations. +const FSOverheadDen = 10 + +// FSOverheadSeal is a map that represents the overheads for different SectorFileType in sectors which are being sealed. +var FSOverheadSeal = map[SectorFileType]int{ // 10x overheads + FTUnsealed: FSOverheadDen, + FTSealed: FSOverheadDen, + FTUpdate: FSOverheadDen, + FTUpdateCache: FSOverheadDen*2 + 1, + FTCache: 141, // 11 layers + D(2x ssize) + C + R' + FTPiece: FSOverheadDen, +} + +// sector size * disk / fs overhead. FSOverheadDen is like the unit of sector size + +// FsOverheadFinalized is a map that represents the finalized overhead for different types of SectorFileType. +// The keys in the map are the SectorFileType values, and the values are integers representing the overhead. +// It is used to calculate the storage space usage for different types of sectors, as shown in the example below: +// The overhead value is retrieved from FsOverheadFinalized by using the SectorFileType value as the key. +// If the overhead value is not found in the map, an error is returned indicating that there is no finalized +// overhead information for the given sector type. +var FsOverheadFinalized = map[SectorFileType]int{ + FTUnsealed: FSOverheadDen, + FTSealed: FSOverheadDen, + FTUpdate: FSOverheadDen, + FTUpdateCache: 1, + FTCache: 1, + FTPiece: FSOverheadDen, +} + +// SectorFileType represents the type of a sector file +// TypeFromString converts a string to a SectorFileType +type SectorFileType int + +// TypeFromString converts a string representation of a SectorFileType to its corresponding value. +// It returns the SectorFileType and nil error if the string matches one of the existing types. +// If the string does not match any type, it returns 0 and an error. +func TypeFromString(s string) (SectorFileType, error) { + switch s { + case "unsealed": + return FTUnsealed, nil + case "sealed": + return FTSealed, nil + case "cache": + return FTCache, nil + case "update": + return FTUpdate, nil + case "update-cache": + return FTUpdateCache, nil + case "piece": + return FTPiece, nil + default: + return 0, xerrors.Errorf("unknown sector file type '%s'", s) + } +} + +// String returns a string representation of the SectorFileType. +func (t SectorFileType) String() string { + switch t { + case FTUnsealed: + return "unsealed" + case FTSealed: + return "sealed" + case FTCache: + return "cache" + case FTUpdate: + return "update" + case FTUpdateCache: + return "update-cache" + case FTPiece: + return "piece" + default: + return fmt.Sprintf("", t, (t & ((1 << FileTypes) - 1)).Strings()) + } +} + +// Strings returns a slice of strings representing the names of the SectorFileType values that are set in the receiver value. +// Example usage: +// +// fileType := SectorFileType(FTSealed | FTCache) +// names := fileType.Strings() // names = ["sealed", "cache"] +// fmt.Println(names) +func (t SectorFileType) Strings() []string { + var out []string + for _, fileType := range PathTypes { + if fileType&t == 0 { + continue + } + + out = append(out, fileType.String()) + } + return out +} + +// AllSet returns a slice of SectorFileType values that are set in the SectorFileType receiver value +func (t SectorFileType) AllSet() []SectorFileType { + var out []SectorFileType + for _, fileType := range PathTypes { + if fileType&t == 0 { + continue + } + + out = append(out, fileType) + } + return out +} + +// Has checks if the SectorFileType has a specific singleType. +func (t SectorFileType) Has(singleType SectorFileType) bool { + return t&singleType == singleType +} + +// SealSpaceUse calculates the amount of space needed for sealing the sector +// based on the given sector size. It iterates over the different path types +// and calculates the space needed for each path type using the FSOverheadSeal +// map. The overhead value is multiplied by the sector size and divided by the +// FSOverheadDen constant. The total space needed is accumulated and returned. +// If there is no seal overhead information for a particular path type, an error +// is returned. +// +// Example usage: +// +// fileType := FTSealed | FTCache +// sectorSize := abi.SectorSize(32 << 20) // 32 MiB +// spaceNeeded, err := fileType.SealSpaceUse(sectorSize) +// +// Parameters: +// +// ssize: The size of the sector +// +// Returns: +// +// uint64: The amount of space needed for sealing the sector +// error: If there is no seal overhead information for a path type +func (t SectorFileType) SealSpaceUse(ssize abi.SectorSize) (uint64, error) { + var need uint64 + for _, pathType := range PathTypes { + if !t.Has(pathType) { + continue + } + + oh, ok := FSOverheadSeal[pathType] + if !ok { + return 0, xerrors.Errorf("no seal overhead info for %s", pathType) + } + + need += uint64(oh) * uint64(ssize) / FSOverheadDen + } + + return need, nil +} + +// SubAllowed takes in two parameters: allowTypes and denyTypes, both of which are slices of strings. +// If allowTypes is not empty, the method sets a denyMask with all bits set to 1, and then iterates over each allowType, +// converting it to a SectorFileType using the TypeFromString function and unsetting the corresponding bit in the denyMask. +// If a string in allowTypes cannot be converted to a valid SectorFileType, it is ignored. +// After processing allowTypes, the method iterates over each denyType, converting it to a SectorFileType using the TypeFromString function +// and setting the corresponding bit in the denyMask. +// If a string in denyTypes cannot be converted to a valid SectorFileType, it is ignored. +// Finally, the method returns the bitwise AND of the original SectorFileType and the denyMask. +// The returned SectorFileType will only allow the types specified in allowTypes and exclude the types specified in denyTypes.` +func (t SectorFileType) SubAllowed(allowTypes []string, denyTypes []string) SectorFileType { + var denyMask SectorFileType // 1s deny + + if len(allowTypes) > 0 { + denyMask = ^denyMask + + for _, allowType := range allowTypes { + pt, err := TypeFromString(allowType) + if err != nil { + // we've told the user about this already, don't spam logs and ignore + continue + } + + denyMask = denyMask & (^pt) // unset allowed types + } + } + + for _, denyType := range denyTypes { + pt, err := TypeFromString(denyType) + if err != nil { + // we've told the user about this already, don't spam logs and ignore + continue + } + denyMask |= pt + } + + return t & denyMask +} + +// Unset removes the specified sector file type(s) from the current SectorFileType value. +// It performs a bitwise AND operation between the current value and the bitwise complement of the toUnset value. +// The result is returned as a new SectorFileType value. +// Any bits that are set in toUnset will be cleared in the result. +// Usage: result = value.Unset(typesToUnset) +func (t SectorFileType) Unset(toUnset SectorFileType) SectorFileType { + return t &^ toUnset +} + +// AnyAllowed checks if the SectorFileType has any allowed types and no denied types. +func (t SectorFileType) AnyAllowed(allowTypes []string, denyTypes []string) bool { + return t.SubAllowed(allowTypes, denyTypes) != t +} + +// Allowed checks if the SectorFileType is allowed based on the given allowTypes and denyTypes. +// Returns true if the SectorFileType is allowed, otherwise false. +func (t SectorFileType) Allowed(allowTypes []string, denyTypes []string) bool { + return t.SubAllowed(allowTypes, denyTypes) == 0 +} + +// StoreSpaceUse calculates the space used for storing sectors of a specific file type. +// It takes the sector size as input and returns the total space needed in bytes and an error, if any. +// The calculation is based on the finalized overhead information for the file type. +// If the overhead information is not available for a particular file type, an error will be returned. +func (t SectorFileType) StoreSpaceUse(ssize abi.SectorSize) (uint64, error) { + var need uint64 + for _, pathType := range PathTypes { + if !t.Has(pathType) { + continue + } + + oh, ok := FsOverheadFinalized[pathType] + if !ok { + return 0, xerrors.Errorf("no finalized overhead info for %s", pathType) + } + + need += uint64(oh) * uint64(ssize) / FSOverheadDen + } + + return need, nil +} + +// All returns an array indicating whether each FileTypes flag is set in the SectorFileType. +func (t SectorFileType) All() [FileTypes]bool { + var out [FileTypes]bool + + for i := range out { + out[i] = t&(1< 0 + } + + return out +} + +// IsNone checks if the SectorFileType value is equal to zero. +// It returns true if the value is zero, indicating that the type is none. +func (t SectorFileType) IsNone() bool { + return t == 0 +} + +// SectorPaths represents the paths for different sector files. +type SectorPaths struct { + ID abi.SectorID + + Unsealed string + Sealed string + Cache string + Update string + UpdateCache string + Piece string +} + +// HasAllSet checks if all paths of a SectorPaths struct are set for a given SectorFileType. +func (sp SectorPaths) HasAllSet(ft SectorFileType) bool { + for _, fileType := range ft.AllSet() { + if PathByType(sp, fileType) == "" { + return false + } + } + + return true +} + +// Subset returns a new instance of SectorPaths that contains only the paths specified by the filter SectorFileType. +// It iterates over each fileType in the filter, retrieves the corresponding path from the original SectorPaths instance, and sets it in the new instance. +// Finally, it sets the ID field of the new instance to be the same as the original instance. +func (sp SectorPaths) Subset(filter SectorFileType) SectorPaths { + var out SectorPaths + + for _, fileType := range filter.AllSet() { + SetPathByType(&out, fileType, PathByType(sp, fileType)) + } + + out.ID = sp.ID + + return out +} + +// ParseSectorID parses a sector ID from a given base name. +// It expects the format "s-t0%d-%d", where the first %d represents the miner ID +// and the second %d represents the sector number. +// +// Parameters: +// - baseName: The base name from which to parse the sector ID. +// +// Returns: +// - abi.SectorID: The parsed sector ID. +// - error: An error if parsing fails. +// +// Example usage: +// +// id, err := ParseSectorID(baseName) +// if err != nil { +// // handle error +// } +// // use id +func ParseSectorID(baseName string) (abi.SectorID, error) { + var n abi.SectorNumber + var mid abi.ActorID + read, err := fmt.Sscanf(baseName, "s-t0%d-%d", &mid, &n) + if err != nil { + return abi.SectorID{}, xerrors.Errorf("sscanf sector name ('%s'): %w", baseName, err) + } + + if read != 2 { + return abi.SectorID{}, xerrors.Errorf("parseSectorID expected to scan 2 values, got %d", read) + } + + return abi.SectorID{ + Miner: mid, + Number: n, + }, nil +} + +// SectorName returns the name of a sector in the format "s-t0-" +// +// Parameters: +// - sid: The sector ID +// +// Returns: +// - The name of the sector as a string +func SectorName(sid abi.SectorID) string { + return fmt.Sprintf("s-t0%d-%d", sid.Miner, sid.Number) +} + +// PathByType returns the path associated with the specified fileType in the given SectorPaths. +// It panics if the requested path type is unknown. +func PathByType(sps SectorPaths, fileType SectorFileType) string { + switch fileType { + case FTUnsealed: + return sps.Unsealed + case FTSealed: + return sps.Sealed + case FTCache: + return sps.Cache + case FTUpdate: + return sps.Update + case FTUpdateCache: + return sps.UpdateCache + case FTPiece: + return sps.Piece + } + + panic("requested unknown path type") +} + +func SetPathByType(sps *SectorPaths, fileType SectorFileType, p string) { + switch fileType { + case FTUnsealed: + sps.Unsealed = p + case FTSealed: + sps.Sealed = p + case FTCache: + sps.Cache = p + case FTUpdate: + sps.Update = p + case FTUpdateCache: + sps.UpdateCache = p + case FTPiece: + sps.Piece = p + } +} + +// PathsWithIDs represents paths and IDs for sector files. +type PathsWithIDs struct { + Paths SectorPaths + IDs SectorPaths +} + +// HasAllSet checks if all paths and IDs in PathsWithIDs have a corresponding path set for the specified SectorFileType. +// It returns true if all paths and IDs are set, and false otherwise. +func (p PathsWithIDs) HasAllSet(ft SectorFileType) bool { + return p.Paths.HasAllSet(ft) && p.IDs.HasAllSet(ft) +} diff --git a/lib/storiface/index.go b/lib/storiface/index.go new file mode 100644 index 000000000..3bf695687 --- /dev/null +++ b/lib/storiface/index.go @@ -0,0 +1,134 @@ +package storiface + +import ( + "strings" + + "github.com/filecoin-project/go-state-types/abi" + + "github.com/filecoin-project/lotus/storage/sealer/fsutil" +) + +// ID identifies sector storage by UUID. One sector storage should map to one +// +// filesystem, local or networked / shared by multiple machines +type ID string + +const IDSep = "." + +type IDList []ID + +func (il IDList) String() string { + l := make([]string, len(il)) + for i, id := range il { + l[i] = string(id) + } + return strings.Join(l, IDSep) +} + +func ParseIDList(s string) IDList { + strs := strings.Split(s, IDSep) + out := make([]ID, len(strs)) + for i, str := range strs { + out[i] = ID(str) + } + return out +} + +type Group = string + +type StorageInfo struct { + // ID is the UUID of the storage path + ID ID + + // URLs for remote access + URLs []string // TODO: Support non-http transports + + // Storage path weight; higher number means that the path will be preferred more often + Weight uint64 + + // MaxStorage is the number of bytes allowed to be used by files in the + // storage path + MaxStorage uint64 + + // CanStore is true when the path is allowed to be used for io-intensive + // sealing operations + CanSeal bool + + // CanStore is true when the path is allowed to be used for long-term storage + CanStore bool + + // Groups is the list of path groups this path belongs to + Groups []Group + + // AllowTo is the list of paths to which data from this path can be moved to + AllowTo []Group + + // AllowTypes lists sector file types which are allowed to be put into this + // path. If empty, all file types are allowed. + // + // Valid values: + // - "unsealed" + // - "sealed" + // - "cache" + // - "update" + // - "update-cache" + // Any other value will generate a warning and be ignored. + AllowTypes []string + + // DenyTypes lists sector file types which aren't allowed to be put into this + // path. + // + // Valid values: + // - "unsealed" + // - "sealed" + // - "cache" + // - "update" + // - "update-cache" + // Any other value will generate a warning and be ignored. + DenyTypes []string + + // AllowMiners lists miner IDs which are allowed to store their sector data into + // this path. If empty, all miner IDs are allowed + AllowMiners []string + + // DenyMiners lists miner IDs which are denied to store their sector data into + // this path + DenyMiners []string +} + +type HealthReport struct { + Stat fsutil.FsStat + Err string +} + +type SectorStorageInfo struct { + ID ID + URLs []string // TODO: Support non-http transports + BaseURLs []string + Weight uint64 + + CanSeal bool + CanStore bool + + Primary bool + + AllowTypes []string + DenyTypes []string + AllowMiners []string + DenyMiners []string +} + +type Decl struct { + abi.SectorID + SectorFileType +} + +type StoragePath struct { + ID ID + Weight uint64 + + LocalPath string + + CanSeal bool + CanStore bool +} diff --git a/lib/storiface/paths.go b/lib/storiface/paths.go new file mode 100644 index 000000000..0f0eaeadf --- /dev/null +++ b/lib/storiface/paths.go @@ -0,0 +1,39 @@ +package storiface + +import "github.com/filecoin-project/go-state-types/abi" + +type PathType string + +const ( + PathStorage PathType = "storage" + PathSealing PathType = "sealing" +) + +type AcquireMode string + +const ( + AcquireMove AcquireMode = "move" + AcquireCopy AcquireMode = "copy" +) + +type SectorLock struct { + Sector abi.SectorID + Write [FileTypes]uint + Read [FileTypes]uint +} + +type SectorLocks struct { + Locks []SectorLock +} + +type AcquireSettings struct { + Into *PathsWithIDs +} + +type AcquireOption func(*AcquireSettings) + +func AcquireInto(pathIDs PathsWithIDs) AcquireOption { + return func(settings *AcquireSettings) { + settings.Into = &pathIDs + } +} diff --git a/lib/storiface/storage.go b/lib/storiface/storage.go new file mode 100644 index 000000000..31afbae24 --- /dev/null +++ b/lib/storiface/storage.go @@ -0,0 +1,154 @@ +package storiface + +import ( + "context" + "io" + "net/http" + + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/proof" +) + +type Data = io.Reader + +type SectorRef struct { + ID abi.SectorID + ProofType abi.RegisteredSealProof +} + +// PieceNumber is a reference to a piece in the storage system; mapping between +// pieces in the storage system and piece CIDs is maintained by the storage index +type PieceNumber uint64 + +func (pn PieceNumber) Ref() SectorRef { + return SectorRef{ + ID: abi.SectorID{Miner: 0, Number: abi.SectorNumber(pn)}, + ProofType: abi.RegisteredSealProof_StackedDrg64GiBV1, // This only cares about TreeD which is the same for all sizes + } +} + +type PreCommit1Out []byte + +type SectorCids struct { + Unsealed cid.Cid + Sealed cid.Cid +} + +type ReplicaUpdateProof []byte + +type Verifier interface { + VerifySeal(proof.SealVerifyInfo) (bool, error) + VerifyAggregateSeals(aggregate proof.AggregateSealVerifyProofAndInfos) (bool, error) + VerifyReplicaUpdate(update proof.ReplicaUpdateInfo) (bool, error) + VerifyWinningPoSt(ctx context.Context, info proof.WinningPoStVerifyInfo) (bool, error) + VerifyWindowPoSt(ctx context.Context, info proof.WindowPoStVerifyInfo) (bool, error) + + GenerateWinningPoStSectorChallenge(context.Context, abi.RegisteredPoStProof, abi.ActorID, abi.PoStRandomness, uint64) ([]uint64, error) +} + +// Prover contains cheap proving-related methods +type Prover interface { + // TODO: move GenerateWinningPoStSectorChallenge from the Verifier interface to here + + AggregateSealProofs(aggregateInfo proof.AggregateSealVerifyProofAndInfos, proofs [][]byte) ([]byte, error) +} + +type SectorLocation struct { + // Local when set to true indicates to lotus that sector data is already + // available locally; When set lotus will skip fetching sector data, and + // only check that sector data exists in sector storage + Local bool + + // URL to the sector data + // For sealed/unsealed sector, lotus expects octet-stream + // For cache, lotus expects a tar archive with cache files + // Valid schemas: + // - http:// / https:// + URL string + + // optional http headers to use when requesting sector data + Headers []SecDataHttpHeader +} + +func (sd *SectorLocation) HttpHeaders() http.Header { + out := http.Header{} + for _, header := range sd.Headers { + out[header.Key] = append(out[header.Key], header.Value) + } + return out +} + +// note: we can't use http.Header as that's backed by a go map, which is all kinds of messy + +type SecDataHttpHeader struct { + Key string + Value string +} + +// StorageConfig .lotusstorage/storage.json +type StorageConfig struct { + StoragePaths []LocalPath +} + +type LocalPath struct { + Path string +} + +// LocalStorageMeta [path]/sectorstore.json +type LocalStorageMeta struct { + ID ID + + // A high weight means data is more likely to be stored in this path + Weight uint64 // 0 = readonly + + // Intermediate data for the sealing process will be stored here + CanSeal bool + + // Finalized sectors that will be proved over time will be stored here + CanStore bool + + // MaxStorage specifies the maximum number of bytes to use for sector storage + // (0 = unlimited) + MaxStorage uint64 + + // List of storage groups this path belongs to + Groups []string + + // List of storage groups to which data from this path can be moved. If none + // are specified, allow to all + AllowTo []string + + // AllowTypes lists sector file types which are allowed to be put into this + // path. If empty, all file types are allowed. + // + // Valid values: + // - "unsealed" + // - "sealed" + // - "cache" + // - "update" + // - "update-cache" + // Any other value will generate a warning and be ignored. + AllowTypes []string + + // DenyTypes lists sector file types which aren't allowed to be put into this + // path. + // + // Valid values: + // - "unsealed" + // - "sealed" + // - "cache" + // - "update" + // - "update-cache" + // Any other value will generate a warning and be ignored. + DenyTypes []string + + // AllowMiners lists miner IDs which are allowed to store their sector data into + // this path. If empty, all miner IDs are allowed + AllowMiners []string + + // DenyMiners lists miner IDs which are denied to store their sector data into + // this path + DenyMiners []string +} diff --git a/lib/storiface/worker.go b/lib/storiface/worker.go new file mode 100644 index 000000000..a6922f5cc --- /dev/null +++ b/lib/storiface/worker.go @@ -0,0 +1,81 @@ +package storiface + +import ( + "errors" + "fmt" + + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/proof" +) + +type WindowPoStResult struct { + PoStProofs proof.PoStProof + Skipped []abi.SectorID +} + +type PostSectorChallenge struct { + SealProof abi.RegisteredSealProof + SectorNumber abi.SectorNumber + SealedCID cid.Cid + Challenge []uint64 + Update bool +} + +type FallbackChallenges struct { + Sectors []abi.SectorNumber + Challenges map[abi.SectorNumber][]uint64 +} + +type ErrorCode int + +const ( + ErrUnknown ErrorCode = iota +) + +const ( + // Temp Errors + ErrTempUnknown ErrorCode = iota + 100 + ErrTempWorkerRestart + ErrTempAllocateSpace +) + +type WorkError interface { + ErrCode() ErrorCode +} + +type CallError struct { + Code ErrorCode + Message string + sub error +} + +func (c *CallError) ErrCode() ErrorCode { + return c.Code +} + +func (c *CallError) Error() string { + return fmt.Sprintf("storage call error %d: %s", c.Code, c.Message) +} + +func (c *CallError) Unwrap() error { + if c.sub != nil { + return c.sub + } + + return errors.New(c.Message) +} + +var _ WorkError = &CallError{} + +func Err(code ErrorCode, sub error) *CallError { + return &CallError{ + Code: code, + Message: sub.Error(), + + sub: sub, + } +} + +type WorkerJob struct{} // dummy diff --git a/market/fakelm/iface.go b/market/fakelm/iface.go index 1bc91b35e..9e0b0c63f 100644 --- a/market/fakelm/iface.go +++ b/market/fakelm/iface.go @@ -8,8 +8,9 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/curio/lib/storiface" + "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/storage/sealer/storiface" ) // MinimalLMApi is a subset of the LotusMiner API that is exposed by Curio diff --git a/market/fakelm/lmimpl.go b/market/fakelm/lmimpl.go index 65a90d959..d8ae26920 100644 --- a/market/fakelm/lmimpl.go +++ b/market/fakelm/lmimpl.go @@ -24,6 +24,7 @@ import ( "github.com/filecoin-project/curio/deps/config" "github.com/filecoin-project/curio/harmony/harmonydb" "github.com/filecoin-project/curio/lib/paths" + storiface "github.com/filecoin-project/curio/lib/storiface" "github.com/filecoin-project/curio/market" lapi "github.com/filecoin-project/lotus/api" @@ -31,7 +32,6 @@ import ( "github.com/filecoin-project/lotus/chain/actors/builtin/miner" sealing "github.com/filecoin-project/lotus/storage/pipeline" lpiece "github.com/filecoin-project/lotus/storage/pipeline/piece" - "github.com/filecoin-project/lotus/storage/sealer/storiface" ) var log = logging.Logger("lmrpc") diff --git a/market/lmrpc/lmrpc.go b/market/lmrpc/lmrpc.go index c148baa3a..d69b5b4e1 100644 --- a/market/lmrpc/lmrpc.go +++ b/market/lmrpc/lmrpc.go @@ -2,12 +2,14 @@ package lmrpc import ( "context" + "encoding/json" "errors" "fmt" "io" "net" "net/http" "net/url" + "reflect" "sort" "strconv" "strings" @@ -27,6 +29,7 @@ import ( "github.com/filecoin-project/curio/deps/config" "github.com/filecoin-project/curio/harmony/harmonydb" "github.com/filecoin-project/curio/lib/paths" + "github.com/filecoin-project/curio/lib/storiface" cumarket "github.com/filecoin-project/curio/market" "github.com/filecoin-project/curio/market/fakelm" @@ -36,7 +39,6 @@ import ( "github.com/filecoin-project/lotus/lib/nullreader" "github.com/filecoin-project/lotus/metrics/proxy" lpiece "github.com/filecoin-project/lotus/storage/pipeline/piece" - "github.com/filecoin-project/lotus/storage/sealer/storiface" ) var log = logging.Logger("lmrpc") @@ -207,25 +209,25 @@ func ServeCurioMarketRPC(db *harmonydb.DB, full api.Chain, maddr address.Address ast.CommonStruct.Internal.AuthNew = lp.AuthNew ast.Internal.ActorAddress = lp.ActorAddress - ast.Internal.WorkerJobs = lp.WorkerJobs + adaptFunc(&ast.Internal.WorkerJobs, lp.WorkerJobs) ast.Internal.SectorsStatus = lp.SectorsStatus ast.Internal.SectorsList = lp.SectorsList ast.Internal.SectorsSummary = lp.SectorsSummary ast.Internal.SectorsListInStates = lp.SectorsListInStates - ast.Internal.StorageRedeclareLocal = lp.StorageRedeclareLocal - ast.Internal.ComputeDataCid = lp.ComputeDataCid + adaptFunc(&ast.Internal.StorageRedeclareLocal, lp.StorageRedeclareLocal) + adaptFunc(&ast.Internal.ComputeDataCid, lp.ComputeDataCid) ast.Internal.SectorAddPieceToAny = sectorAddPieceToAnyOperation(maddr, rootUrl, conf, pieceInfoLk, pieceInfos, pin, db, mi.SectorSize) - ast.Internal.StorageList = si.StorageList - ast.Internal.StorageDetach = si.StorageDetach - ast.Internal.StorageReportHealth = si.StorageReportHealth - ast.Internal.StorageDeclareSector = si.StorageDeclareSector - ast.Internal.StorageDropSector = si.StorageDropSector - ast.Internal.StorageFindSector = si.StorageFindSector - ast.Internal.StorageInfo = si.StorageInfo - ast.Internal.StorageBestAlloc = si.StorageBestAlloc - ast.Internal.StorageLock = si.StorageLock - ast.Internal.StorageTryLock = si.StorageTryLock - ast.Internal.StorageGetLocks = si.StorageGetLocks + adaptFunc(&ast.Internal.StorageList, si.StorageList) + adaptFunc(&ast.Internal.StorageDetach, si.StorageDetach) + adaptFunc(&ast.Internal.StorageReportHealth, si.StorageReportHealth) + adaptFunc(&ast.Internal.StorageDeclareSector, si.StorageDeclareSector) + adaptFunc(&ast.Internal.StorageDropSector, si.StorageDropSector) + adaptFunc(&ast.Internal.StorageFindSector, si.StorageFindSector) + adaptFunc(&ast.Internal.StorageInfo, si.StorageInfo) + adaptFunc(&ast.Internal.StorageBestAlloc, si.StorageBestAlloc) + adaptFunc(&ast.Internal.StorageLock, si.StorageLock) + adaptFunc(&ast.Internal.StorageTryLock, si.StorageTryLock) + adaptFunc(&ast.Internal.StorageGetLocks, si.StorageGetLocks) ast.Internal.SectorStartSealing = pin.SectorStartSealing var pieceHandler http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) { @@ -314,6 +316,55 @@ type pieceInfo struct { done chan struct{} } +// A util to convert jsonrpc methods using incompatible Go types with same jsonrpc representation +// Won't be needed in a post-boost world +func adaptFunc(outFnPtr, inFun any) { + outFnValue := reflect.ValueOf(outFnPtr).Elem() + inFnValue := reflect.ValueOf(inFun) + + adaptedFn := reflect.MakeFunc(outFnValue.Type(), func(args []reflect.Value) []reflect.Value { + inArgs := make([]reflect.Value, inFnValue.Type().NumIn()) + for i := 0; i < len(inArgs); i++ { + if i < len(args) { + if args[i].Type() == inFnValue.Type().In(i) { + inArgs[i] = args[i] + } else { + // Convert using JSON + jsonData, _ := json.Marshal(args[i].Interface()) + newArg := reflect.New(inFnValue.Type().In(i)).Interface() + _ = json.Unmarshal(jsonData, newArg) + inArgs[i] = reflect.ValueOf(newArg).Elem() + } + } else { + inArgs[i] = reflect.Zero(inFnValue.Type().In(i)) + } + } + + results := inFnValue.Call(inArgs) + outResults := make([]reflect.Value, outFnValue.Type().NumOut()) + + for i := 0; i < len(outResults); i++ { + if i < len(results) { + if results[i].Type() == outFnValue.Type().Out(i) { + outResults[i] = results[i] + } else { + // Convert using JSON + jsonData, _ := json.Marshal(results[i].Interface()) + newResult := reflect.New(outFnValue.Type().Out(i)).Interface() + _ = json.Unmarshal(jsonData, newResult) + outResults[i] = reflect.ValueOf(newResult).Elem() + } + } else { + outResults[i] = reflect.Zero(outFnValue.Type().Out(i)) + } + } + + return outResults + }) + + outFnValue.Set(adaptedFn) +} + type PieceIngester interface { AllocatePieceToSector(ctx context.Context, maddr address.Address, piece lpiece.PieceDealInfo, rawSize int64, source url.URL, header http.Header) (lapi.SectorOffset, error) } diff --git a/scripts/docgen/docgen.go b/scripts/docgen/docgen.go index 73ad48c1c..8781c1466 100644 --- a/scripts/docgen/docgen.go +++ b/scripts/docgen/docgen.go @@ -9,10 +9,8 @@ import ( "path/filepath" "reflect" "strings" - "time" "unicode" - "github.com/google/uuid" "github.com/ipfs/go-cid" "github.com/filecoin-project/go-address" @@ -21,11 +19,10 @@ import ( "github.com/filecoin-project/go-state-types/builtin/v13/verifreg" "github.com/filecoin-project/curio/api" + storiface "github.com/filecoin-project/curio/lib/storiface" lapi "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/storage/sealer/sealtasks" - "github.com/filecoin-project/lotus/storage/sealer/storiface" ) var ExampleValues = map[reflect.Type]interface{}{ @@ -213,21 +210,6 @@ func init() { addExample(map[storiface.ID]string{ "76f1988b-ef30-4d7e-b3ec-9a627f4ba5a8": "/data/path", }) - addExample(map[uuid.UUID][]storiface.WorkerJob{ - uuid.MustParse("ef8d99a2-6865-4189-8ffa-9fef0f806eee"): { - { - ID: storiface.CallID{ - Sector: abi.SectorID{Miner: 1000, Number: 100}, - ID: uuid.MustParse("76081ba0-61bd-45a5-bc08-af05f1c26e5d"), - }, - Sector: abi.SectorID{Miner: 1000, Number: 100}, - Task: sealtasks.TTPreCommit2, - RunWait: 0, - Start: time.Unix(1605172927, 0).UTC(), - Hostname: "host", - }, - }, - }) //addExample(map[uuid.UUID]storiface.WorkerStats{ // uuid.MustParse("ef8d99a2-6865-4189-8ffa-9fef0f806eee"): { // Info: storiface.WorkerInfo{ diff --git a/tasks/gc/storage_endpoint_gc.go b/tasks/gc/storage_endpoint_gc.go index f923bb541..f6bb00b1d 100644 --- a/tasks/gc/storage_endpoint_gc.go +++ b/tasks/gc/storage_endpoint_gc.go @@ -14,10 +14,10 @@ import ( "github.com/filecoin-project/curio/harmony/harmonytask" "github.com/filecoin-project/curio/harmony/resources" "github.com/filecoin-project/curio/lib/paths" + "github.com/filecoin-project/curio/lib/storiface" "github.com/filecoin-project/lotus/lib/result" "github.com/filecoin-project/lotus/storage/sealer/fsutil" - "github.com/filecoin-project/lotus/storage/sealer/storiface" ) var log = logging.Logger("curiogc") diff --git a/tasks/gc/storage_gc_mark.go b/tasks/gc/storage_gc_mark.go index 91cecf0ed..9a184bb84 100644 --- a/tasks/gc/storage_gc_mark.go +++ b/tasks/gc/storage_gc_mark.go @@ -16,11 +16,11 @@ import ( "github.com/filecoin-project/curio/harmony/resources" "github.com/filecoin-project/curio/lib/curiochain" "github.com/filecoin-project/curio/lib/paths" + "github.com/filecoin-project/curio/lib/storiface" "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/storage/sealer/storiface" ) const StorageGCInterval = 9 * time.Minute diff --git a/tasks/gc/storage_gc_sweep.go b/tasks/gc/storage_gc_sweep.go index c7b0230a0..1666fdf21 100644 --- a/tasks/gc/storage_gc_sweep.go +++ b/tasks/gc/storage_gc_sweep.go @@ -13,8 +13,7 @@ import ( "github.com/filecoin-project/curio/harmony/harmonytask" "github.com/filecoin-project/curio/harmony/resources" "github.com/filecoin-project/curio/lib/paths" - - "github.com/filecoin-project/lotus/storage/sealer/storiface" + storiface "github.com/filecoin-project/curio/lib/storiface" ) type StorageGCSweep struct { diff --git a/tasks/piece/task_cleanup_piece.go b/tasks/piece/task_cleanup_piece.go index c650b4a0c..205221ee8 100644 --- a/tasks/piece/task_cleanup_piece.go +++ b/tasks/piece/task_cleanup_piece.go @@ -11,8 +11,7 @@ import ( "github.com/filecoin-project/curio/harmony/resources" "github.com/filecoin-project/curio/lib/ffi" "github.com/filecoin-project/curio/lib/promise" - - "github.com/filecoin-project/lotus/storage/sealer/storiface" + "github.com/filecoin-project/curio/lib/storiface" ) type CleanupPieceTask struct { diff --git a/tasks/piece/task_park_piece.go b/tasks/piece/task_park_piece.go index 4775c670d..b4fa5ba6f 100644 --- a/tasks/piece/task_park_piece.go +++ b/tasks/piece/task_park_piece.go @@ -17,8 +17,7 @@ import ( ffi2 "github.com/filecoin-project/curio/lib/ffi" "github.com/filecoin-project/curio/lib/paths" "github.com/filecoin-project/curio/lib/promise" - - "github.com/filecoin-project/lotus/storage/sealer/storiface" + storiface "github.com/filecoin-project/curio/lib/storiface" ) var log = logging.Logger("cu-piece") diff --git a/tasks/seal/task_finalize.go b/tasks/seal/task_finalize.go index 72cd3c71b..15277c75a 100644 --- a/tasks/seal/task_finalize.go +++ b/tasks/seal/task_finalize.go @@ -12,8 +12,7 @@ import ( "github.com/filecoin-project/curio/harmony/resources" "github.com/filecoin-project/curio/lib/ffi" "github.com/filecoin-project/curio/lib/slotmgr" - - "github.com/filecoin-project/lotus/storage/sealer/storiface" + storiface "github.com/filecoin-project/curio/lib/storiface" ) type FinalizeTask struct { diff --git a/tasks/seal/task_movestorage.go b/tasks/seal/task_movestorage.go index 63add8184..c83e1d167 100644 --- a/tasks/seal/task_movestorage.go +++ b/tasks/seal/task_movestorage.go @@ -12,8 +12,7 @@ import ( "github.com/filecoin-project/curio/harmony/resources" ffi2 "github.com/filecoin-project/curio/lib/ffi" "github.com/filecoin-project/curio/lib/paths" - - "github.com/filecoin-project/lotus/storage/sealer/storiface" + storiface "github.com/filecoin-project/curio/lib/storiface" ) type MoveStorageTask struct { diff --git a/tasks/seal/task_porep.go b/tasks/seal/task_porep.go index 7326c8236..0a2f352da 100644 --- a/tasks/seal/task_porep.go +++ b/tasks/seal/task_porep.go @@ -15,9 +15,9 @@ import ( "github.com/filecoin-project/curio/harmony/harmonytask" "github.com/filecoin-project/curio/harmony/resources" "github.com/filecoin-project/curio/lib/ffi" + "github.com/filecoin-project/curio/lib/storiface" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/storage/sealer/storiface" ) type PoRepAPI interface { diff --git a/tasks/seal/task_sdr.go b/tasks/seal/task_sdr.go index c088bccb8..f990f1b8c 100644 --- a/tasks/seal/task_sdr.go +++ b/tasks/seal/task_sdr.go @@ -17,10 +17,10 @@ import ( "github.com/filecoin-project/curio/lib/dealdata" ffi2 "github.com/filecoin-project/curio/lib/ffi" "github.com/filecoin-project/curio/lib/paths" + storiface "github.com/filecoin-project/curio/lib/storiface" "github.com/filecoin-project/lotus/chain/actors/policy" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/storage/sealer/storiface" ) var IsDevnet = build.BlockDelaySecs < 30 diff --git a/tasks/seal/task_synth_proofs.go b/tasks/seal/task_synth_proofs.go index 0eceafae9..70f2d3416 100644 --- a/tasks/seal/task_synth_proofs.go +++ b/tasks/seal/task_synth_proofs.go @@ -15,8 +15,7 @@ import ( "github.com/filecoin-project/curio/lib/dealdata" "github.com/filecoin-project/curio/lib/ffi" "github.com/filecoin-project/curio/lib/paths" - - "github.com/filecoin-project/lotus/storage/sealer/storiface" + storiface "github.com/filecoin-project/curio/lib/storiface" ) type SyntheticProofTask struct { diff --git a/tasks/seal/task_treed.go b/tasks/seal/task_treed.go index dec8bf3c5..19dd598e2 100644 --- a/tasks/seal/task_treed.go +++ b/tasks/seal/task_treed.go @@ -12,8 +12,7 @@ import ( "github.com/filecoin-project/curio/harmony/resources" "github.com/filecoin-project/curio/lib/dealdata" ffi2 "github.com/filecoin-project/curio/lib/ffi" - - "github.com/filecoin-project/lotus/storage/sealer/storiface" + storiface "github.com/filecoin-project/curio/lib/storiface" ) type TreeDTask struct { diff --git a/tasks/seal/task_treerc.go b/tasks/seal/task_treerc.go index 93ec7211e..120441ef7 100644 --- a/tasks/seal/task_treerc.go +++ b/tasks/seal/task_treerc.go @@ -14,8 +14,7 @@ import ( "github.com/filecoin-project/curio/lib/dealdata" ffi2 "github.com/filecoin-project/curio/lib/ffi" "github.com/filecoin-project/curio/lib/paths" - - "github.com/filecoin-project/lotus/storage/sealer/storiface" + storiface "github.com/filecoin-project/curio/lib/storiface" ) type TreeRCTask struct { diff --git a/tasks/sealsupra/task_supraseal.go b/tasks/sealsupra/task_supraseal.go index 9bd6997af..36dc235aa 100644 --- a/tasks/sealsupra/task_supraseal.go +++ b/tasks/sealsupra/task_supraseal.go @@ -26,11 +26,11 @@ import ( "github.com/filecoin-project/curio/lib/passcall" "github.com/filecoin-project/curio/lib/paths" "github.com/filecoin-project/curio/lib/slotmgr" + storiface "github.com/filecoin-project/curio/lib/storiface" "github.com/filecoin-project/curio/lib/supraffi" "github.com/filecoin-project/curio/tasks/seal" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/storage/sealer/storiface" ) const suprasealConfigEnv = "SUPRASEAL_CONFIG" diff --git a/tasks/snap/task_encode.go b/tasks/snap/task_encode.go index dcc4b6239..42ab18e25 100644 --- a/tasks/snap/task_encode.go +++ b/tasks/snap/task_encode.go @@ -16,9 +16,8 @@ import ( "github.com/filecoin-project/curio/lib/dealdata" "github.com/filecoin-project/curio/lib/ffi" "github.com/filecoin-project/curio/lib/passcall" + storiface "github.com/filecoin-project/curio/lib/storiface" "github.com/filecoin-project/curio/tasks/seal" - - "github.com/filecoin-project/lotus/storage/sealer/storiface" ) const MinSnapSchedInterval = 10 * time.Second diff --git a/tasks/snap/task_movestorage.go b/tasks/snap/task_movestorage.go index a269e365b..ddabc1cf0 100644 --- a/tasks/snap/task_movestorage.go +++ b/tasks/snap/task_movestorage.go @@ -14,9 +14,8 @@ import ( "github.com/filecoin-project/curio/lib/ffi" "github.com/filecoin-project/curio/lib/passcall" "github.com/filecoin-project/curio/lib/paths" + storiface "github.com/filecoin-project/curio/lib/storiface" "github.com/filecoin-project/curio/tasks/seal" - - "github.com/filecoin-project/lotus/storage/sealer/storiface" ) type MoveStorageTask struct { diff --git a/tasks/snap/task_prove.go b/tasks/snap/task_prove.go index 0fea1079a..90467ef13 100644 --- a/tasks/snap/task_prove.go +++ b/tasks/snap/task_prove.go @@ -14,9 +14,8 @@ import ( "github.com/filecoin-project/curio/harmony/resources" "github.com/filecoin-project/curio/lib/ffi" "github.com/filecoin-project/curio/lib/passcall" + "github.com/filecoin-project/curio/lib/storiface" "github.com/filecoin-project/curio/tasks/seal" - - "github.com/filecoin-project/lotus/storage/sealer/storiface" ) type ProveTask struct { diff --git a/tasks/window/compute_do.go b/tasks/window/compute_do.go index bf4ffe135..fae4f886a 100644 --- a/tasks/window/compute_do.go +++ b/tasks/window/compute_do.go @@ -25,11 +25,10 @@ import ( "github.com/filecoin-project/curio/build" "github.com/filecoin-project/curio/lib/ffiselect" + "github.com/filecoin-project/curio/lib/storiface" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/storage/sealer" - "github.com/filecoin-project/lotus/storage/sealer/storiface" ) const disablePreChecks = false // todo config @@ -261,7 +260,11 @@ type CheckSectorsAPI interface { StateMinerSectors(ctx context.Context, addr address.Address, bf *bitfield.BitField, tsk types.TipSetKey) ([]*miner.SectorOnChainInfo, error) } -func checkSectors(ctx context.Context, api CheckSectorsAPI, ft sealer.FaultTracker, +type FaultTracker interface { + CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storiface.SectorRef, rg storiface.RGetter) (map[abi.SectorID]string, error) +} + +func checkSectors(ctx context.Context, api CheckSectorsAPI, ft FaultTracker, maddr address.Address, check bitfield.BitField, tsk types.TipSetKey) (bitfield.BitField, error) { mid, err := address.IDFromAddress(maddr) if err != nil { diff --git a/tasks/window/compute_task.go b/tasks/window/compute_task.go index 6f375d2dd..2d7cf246a 100644 --- a/tasks/window/compute_task.go +++ b/tasks/window/compute_task.go @@ -25,14 +25,13 @@ import ( "github.com/filecoin-project/curio/lib/chainsched" "github.com/filecoin-project/curio/lib/paths" "github.com/filecoin-project/curio/lib/promise" + storiface "github.com/filecoin-project/curio/lib/storiface" + "github.com/filecoin-project/curio/tasks/seal" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/node/modules/dtypes" - "github.com/filecoin-project/lotus/storage/sealer" - "github.com/filecoin-project/lotus/storage/sealer/sealtasks" - "github.com/filecoin-project/lotus/storage/sealer/storiface" ) var log = logging.Logger("curio/window") @@ -64,7 +63,7 @@ type WdPostTask struct { api WDPoStAPI db *harmonydb.DB - faultTracker sealer.FaultTracker + faultTracker FaultTracker storage paths.Store verifier storiface.Verifier paramsReady func() (bool, error) @@ -86,7 +85,7 @@ type wdTaskIdentity struct { func NewWdPostTask(db *harmonydb.DB, api WDPoStAPI, - faultTracker sealer.FaultTracker, + faultTracker FaultTracker, storage paths.Store, verifier storiface.Verifier, paramck func() (bool, error), @@ -357,9 +356,14 @@ func (t *WdPostTask) CanAccept(ids []harmonytask.TaskID, te *harmonytask.TaskEng return &tasks[0].TaskID, nil } -var res = storiface.ResourceTable[sealtasks.TTGenerateWindowPoSt] - func (t *WdPostTask) TypeDetails() harmonytask.TaskTypeDetails { + gpu := 1.0 + ram := uint64(25 << 30) + if seal.IsDevnet { + gpu = 0 + ram = 1 << 30 + } + return harmonytask.TaskTypeDetails{ Name: "WdPost", Max: t.max, @@ -368,17 +372,10 @@ func (t *WdPostTask) TypeDetails() harmonytask.TaskTypeDetails { Cost: resources.Resources{ Cpu: 1, - // todo set to something for 32/64G sector sizes? Technically windowPoSt is happy on a CPU - // but it will use a GPU if available - Gpu: 0, + Gpu: gpu, // RAM of smallest proof's max is listed here - Ram: lo.Reduce(lo.Keys(res), func(i uint64, k abi.RegisteredSealProof, _ int) uint64 { - if res[k].MaxMemory < i { - return res[k].MaxMemory - } - return i - }, 1<<63), + Ram: ram, }, } } diff --git a/tasks/window/faults_simple.go b/tasks/window/faults_simple.go index bda1be236..9e38e5e5c 100644 --- a/tasks/window/faults_simple.go +++ b/tasks/window/faults_simple.go @@ -13,8 +13,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/curio/lib/paths" - - "github.com/filecoin-project/lotus/storage/sealer/storiface" + "github.com/filecoin-project/curio/lib/storiface" ) type SimpleFaultTracker struct { diff --git a/tasks/window/recover_task.go b/tasks/window/recover_task.go index 673e7a79f..722a08001 100644 --- a/tasks/window/recover_task.go +++ b/tasks/window/recover_task.go @@ -24,14 +24,13 @@ import ( "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/node/modules/dtypes" - "github.com/filecoin-project/lotus/storage/sealer" ) type WdPostRecoverDeclareTask struct { sender *message.Sender db *harmonydb.DB api WdPostRecoverDeclareTaskApi - faultTracker sealer.FaultTracker + faultTracker FaultTracker maxDeclareRecoveriesGasFee types.FIL as *multictladdr.MultiAddressSelector @@ -60,7 +59,7 @@ type WdPostRecoverDeclareTaskApi interface { func NewWdPostRecoverDeclareTask(sender *message.Sender, db *harmonydb.DB, api WdPostRecoverDeclareTaskApi, - faultTracker sealer.FaultTracker, + faultTracker FaultTracker, as *multictladdr.MultiAddressSelector, pcs *chainsched.CurioChainSched, diff --git a/tasks/winning/winning_task.go b/tasks/winning/winning_task.go index d738a9e67..c1c3bbe36 100644 --- a/tasks/winning/winning_task.go +++ b/tasks/winning/winning_task.go @@ -28,6 +28,7 @@ import ( "github.com/filecoin-project/curio/lib/ffiselect" "github.com/filecoin-project/curio/lib/paths" "github.com/filecoin-project/curio/lib/promise" + "github.com/filecoin-project/curio/lib/storiface" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors/policy" @@ -35,7 +36,6 @@ import ( lrand "github.com/filecoin-project/lotus/chain/rand" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/node/modules/dtypes" - "github.com/filecoin-project/lotus/storage/sealer/storiface" ) var log = logging.Logger("curio/winning") diff --git a/web/api/sector/sector.go b/web/api/sector/sector.go index 12a2a8a34..16771715d 100644 --- a/web/api/sector/sector.go +++ b/web/api/sector/sector.go @@ -22,6 +22,7 @@ import ( "github.com/filecoin-project/go-state-types/builtin/v9/market" "github.com/filecoin-project/curio/deps" + "github.com/filecoin-project/curio/lib/storiface" "github.com/filecoin-project/curio/web/api/apihelper" "github.com/filecoin-project/lotus/blockstore" @@ -29,7 +30,6 @@ import ( "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/cli/spcli" - "github.com/filecoin-project/lotus/storage/sealer/storiface" ) const verifiedPowerGainMul = 9 diff --git a/web/api/webrpc/sector.go b/web/api/webrpc/sector.go index 2c4e192db..ea2e70fba 100644 --- a/web/api/webrpc/sector.go +++ b/web/api/webrpc/sector.go @@ -13,9 +13,9 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/curio/lib/paths" + "github.com/filecoin-project/curio/lib/storiface" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/storage/sealer/storiface" ) type SectorInfo struct { diff --git a/web/api/webrpc/storage_stats.go b/web/api/webrpc/storage_stats.go index e87cbb863..18e20cd67 100644 --- a/web/api/webrpc/storage_stats.go +++ b/web/api/webrpc/storage_stats.go @@ -4,8 +4,9 @@ import ( "context" "time" + "github.com/filecoin-project/curio/lib/storiface" + "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/storage/sealer/storiface" ) type StorageGCStats struct {