diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eeab27d67..1d5b8c5e8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -181,6 +181,8 @@ jobs: target: "./itests/pdp_prove_test.go" - name: test-itest-pieces-unseal target: "./itests/pieces_unseal_test.go" + - name: test-itest-remoteseal + target: "./itests/remoteseal_test.go" - name: test-itest-retrievals-test target: "./itests/retrievals_test.go" - name: test-itest-sql-idempotent-test @@ -210,6 +212,7 @@ jobs: - name: Run tests with coverage env: + CURIO_OPTIMAL_LIBFILCRYPTO: 0 FFI_USE_OPENCL: 1 run: | mkdir -p coverage diff --git a/cmd/curio/tasks/tasks.go b/cmd/curio/tasks/tasks.go index ba113fb39..a9a30932f 100644 --- a/cmd/curio/tasks/tasks.go +++ b/cmd/curio/tasks/tasks.go @@ -39,6 +39,7 @@ import ( "github.com/filecoin-project/curio/lib/slotmgr" "github.com/filecoin-project/curio/lib/storiface" "github.com/filecoin-project/curio/market/libp2p" + "github.com/filecoin-project/curio/market/sealmarket" "github.com/filecoin-project/curio/tasks/balancemgr" "github.com/filecoin-project/curio/tasks/expmgr" "github.com/filecoin-project/curio/tasks/f3" @@ -51,6 +52,7 @@ import ( "github.com/filecoin-project/curio/tasks/pdpv0" piece2 "github.com/filecoin-project/curio/tasks/piece" "github.com/filecoin-project/curio/tasks/proofshare" + "github.com/filecoin-project/curio/tasks/remoteseal" "github.com/filecoin-project/curio/tasks/scrub" "github.com/filecoin-project/curio/tasks/seal" "github.com/filecoin-project/curio/tasks/sealsupra" @@ -234,7 +236,9 @@ func StartTasks(ctx context.Context, dependencies *deps.Deps, shutdownChan chan cfg.Subsystems.EnableUpdateSubmit || cfg.Subsystems.EnableCommP || cfg.Subsystems.EnableProofShare || - cfg.Subsystems.EnableRemoteProofs + cfg.Subsystems.EnableRemoteProofs || + cfg.Subsystems.EnableRemoteSealProvider || + cfg.Subsystems.EnableRemoteSealClient var p2Active sealsupra.P2Active if hasAnySealingTask { @@ -375,11 +379,15 @@ func StartTasks(ctx context.Context, dependencies *deps.Deps, shutdownChan chan pdpv0IpniTask := indexing.NewPDPV0IPNITask(db, cfg, idxMax, iStore) activeTasks = append(activeTasks, ipniTask, indexingTask, pdpv1IdxTask, pdpv1IPNITask, fixRawSizeTask, pdpv0IndexingTask, pdpv0IpniTask) + // Create SealMarket for remote seal HTTP API + if cfg.Subsystems.EnableRemoteSealProvider || cfg.Subsystems.EnableRemoteSealClient { + sdeps.SealMarket = sealmarket.NewSealMarket(db, stor, full) + } + if cfg.HTTP.Enable { - // TODO: Put this back once PDPv1 is also being used - //if !cfg.Subsystems.EnableDealMarket { - // return nil, xerrors.New("deal market must be enabled on HTTP server") - //} + if !cfg.Subsystems.EnableDealMarket && !cfg.Subsystems.EnableRemoteSealProvider && !cfg.Subsystems.EnableRemoteSealClient { + return nil, xerrors.New("deal market or remote seal must be enabled on HTTP server") + } err = cuhttp.StartHTTPServer(ctx, dependencies, &sdeps) if err != nil { return nil, xerrors.Errorf("failed to start the HTTP server: %w", err) @@ -450,6 +458,11 @@ func addSealingTasks( var slotMgr *slotmgr.SlotMgr var addFinalize bool + // Create the provider poller unconditionally so that sealing nodes (SDR/Batch) + // can register the RSealProviderFinalize task even without EnableRemoteSealProvider. + // RunPoller is started by the addFinalize block or the EnableRemoteSealProvider block. + provPoller := remoteseal.NewProviderPoller(db) + // NOTE: Tasks with the LEAST priority are at the top if cfg.Subsystems.EnableCommP { scrubUnsealedTask := scrub.NewCommDCheckTask(db, slr) @@ -477,21 +490,48 @@ func addSealingTasks( if cfg.Subsystems.EnableSealSDR { sdrMax := taskhelp.Max(cfg.Subsystems.SealSDRMaxTasks) - sdrTask := seal.NewSDRTask(full, db, sp, slr, sdrMax, cfg.Subsystems.SealSDRMinTasks) + // provPoller is passed so SDR registers its AddTaskFunc with both SealPoller + // and RSealProviderPoller. When EnableRemoteSealProvider is off, only the + // local SealPoller is used for scheduling SDR tasks. + var sdrProvPoller seal.ProviderPollerSDR + if cfg.Subsystems.EnableRemoteSealProvider { + sdrProvPoller = provPoller + } + sdrTask := seal.NewSDRTask(full, db, sp, slr, sdrMax, cfg.Subsystems.SealSDRMinTasks, sdrProvPoller) keyTask := unseal.NewTaskUnsealSDR(slr, db, sdrMax, full) activeTasks = append(activeTasks, sdrTask, keyTask) } if cfg.Subsystems.EnableSealSDRTrees { - treeDTask := seal.NewTreeDTask(sp, db, slr, cfg.Subsystems.SealSDRTreesMaxTasks, cfg.Subsystems.BindSDRTreeToNode) - treeRCTask := seal.NewTreeRCTask(sp, db, slr, cfg.Subsystems.SealSDRTreesMaxTasks) + var treeDProvPoller seal.ProviderPollerTreeD + var treeRCProvPoller seal.ProviderPollerTreeRC + if cfg.Subsystems.EnableRemoteSealProvider { + treeDProvPoller = provPoller + treeRCProvPoller = provPoller + } + treeDTask := seal.NewTreeDTask(sp, db, slr, cfg.Subsystems.SealSDRTreesMaxTasks, cfg.Subsystems.BindSDRTreeToNode, treeDProvPoller) + treeRCTask := seal.NewTreeRCTask(sp, db, slr, cfg.Subsystems.SealSDRTreesMaxTasks, treeRCProvPoller) synthTask := seal.NewSyntheticProofTask(sp, db, slr, cfg.Subsystems.SyntheticPoRepMaxTasks) activeTasks = append(activeTasks, treeDTask, synthTask, treeRCTask) addFinalize = true } + // Remote seal client needs the Finalize task to run after PoRep. + // The client skips SDR/Trees (done by provider) but still runs the + // standard pipeline from precommit onward, which requires Finalize. + if cfg.Subsystems.EnableRemoteSealClient { + addFinalize = true + } if addFinalize { finalizeTask := seal.NewFinalizeTask(cfg.Subsystems.FinalizeMaxTasks, sp, slr, db, slotMgr) - activeTasks = append(activeTasks, finalizeTask) + // RSealProviderFinalize must run on sealing nodes (SDR/Batch) that hold + // the sector data and batch slots, not only on EnableRemoteSealProvider nodes. + provFinalizeTask := remoteseal.NewProviderFinalizeTask(db, provPoller, slr, slotMgr, cfg.Subsystems.FinalizeMaxTasks) + activeTasks = append(activeTasks, finalizeTask, provFinalizeTask) + + // Start the provider poller so it can schedule RSealProviderFinalize tasks + // on sealing nodes. The poller safely skips task types whose AddTaskFunc + // hasn't been registered (e.g. Notify, Cleanup on non-provider nodes). + go provPoller.RunPoller(ctx) } if cfg.Subsystems.EnableSendPrecommitMsg { @@ -564,6 +604,48 @@ func addSealingTasks( activeTasks = append(activeTasks, remoteUploadTask, remotePollTask, remoteSendTask) } + // Remote seal provider tasks (Notify, Cleanup; Finalize is registered above with addFinalize) + if cfg.Subsystems.EnableRemoteSealProvider { + if !addFinalize { + // If no sealing tasks started the poller, start it now for the provider. + go provPoller.RunPoller(ctx) + } + + provMaxTasks := cfg.Subsystems.RemoteSealProviderMaxTasks + cleanupTimeout := cfg.Subsystems.RemoteSealCleanupTimeout + + notifyTask := remoteseal.NewProviderNotifyTask(db, provPoller, provMaxTasks, cleanupTimeout) + provCleanupTask := remoteseal.NewProviderCleanupTask(db, provPoller, stor, slotMgr, cfg.Subsystems.FinalizeMaxTasks) + + activeTasks = append(activeTasks, notifyTask, provCleanupTask) + + if !addFinalize { + // If the provider finalize task wasn't already registered via addFinalize + // (e.g. provider-only node without local sealing), register it here. + provFinalizeTask := remoteseal.NewProviderFinalizeTask(db, provPoller, slr, slotMgr, cfg.Subsystems.FinalizeMaxTasks) + activeTasks = append(activeTasks, provFinalizeTask) + } + + // Provider-side SDR/Tree tasks are handled by the existing SDR/TreeD/TreeRC tasks + // via UNION ALL queries - they just need to be enabled (EnableSealSDR/EnableSealSDRTrees). + // The SDR/TreeD/TreeRC tasks register their AddTaskFunc with provPoller in their Adder() methods. + } + + // Remote seal client tasks + if cfg.Subsystems.EnableRemoteSealClient { + clientPoller := remoteseal.NewRSealClientPoller(db) + go clientPoller.RunPoller(ctx) + + rsealClient := remoteseal.NewRSealClient() + + delegateTask := remoteseal.NewRSealDelegate(db, full, rsealClient) + pollTask := remoteseal.NewRSealClientPoll(db, rsealClient, clientPoller) + fetchTask := remoteseal.NewRSealClientFetch(db, rsealClient, slr, clientPoller, cfg.Subsystems.RSealClientFetchMaxTasks) + cleanupTask := remoteseal.NewRSealClientCleanup(db, rsealClient, clientPoller) + + activeTasks = append(activeTasks, delegateTask, pollTask, fetchTask, cleanupTask) + } + // harmony treats the first task as highest priority, so reverse the order // (we could have just appended to this list in the reverse order, but defining // tasks in pipeline order is more intuitive) diff --git a/cmd/curio/test-supra.go b/cmd/curio/test-supra.go index f7fbe9d01..b7b038539 100644 --- a/cmd/curio/test-supra.go +++ b/cmd/curio/test-supra.go @@ -12,9 +12,12 @@ import ( "github.com/urfave/cli/v2" "golang.org/x/xerrors" + ffi "github.com/filecoin-project/filecoin-ffi" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/curio/build" "github.com/filecoin-project/curio/cmd/curio/internal/translations" + "github.com/filecoin-project/curio/harmony/resources" "github.com/filecoin-project/curio/lib/ffi/cunative" "github.com/filecoin-project/curio/lib/supraffi" ) @@ -64,6 +67,26 @@ var testSupraSystemInfoCmd = &cli.Command{ fmt.Println("CUDA:") fmt.Printf(" Usable CUDA GPU detected: %s\n", yesNo(supraffi.HasUsableCUDAGPU())) + fmt.Println() + + fmt.Println("GPU Devices (ffi):") + gpuMode := "OpenCL" + if build.IsOpencl != "1" { + gpuMode = "CUDA" + } + fmt.Printf(" Mode: %s\n", gpuMode) + fmt.Printf(" Overprovision factor: %d\n", resources.GpuOverprovisionFactor) + gpus, err := ffi.GetGPUDevices() + if err != nil { + fmt.Printf(" Error listing GPUs: %s\n", err) + } else if len(gpus) == 0 { + fmt.Println(" No GPU devices found") + } else { + fmt.Printf(" Devices (%d):\n", len(gpus)) + for i, name := range gpus { + fmt.Printf(" [%d] %s\n", i, name) + } + } return nil }, diff --git a/cuhttp/server.go b/cuhttp/server.go index 8d5aa8202..14d99278e 100644 --- a/cuhttp/server.go +++ b/cuhttp/server.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "net" "net/http" "os" "strings" @@ -29,6 +30,7 @@ import ( ipni_provider "github.com/filecoin-project/curio/market/ipni/ipni-provider" "github.com/filecoin-project/curio/market/libp2p" "github.com/filecoin-project/curio/market/retrieval" + "github.com/filecoin-project/curio/market/sealmarket" "github.com/filecoin-project/curio/pdp" "github.com/filecoin-project/curio/tasks/message" storage_market "github.com/filecoin-project/curio/tasks/storage-market" @@ -143,6 +145,7 @@ func isWebSocketUpgrade(r *http.Request) bool { type ServiceDeps struct { EthSender *message.SenderETH DealMarket *storage_market.CurioStorageDealMarket + SealMarket *sealmarket.SealMarket AlertTask *alertmanager.AlertTask } @@ -206,7 +209,6 @@ func StartHTTPServer(ctx context.Context, d *deps.Deps, sd *ServiceDeps) error { // Set up the HTTP server with proper timeouts server := &http.Server{ - Addr: cfg.ListenAddress, Handler: libp2pConnMiddleware(loggingMiddleware(compressionMw(chiRouter))), // Attach middlewares ReadTimeout: cfg.ReadTimeout, WriteTimeout: time.Hour * 2, @@ -225,19 +227,27 @@ func StartHTTPServer(ctx context.Context, d *deps.Deps, sd *ServiceDeps) error { server.TLSConfig = certManager.TLSConfig() } - // We don't need to run an HTTP server. Any HTTP request should simply be handled as HTTPS. + // Bind the listener before starting the goroutine so the caller knows + // the actual address (important when ListenAddress uses port 0). + ln, err := net.Listen("tcp", cfg.ListenAddress) + if err != nil { + return xerrors.Errorf("binding HTTP listener on %s: %w", cfg.ListenAddress, err) + } + + d.HTTPListenAddr = ln.Addr().String() + log.Infof("HTTP server listening on %s (requested %s)", d.HTTPListenAddr, cfg.ListenAddress) - // Start the server with TLS + // Start the server go func() { - log.Infof("Starting HTTPS server for https://%s on %s", cfg.DomainName, cfg.ListenAddress) + log.Infof("Starting HTTP server for %s on %s", cfg.DomainName, d.HTTPListenAddr) var serr error if !cfg.DelegateTLS { - serr = server.ListenAndServeTLS("", "") + serr = server.ServeTLS(ln, "", "") } else { - serr = server.ListenAndServe() + serr = server.Serve(ln) } if serr != nil && !errors.Is(serr, http.ErrServerClosed) { - log.Errorf("Failed to start HTTPS server: %s", serr) + log.Errorf("Failed to start HTTP server: %s", serr) panic(serr) } }() @@ -294,37 +304,40 @@ func (c cache) Delete(ctx context.Context, key string) error { var _ autocert.Cache = cache{} func attachRouters(ctx context.Context, r *chi.Mux, d *deps.Deps, sd *ServiceDeps) (*chi.Mux, error) { - // Create denylist filter for retrieval endpoints - df := denylist.NewFilter(ctx, d.Cfg.HTTP.DenylistServers) - - // Attach retrievals with denylist filtering at both URL and blockstore level - rp := retrieval.NewRetrievalProvider(ctx, d.DB, d.IndexStore, d.CachedPieceReader, df) - retrieval.Router(r, rp, df) + // Deal market routers (retrieval, IPNI, libp2p, market handler) are only + // attached when the deal market subsystem is enabled. Other HTTP features + // (e.g. remote seal) can run without these dependencies. + if d.Cfg.Subsystems.EnableDealMarket { + df := denylist.NewFilter(ctx, d.Cfg.HTTP.DenylistServers) + rp := retrieval.NewRetrievalProvider(ctx, d.DB, d.IndexStore, d.CachedPieceReader, df) + retrieval.Router(r, rp, df) + + ipp, err := ipni_provider.NewProvider(d) + if err != nil { + return nil, xerrors.Errorf("failed to create new ipni provider: %w", err) + } + ipni_provider.Routes(r, ipp) - // Attach IPNI - ipp, err := ipni_provider.NewProvider(d) - if err != nil { - return nil, xerrors.Errorf("failed to create new ipni provider: %w", err) - } - ipni_provider.Routes(r, ipp) + go ipp.StartPublishing(ctx) - go ipp.StartPublishing(ctx) + rd := libp2p.NewRedirector(d.DB) + libp2p.Router(r, rd) - // Attach LibP2P redirector - rd := libp2p.NewRedirector(d.DB) - libp2p.Router(r, rd) + dh, err := mhttp.NewMarketHandler(d.DB, d.Cfg, sd.DealMarket, must.One(d.EthClient.Get()), d.Chain, sd.EthSender, d.LocalStore) + if err != nil { + return nil, xerrors.Errorf("failed to create new market handler: %w", err) + } + mhttp.Router(r, dh) + } if sd.EthSender != nil { pdsvc := pdp.NewPDPService(ctx, d.DB, d.LocalStore, must.One(d.EthClient.Get()), d.Chain, sd.EthSender, sd.AlertTask) pdp.Routes(r, pdsvc) } - // Attach the market handler - dh, err := mhttp.NewMarketHandler(d.DB, d.Cfg, sd.DealMarket, must.One(d.EthClient.Get()), d.Chain, sd.EthSender, d.LocalStore) - if err != nil { - return nil, xerrors.Errorf("failed to create new market handler: %w", err) + if sd.SealMarket != nil { + sealmarket.Routes(r, sd.SealMarket) } - mhttp.Router(r, dh) return r, nil } diff --git a/deps/config/doc_gen.go b/deps/config/doc_gen.go index 2aef1b176..3c3add972 100644 --- a/deps/config/doc_gen.go +++ b/deps/config/doc_gen.go @@ -815,6 +815,14 @@ NOTE: This definitely is not safe on PoSt nodes.`, Comment: `EnableWebGui enables the web GUI on this curio instance. The UI has minimal local overhead, but it should only need to be run on a single machine in the cluster. (Default: false)`, + }, + { + Name: "WebInstanceColorHue", + Type: "uint8", + + Comment: `WebInstanceColorHue sets a hue value (0-255) to visually identify this cluster instance in the web GUI. +When set, the UI displays a 3px right border with a pastel desaturated color of the specified hue. +Examples: 0=LightCoral, 60=Khaki, 120=LightGreen, 240=LightSteelBlue`, }, { Name: "GuiAddress", @@ -842,6 +850,48 @@ also be bounded by resources available on the machine. (Default: 0 - unlimited)` Comment: `EnableBatchSeal enabled SupraSeal batch sealing on the node. (Default: false)`, }, + { + Name: "EnableRemoteSealProvider", + Type: "bool", + + Comment: `EnableRemoteSealProvider enables the remote seal provider on this node. +When enabled, this node will accept seal orders from remote clients and perform +SDR + tree computation on their behalf. (Default: false)`, + }, + { + Name: "RemoteSealProviderMaxTasks", + Type: "int", + + Comment: `RemoteSealProviderMaxTasks limits how many concurrent remote seal orders the provider +will process. This controls the number of Notify, Finalize, and Cleanup tasks that can +run simultaneously. SDR/Tree concurrency is controlled by the existing SealSDRMaxTasks +and SealSDRTreesMaxTasks settings. Set to 0 for unlimited. (Default: 0 - unlimited)`, + }, + { + Name: "RemoteSealCleanupTimeout", + Type: "time.Duration", + + Comment: `RemoteSealCleanupTimeout is how long the provider keeps sealed sector data after +notifying the client of completion. If the client doesn't trigger cleanup within this +period, the provider automatically cleans up the data. (Default: 72h)`, + }, + { + Name: "EnableRemoteSealClient", + Type: "bool", + + Comment: `EnableRemoteSealClient enables the remote seal client on this node. +When enabled, this node can delegate SDR + tree computation to remote providers +configured in the rseal_client_providers table. (Default: false)`, + }, + { + Name: "RSealClientFetchMaxTasks", + Type: "int", + + Comment: `RSealClientFetchMaxTasks limits how many concurrent remote seal fetch tasks can +run on this node. Each fetch downloads ~32 GiB of sealed data from a remote +provider, so this effectively caps concurrent download bandwidth usage. +Set to 0 for unlimited. (Default: 8)`, + }, { Name: "EnableDealMarket", Type: "bool", diff --git a/deps/config/types.go b/deps/config/types.go index 2e6f2af06..edcd4c787 100644 --- a/deps/config/types.go +++ b/deps/config/types.go @@ -19,6 +19,8 @@ func DefaultCurioConfig() *CurioConfig { IndexingMaxTasks: 8, RemoteProofMaxUploads: 15, ParkPieceMinFreeStoragePercent: 5, + RemoteSealCleanupTimeout: 72 * time.Hour, + RSealClientFetchMaxTasks: 8, }, Fees: CurioFees{ MaxPreCommitBatchGasFee: BatchFeeConfig{ @@ -389,6 +391,11 @@ type CurioSubsystemsConfig struct { // only need to be run on a single machine in the cluster. (Default: false) EnableWebGui bool + // WebInstanceColorHue sets a hue value (0-255) to visually identify this cluster instance in the web GUI. + // When set, the UI displays a 3px right border with a pastel desaturated color of the specified hue. + // Examples: 0=LightCoral, 60=Khaki, 120=LightGreen, 240=LightSteelBlue + WebInstanceColorHue uint8 + // The address that should listen for Web GUI requests. It should be in form "x.x.x.x:1234" (Default: 0.0.0.0:4701) GuiAddress string @@ -403,6 +410,33 @@ type CurioSubsystemsConfig struct { // EnableBatchSeal enabled SupraSeal batch sealing on the node. (Default: false) EnableBatchSeal bool + // EnableRemoteSealProvider enables the remote seal provider on this node. + // When enabled, this node will accept seal orders from remote clients and perform + // SDR + tree computation on their behalf. (Default: false) + EnableRemoteSealProvider bool + + // RemoteSealProviderMaxTasks limits how many concurrent remote seal orders the provider + // will process. This controls the number of Notify, Finalize, and Cleanup tasks that can + // run simultaneously. SDR/Tree concurrency is controlled by the existing SealSDRMaxTasks + // and SealSDRTreesMaxTasks settings. Set to 0 for unlimited. (Default: 0 - unlimited) + RemoteSealProviderMaxTasks int + + // RemoteSealCleanupTimeout is how long the provider keeps sealed sector data after + // notifying the client of completion. If the client doesn't trigger cleanup within this + // period, the provider automatically cleans up the data. (Default: 72h) + RemoteSealCleanupTimeout time.Duration + + // EnableRemoteSealClient enables the remote seal client on this node. + // When enabled, this node can delegate SDR + tree computation to remote providers + // configured in the rseal_client_providers table. (Default: false) + EnableRemoteSealClient bool + + // RSealClientFetchMaxTasks limits how many concurrent remote seal fetch tasks can + // run on this node. Each fetch downloads ~32 GiB of sealed data from a remote + // provider, so this effectively caps concurrent download bandwidth usage. + // Set to 0 for unlimited. (Default: 8) + RSealClientFetchMaxTasks int + // EnableDealMarket enabled the deal market on the node. This would also enable libp2p on the node, if configured. (Default: false) EnableDealMarket bool diff --git a/deps/deps.go b/deps/deps.go index 0371b2d26..6e2e1bbf8 100644 --- a/deps/deps.go +++ b/deps/deps.go @@ -169,6 +169,7 @@ type Deps struct { LocalPaths *paths.BasicLocalStorage Prover storiface.Prover ListenAddr string + HTTPListenAddr string // actual address of the HTTP server (set after bind) Name string MachineID *int64 Alert *alertmanager.AlertNow @@ -309,16 +310,19 @@ func (deps *Deps) PopulateRemainingDeps(ctx context.Context, cctx *cli.Context, if deps.ListenAddr == "" { listenAddr := cctx.String("listen") const unspecifiedAddress = "0.0.0.0" - addressSlice := strings.Split(listenAddr, ":") - if ip := net.ParseIP(addressSlice[0]); ip != nil { - if ip.String() == unspecifiedAddress { - rip, err := deps.DB.GetRoutableIP() - if err != nil { - return err + // Use net.SplitHostPort to properly handle both IPv4 and IPv6 addresses + host, port, err := net.SplitHostPort(listenAddr) + if err == nil { + if ip := net.ParseIP(host); ip != nil { + if ip.String() == unspecifiedAddress { + rip, err := deps.DB.GetRoutableIP() + if err != nil { + return err + } + deps.ListenAddr = net.JoinHostPort(rip, port) + } else { + deps.ListenAddr = net.JoinHostPort(ip.String(), port) } - deps.ListenAddr = rip + ":" + addressSlice[1] - } else { - deps.ListenAddr = ip.String() + ":" + addressSlice[1] } } } diff --git a/documentation/en/SUMMARY.md b/documentation/en/SUMMARY.md index 64a4a7a06..f8f860458 100644 --- a/documentation/en/SUMMARY.md +++ b/documentation/en/SUMMARY.md @@ -70,4 +70,7 @@ * [Snark Market (Consumer)](experimental-features/Snark-Market-Consumer.md) * [Market 2.0 API](experimental-features/market-2.0-api.md) * [Wallet Exporter](experimental-features/Wallet-Exporter.md) + * [Remote Seal](remote-seal.md) + * [Client Guide](remote-seal-client.md) + * [Provider Guide](remote-seal-provider.md) * [cuzk Proving Daemon](experimental-features/cuzk-proving-daemon.md) diff --git a/documentation/en/configuration/default-curio-configuration.md b/documentation/en/configuration/default-curio-configuration.md index 6247b3ade..097feaa5f 100644 --- a/documentation/en/configuration/default-curio-configuration.md +++ b/documentation/en/configuration/default-curio-configuration.md @@ -246,6 +246,13 @@ description: The default curio configuration # type: bool #EnableWebGui = false + # WebInstanceColorHue sets a hue value (0-255) to visually identify this cluster instance in the web GUI. + # When set, the UI displays a 3px right border with a pastel desaturated color of the specified hue. + # Examples: 0=LightCoral, 60=Khaki, 120=LightGreen, 240=LightSteelBlue + # + # type: uint8 + #WebInstanceColorHue = 0 + # The address that should listen for Web GUI requests. It should be in form "x.x.x.x:1234" (Default: 0.0.0.0:4701) # # type: string @@ -268,6 +275,35 @@ description: The default curio configuration # type: bool #EnableBatchSeal = false + # EnableRemoteSealProvider enables the remote seal provider on this node. + # When enabled, this node will accept seal orders from remote clients and perform + # SDR + tree computation on their behalf. (Default: false) + # + # type: bool + #EnableRemoteSealProvider = false + + # RemoteSealProviderMaxTasks limits how many concurrent remote seal orders the provider + # will process. This controls the number of Notify, Finalize, and Cleanup tasks that can + # run simultaneously. SDR/Tree concurrency is controlled by the existing SealSDRMaxTasks + # and SealSDRTreesMaxTasks settings. Set to 0 for unlimited. (Default: 0 - unlimited) + # + # type: int + #RemoteSealProviderMaxTasks = 0 + + # RemoteSealCleanupTimeout is how long the provider keeps sealed sector data after + # notifying the client of completion. If the client doesn't trigger cleanup within this + # period, the provider automatically cleans up the data. (Default: 72h) + # + # type: time.Duration + #RemoteSealCleanupTimeout = "72h0m0s" + + # EnableRemoteSealClient enables the remote seal client on this node. + # When enabled, this node can delegate SDR + tree computation to remote providers + # configured in the rseal_client_providers table. (Default: false) + # + # type: bool + #EnableRemoteSealClient = false + # EnableDealMarket enabled the deal market on the node. This would also enable libp2p on the node, if configured. (Default: false) # # type: bool diff --git a/documentation/en/configuration/metrics-reference.md b/documentation/en/configuration/metrics-reference.md index e497b6397..cd6c1f8d6 100644 --- a/documentation/en/configuration/metrics-reference.md +++ b/documentation/en/configuration/metrics-reference.md @@ -199,6 +199,14 @@ This document lists all Prometheus metrics exported by Curio. All metrics use th | `curio_pdp/piece_by_cid_request_duration_ms` | gauge | Time spent retrieving a piece by cid for PDP | | `curio_pdp/piece_bytes_served_count` | gauge/counter | Counter of the number of bytes served by PDP since startup | | `curio_retrieval_info` | gauge/counter | Arbitrary counter to tag node info to | +| `curio_rseal/active_requests` | gauge/counter | Number of active remote seal HTTP requests | +| `curio_rseal/commit1_compute_duration_ms` | gauge | Remote seal commit1 FFI compute duration | +| `curio_rseal/orders_total` | gauge/counter | Counter of remote seal order attempts | +| `curio_rseal/request_count` | gauge/counter | Counter of remote seal HTTP requests | +| `curio_rseal/request_duration_ms` | gauge | Remote seal HTTP request duration | +| `curio_rseal/response_bytes_count` | gauge/counter | Sum of remote seal HTTP response bytes | +| `curio_rseal/response_status_count` | gauge/counter | Counter of remote seal HTTP response status codes | +| `curio_rseal/slots_issued_total` | gauge/counter | Counter of remote seal slot tokens issued | ## GC Metrics diff --git a/documentation/en/installation.md b/documentation/en/installation.md index 488561d4a..32181a9b6 100644 --- a/documentation/en/installation.md +++ b/documentation/en/installation.md @@ -278,7 +278,7 @@ We recommend that macOS users use [Homebrew](https://brew.sh/) to install each o Use the command `brew install` to install the following packages: ```shell -brew install jq pkg-config hwloc coreutils +brew install jq pkg-config hwloc coreutils aria2 brew install go@1.24 ``` diff --git a/documentation/en/remote-seal-client.md b/documentation/en/remote-seal-client.md new file mode 100644 index 000000000..ca46c1282 --- /dev/null +++ b/documentation/en/remote-seal-client.md @@ -0,0 +1,212 @@ +# Remote Seal Client Guide + +This guide covers setting up and operating a Curio node as a **remote seal +client** -- a node that delegates SDR and tree computation to a remote provider +while handling the rest of the sealing pipeline locally. + +## Prerequisites + +- A Curio node with access to the shared HarmonyDB cluster +- HTTP/HTTPS enabled on the node (the provider sends completion callbacks to + the client) +- A miner actor and wallet for on-chain messages +- Storage for sealed sectors (long-term) and cache (temporary) +- GPU for PoRep SNARK proof computation +- A configured remote seal provider (you need a connect string from the + provider operator) + +## Configuration + +Enable the remote seal client in your Curio configuration layer: + +```toml +[Subsystems] +# Core requirement: enable the remote seal client +EnableRemoteSealClient = true + +# The client still needs the rest of the pipeline: +EnableSendPrecommitMsg = true +EnablePoRepProof = true +EnableSendCommitMsg = true +EnableMoveStorage = true + +# Optional but recommended: enable finalize on this node +# FinalizeMaxTasks = 10 + +[HTTP] +# HTTP must be enabled -- the provider sends /complete callbacks here +Enable = true +DomainName = "seal-client.example.com" +ListenAddress = "0.0.0.0:12300" +``` + +The client does **not** need `EnableSealSDR` or `EnableSealSDRTrees` -- those +run on the provider. + +## Adding a Provider + +### Using the Web UI + +1. Obtain a **connect string** from the provider operator. +2. Navigate to the Remote Seal section in the Curio web UI. +3. Click "Add Provider" and provide: + - **SP ID**: The miner actor ID that should use this provider (e.g. `1234`) + - **Connect String**: The base64 string from the provider operator +4. The system decodes the connect string and stores the provider URL and token. + +The connect string contains: +```json +{"url": "https://seal-provider.example.com", "token": "hex-encoded-auth-token"} +``` + +### Enabling/Disabling Providers + +Providers can be toggled on/off in the UI. When disabled, no new sectors will +be delegated to that provider, but existing in-flight sectors will complete +normally. + +### Multiple Providers + +You can configure multiple providers for the same miner, or different providers +for different miners in a multi-miner cluster. The delegate task picks an +available provider per-sector. + +## How It Works + +### Client Pipeline States + +When a sector is delegated, two pipeline tables track its state: + +**`rseal_client_pipeline`** (remote-seal-specific state): +``` +RSealDelegate creates entry + sends /order + | + v +RSealClientPoll (polls /status, or receives /complete callback) + | Applies completion: after_sdr, after_tree_d, after_tree_c, + | after_tree_r = TRUE in BOTH rseal_client_pipeline + | AND sectors_sdr_pipeline + v +RSealClientFetch (downloads sealed sector + cache from provider) + | Downloads 32 GiB sealed file (GET /sealed-data, supports Range) + | Downloads ~73 MiB cache tar (GET /cache-data) + | Writes c1.url file in cache directory + | after_fetch = TRUE + v +[Normal pipeline: precommit -> PoRep -> commit -> finalize -> move-storage] + v +RSealClientCleanup (after PoRep completes) + | POST /finalize (provider drops layers) + | POST /cleanup (provider removes all data) + | after_cleanup = TRUE +``` + +**`sectors_sdr_pipeline`** (standard pipeline, remote sectors are marked): +``` +After ApplyRemoteCompletion: + after_sdr = TRUE + after_tree_d = TRUE + after_tree_c = TRUE + after_tree_r = TRUE + after_synth = TRUE (remote sectors skip local synth) + ticket_epoch, ticket_value = from provider + tree_d_cid, tree_r_cid = from provider + all task_id_* = NULL (ready for next stages) + +The SealPoller detects this sector has a rseal_client_pipeline entry +(is_remote = true) and skips SDR/tree/synth scheduling, but continues +with: precommit -> PoRep -> finalize -> move-storage -> commit +``` + +### Client Tasks + +| Task | Schedule | Purpose | +|---|---|---| +| **RSealDelegate** | IAmBored (every 15s) | Finds unclaimed sectors, checks provider availability, sends orders | +| **RSealClientPoll** | Poller creates when `after_sdr=FALSE, task_id_sdr=NULL` | Polls provider status in a 30s loop until complete/failed | +| **RSealClientFetch** | Poller creates when `after_sdr=TRUE, after_fetch=FALSE` | Downloads sealed data and cache from provider | +| **RSealClientCleanup** | Poller creates when `after_porep=TRUE, after_cleanup=FALSE` | Sends finalize + cleanup to provider | + +### How C1 (Commit Phase 1) Works + +The PoRep SNARK proof requires a "vanilla proof" (C1 output) that can only be +computed where the full cache directory exists. For remote-sealed sectors, +the cache on the client does not have SDR layers, so C1 must be fetched from +the provider. + +This is handled transparently: + +1. During fetch, `RSealClientFetch` writes a `c1.url` file in the sector's + cache directory containing the provider's `/commit1` endpoint URL and auth + token. +2. When `PoRepSnark` runs, it calls `GeneratePoRepVanillaProof()`. +3. The function detects the `c1.url` file and makes an HTTP POST to the + provider's `/commit1` endpoint with the interactive randomness seed. +4. The provider computes C1 and returns the raw bytes (~50-128 MiB). +5. The result is cached as `commit-phase1-output` in the cache directory. +6. The SNARK proof is computed normally. + +The PoRep task itself is completely unaware of remote seal -- the C1 fetch is +transparent. + +### Completion: Callback vs Polling + +The client learns about provider completion through two paths: + +1. **Callback** (primary): The provider sends `POST /complete` to the client's + HTTP endpoint. This is immediate and carries the ticket + CIDs. +2. **Polling** (fallback): `RSealClientPoll` queries `POST /status` every 30s. + If the callback was missed (network issue, restart), the poll task catches + it. + +Both paths call `ApplyRemoteCompletion()` which is idempotent. + +### Failure Handling + +If the provider reports a sector as `"failed"`: +- `rseal_client_pipeline.failed = TRUE` with reason from provider +- `sectors_sdr_pipeline` task_ids are cleared +- The sector can potentially be retried (manually or by a future retry + mechanism) + +If the provider is unreachable: +- The poll task retries with 30s intervals +- After 10 consecutive failures (`MaxFailures`), the task stops and needs + manual intervention + +## Network Requirements + +The client makes these HTTP calls to the provider: + +| Call | Data Size | Notes | +|---|---|---| +| `POST /available` | ~100 bytes | Quick availability check | +| `POST /order` | ~200 bytes | Creates seal order | +| `POST /status` | ~200 bytes | Polls state (every 30s while waiting) | +| `GET /sealed-data` | **32 GiB** | Supports HTTP Range for resumable downloads | +| `GET /cache-data` | **~73 MiB** | Tar archive | +| `POST /commit1` | **~50-128 MiB response** | Raw bytes, computed on demand | +| `POST /finalize` | ~200 bytes | Signals layer drop | +| `POST /cleanup` | ~200 bytes | Signals full cleanup | + +The provider makes one callback to the client: + +| Call | Data Size | Notes | +|---|---|---| +| `POST /complete` | ~500 bytes | CIDs + ticket data | + +Ensure the network between client and provider can handle the 32 GiB sealed +sector transfer. HTTP Range headers are supported, so aria2c or similar +multi-connection download tools can be used. + +## Monitoring + +Client pipeline status is visible in the Curio web UI under the Remote Seal +section. Each delegated sector shows: + +- Provider name +- Current state (after_sdr, after_fetch, etc.) +- Failure information if applicable + +The standard `sectors_sdr_pipeline` view also shows remote sectors, but with +SDR/tree/synth stages already marked as complete. diff --git a/documentation/en/remote-seal-flow.dot b/documentation/en/remote-seal-flow.dot new file mode 100644 index 000000000..222b49a79 --- /dev/null +++ b/documentation/en/remote-seal-flow.dot @@ -0,0 +1,134 @@ +digraph RemoteSeal { + rankdir=TB; + compound=true; + fontname="Helvetica"; + node [fontname="Helvetica", fontsize=11]; + edge [fontname="Helvetica", fontsize=9]; + newrank=true; + + // ===== CLIENT SIDE ===== + subgraph cluster_client { + label="CLIENT CURIO NODE"; + style=filled; + fillcolor="#e8f0fe"; + color="#4285f4"; + fontsize=14; + fontcolor="#1a73e8"; + + // Normal pipeline entry + sector_created [label="Sector Created\n(sectors_sdr_pipeline)", shape=box, style="filled,rounded", fillcolor="#fff3e0", color="#e65100"]; + + subgraph cluster_client_remote { + label="Remote Seal Client Tasks"; + style=dashed; + color="#666"; + fontsize=11; + fontcolor="#666"; + + delegate [label="RSealDelegate\n(IAmBored 15s)", shape=box, style="filled,rounded", fillcolor="#c8e6c9", color="#2e7d32"]; + client_poll [label="RSealClientPoll\n(loop 30s, CanYield)", shape=box, style="filled,rounded", fillcolor="#c8e6c9", color="#2e7d32"]; + client_fetch [label="RSealClientFetch\n(download sealed+cache)", shape=box, style="filled,rounded", fillcolor="#c8e6c9", color="#2e7d32"]; + client_cleanup [label="RSealClientCleanup\n(finalize+cleanup)", shape=box, style="filled,rounded", fillcolor="#c8e6c9", color="#2e7d32"]; + } + + subgraph cluster_normal_pipeline { + label="Normal Seal Pipeline (client-side)"; + style=dashed; + color="#666"; + fontsize=11; + fontcolor="#666"; + + precommit [label="PrecommitSubmit\n(on-chain msg)", shape=box, style="filled,rounded", fillcolor="#bbdefb", color="#1565c0"]; + porep [label="PoRepSnark\n(fetches C1 via c1.url)", shape=box, style="filled,rounded", fillcolor="#bbdefb", color="#1565c0"]; + finalize_local [label="Finalize\n(move storage)", shape=box, style="filled,rounded", fillcolor="#bbdefb", color="#1565c0"]; + commit [label="CommitSubmit\n(on-chain msg)", shape=box, style="filled,rounded", fillcolor="#bbdefb", color="#1565c0"]; + } + + complete_recv [label="/complete\ncallback", shape=diamond, style="filled", fillcolor="#fff9c4", color="#f9a825", fontsize=9]; + } + + // ===== PROVIDER SIDE ===== + subgraph cluster_provider { + label="PROVIDER CURIO NODE"; + style=filled; + fillcolor="#fce4ec"; + color="#c62828"; + fontsize=14; + fontcolor="#c62828"; + + order_recv [label="Order Received\n(rseal_provider_pipeline)", shape=box, style="filled,rounded", fillcolor="#fff3e0", color="#e65100"]; + + subgraph cluster_prov_compute { + label="Computation (existing seal tasks via UNION ALL)"; + style=dashed; + color="#666"; + fontsize=11; + fontcolor="#666"; + + prov_sdr [label="SDR\n(computes ticket\nfrom chain)", shape=box, style="filled,rounded", fillcolor="#f8bbd0", color="#880e4f"]; + prov_treed [label="TreeD", shape=box, style="filled,rounded", fillcolor="#f8bbd0", color="#880e4f"]; + prov_treerc [label="TreeRC\n(TreeC + TreeR)", shape=box, style="filled,rounded", fillcolor="#f8bbd0", color="#880e4f"]; + } + + subgraph cluster_prov_lifecycle { + label="Provider Lifecycle Tasks"; + style=dashed; + color="#666"; + fontsize=11; + fontcolor="#666"; + + prov_notify [label="RSealProvNotify\n(POST /complete)", shape=box, style="filled,rounded", fillcolor="#e1bee7", color="#6a1b9a"]; + prov_finalize [label="RSealProvFinalize\n(drop layers)", shape=box, style="filled,rounded", fillcolor="#e1bee7", color="#6a1b9a"]; + prov_cleanup [label="RSealProvCleanup\n(remove all data)", shape=box, style="filled,rounded", fillcolor="#e1bee7", color="#6a1b9a"]; + } + + c1_handler [label="/commit1\nhandler", shape=diamond, style="filled", fillcolor="#fff9c4", color="#f9a825", fontsize=9]; + } + + // ===== CLIENT INTERNAL FLOW ===== + sector_created -> delegate [label="poller detects\nunassigned sector"]; + delegate -> client_poll [label="order accepted\n(rseal_client_pipeline created)"]; + client_poll -> complete_recv [style=invis]; // layout hint + {complete_recv, client_poll} -> client_fetch [label="completion applied\n(ApplyRemoteCompletion)"]; + client_fetch -> precommit [label="after_fetch=TRUE\n(c1.url written)"]; + precommit -> porep [label="precommit on-chain\n+ seed epoch"]; + porep -> finalize_local; + finalize_local -> commit; + commit -> client_cleanup [label="after PoRep done"]; + + // ===== PROVIDER INTERNAL FLOW ===== + order_recv -> prov_sdr [label="poller assigns\ntask_id_sdr"]; + prov_sdr -> prov_treed [label="after_sdr=TRUE"]; + prov_treed -> prov_treerc [label="after_tree_d=TRUE"]; + prov_treerc -> prov_notify [label="after_tree_r=TRUE"]; + prov_notify -> c1_handler [label="waits for\nclient request", style=dashed]; + c1_handler -> prov_finalize [label="after_c1_supplied=TRUE"]; + prov_finalize -> prov_cleanup [label="cleanup_requested\nor 72h timeout"]; + + // ===== CROSS-NODE HTTP CALLS ===== + delegate -> order_recv [label="POST /available\nPOST /order", color="#e65100", fontcolor="#e65100", style=bold, constraint=false]; + client_poll -> order_recv [label="POST /status\n(polls state)", color="#e65100", fontcolor="#e65100", style=dashed, constraint=false]; + prov_notify -> complete_recv [label="POST /complete\n(ticket+CIDs)", color="#c62828", fontcolor="#c62828", style=bold, constraint=false]; + client_fetch -> order_recv [label="GET /sealed-data (32GiB)\nGET /cache-data (tar)", color="#e65100", fontcolor="#e65100", style=bold, constraint=false]; + porep -> c1_handler [label="POST /commit1\n(raw bytes ~50-128MiB)", color="#e65100", fontcolor="#e65100", style=bold, constraint=false]; + client_cleanup -> order_recv [label="POST /finalize\nPOST /cleanup", color="#e65100", fontcolor="#e65100", style=bold, constraint=false]; + + // Legend + subgraph cluster_legend { + label="Legend"; + style=filled; + fillcolor=white; + color="#999"; + fontsize=11; + fontcolor="#666"; + + leg_green [label="Client remote\nseal task", shape=box, style="filled,rounded", fillcolor="#c8e6c9", color="#2e7d32", fontsize=9]; + leg_blue [label="Normal pipeline\ntask (client)", shape=box, style="filled,rounded", fillcolor="#bbdefb", color="#1565c0", fontsize=9]; + leg_pink [label="Provider compute\ntask (UNION ALL)", shape=box, style="filled,rounded", fillcolor="#f8bbd0", color="#880e4f", fontsize=9]; + leg_purple [label="Provider lifecycle\ntask", shape=box, style="filled,rounded", fillcolor="#e1bee7", color="#6a1b9a", fontsize=9]; + leg_diamond [label="HTTP endpoint", shape=diamond, style="filled", fillcolor="#fff9c4", color="#f9a825", fontsize=9]; + leg_arrow_bold [label="HTTP call\n(cross-node)", shape=plaintext, fontsize=9]; + + leg_green -> leg_blue -> leg_pink -> leg_purple -> leg_diamond [style=invis]; + } +} diff --git a/documentation/en/remote-seal-flow.svg b/documentation/en/remote-seal-flow.svg new file mode 100644 index 000000000..82c49a877 --- /dev/null +++ b/documentation/en/remote-seal-flow.svg @@ -0,0 +1,386 @@ + + + + + diff --git a/documentation/en/remote-seal-provider.md b/documentation/en/remote-seal-provider.md new file mode 100644 index 000000000..f5b6d8ab4 --- /dev/null +++ b/documentation/en/remote-seal-provider.md @@ -0,0 +1,209 @@ +# Remote Seal Provider Guide + +This guide covers setting up and operating a Curio node as a **remote seal +provider** -- a node that performs SDR and tree computation on behalf of remote +clients. + +## Prerequisites + +- A Curio node with access to the shared HarmonyDB cluster +- HTTP/HTTPS enabled on the node (required for the seal market API) +- Sufficient hardware for SDR + tree computation (GPU recommended, large + temporary storage for layers and sealed sectors) +- A chain node connection (the provider computes SDR tickets from chain + randomness) + +## Configuration + +Enable the remote seal provider in your Curio configuration layer: + +```toml +[Subsystems] +# Core requirement: enable the remote seal provider +EnableRemoteSealProvider = true + +# The provider uses the existing SDR and tree tasks. +# These MUST be enabled on the same node or cluster: +EnableSealSDR = true +EnableSealSDRTrees = true + +[HTTP] +# HTTP must be enabled for the seal market API endpoints +Enable = true +DomainName = "seal-provider.example.com" # used in connect strings +ListenAddress = "0.0.0.0:12300" + +# TLS configuration (recommended for production) +# DelegateTLS = false # set to true if using a reverse proxy for TLS +``` + +## Managing Partners (Clients) + +Partners are clients authorized to send seal orders to this provider. Each +partner has an auth token and a sector allowance. + +### Adding a Partner + +Use the Curio web UI or the RPC API: + +1. Navigate to the Remote Seal section in the UI. +2. Click "Add Partner" and provide: + - **Name**: A human-readable label for this client + - **URL**: The client's base HTTP URL (the provider uses this to send + completion callbacks via `POST /complete`) + - **Allowance**: Maximum number of concurrent sectors this client can have + in the pipeline + +The system generates a random auth token for the partner. + +### Generating a Connect String + +After creating a partner, generate a **connect string** to share with the +client operator: + +1. In the UI, click "Get Connect String" for the partner. +2. The connect string is a base64-encoded JSON payload containing: + - Your provider's HTTPS URL (derived from `HTTP.DomainName`) + - The partner's auth token +3. Share this string with the client operator through a secure channel. + +### Adjusting Allowance + +You can update a partner's allowance at any time. The `allowance_remaining` +counter is decremented each time the partner sends a new order. To allow more +sectors, increase both `allowance_total` and `allowance_remaining`. + +### Removing a Partner + +A partner can only be removed if it has no active pipeline rows (sectors where +`after_cleanup = FALSE`). Ensure all sectors have completed the full lifecycle +before removing. + +## How It Works + +### Provider Pipeline States + +Each sector delegated to the provider goes through these states in +`rseal_provider_pipeline`: + +``` +Order received (row inserted via /order handler) + | + v +SDR (task_id_sdr assigned by provider poller) + | ticket_epoch, ticket_value written by SDR task + | after_sdr = TRUE + v +TreeD (task_id_tree_d assigned) + | tree_d_cid written + | after_tree_d = TRUE + v +TreeRC (task_id_tree_c, task_id_tree_r assigned - same task) + | tree_r_cid written + | after_tree_c = TRUE, after_tree_r = TRUE + v +NotifyClient (RSealProvNotify) + | POST /complete to client with CIDs + ticket + | after_notify_client = TRUE + | cleanup_timeout = NOW() + 72 hours + v +[Wait for client] + | Client fetches sealed data (GET /sealed-data) + | Client fetches cache (GET /cache-data) + | Client requests C1 (POST /commit1) -> after_c1_supplied = TRUE + | Client sends /finalize -> after_c1_supplied = TRUE (if not already) + v +Finalize (RSealProvFinalize) + | Drops SDR layers from storage + | after_finalize = TRUE + v +[Wait for cleanup] + | Client sends /cleanup -> cleanup_requested = TRUE + | OR cleanup_timeout expires (72h after notify) + v +Cleanup (RSealProvCleanup) + | Removes sealed, cache, unsealed data + | Releases batch slot + | after_cleanup = TRUE + v +[GC removes row after 24h] +``` + +### Computation Tasks + +The provider does not have its own SDR or tree task implementations. Instead, +the existing `SDR`, `TreeD`, and `TreeRC` tasks query both `sectors_sdr_pipeline` +and `rseal_provider_pipeline` via SQL `UNION ALL`. This means: + +- The same task code handles both local and remote sectors +- The provider node must have `EnableSealSDR = true` and + `EnableSealSDRTrees = true` to register these task types with harmonytask +- Resource management (GPU, storage) is shared between local and remote sectors + +### Ticket Computation + +The provider computes the SDR ticket directly from its own chain node. There +is no ticket exchange with the client. The ticket (epoch + randomness value) +is stored in `rseal_provider_pipeline` and sent to the client in the +`/complete` notification and `/status` response. + +### C1 (Commit Phase 1) + +When the client needs to compute the PoRep SNARK proof, it sends a +`POST /commit1` request with the interactive randomness seed. The provider: + +1. Looks up the sector's sealed/unsealed CIDs, ticket, and proof type +2. Calls `GeneratePoRepVanillaProof()` (the C1 computation) +3. Returns the result as raw bytes (`application/octet-stream`, ~50-128 MiB) +4. Sets `after_c1_supplied = TRUE` + +The provider must still have the sector's cache directory available at this +point (layers can be present or already finalized, but cache must exist). + +## API Endpoints Served + +These endpoints are served by the provider node under +`/remoteseal/delegated/v0/`: + +| Endpoint | Method | Called by | Purpose | +|---|---|---|---| +| `/capabilities` | GET | Client | Returns supported proof types and batch size | +| `/authorize` | POST | Client | Validates partner token | +| `/available` | POST | Client | Checks slot availability, returns 30s reservation token | +| `/order` | POST | Client | Creates a new seal order | +| `/status` | POST | Client | Returns sector state (pending/sdr/trees/complete/failed) | +| `/sealed-data/{sp}/{sn}` | GET | Client | Serves sealed sector file (32 GiB, supports Range headers) | +| `/cache-data/{sp}/{sn}` | GET | Client | Serves finalized cache as tar (~73 MiB) | +| `/commit1` | POST | Client | Accepts C1 seed, returns C1 output as raw bytes | +| `/finalize` | POST | Client | Client signals layers can be dropped | +| `/cleanup` | POST | Client | Client requests full data removal | + +Additionally, the provider calls this endpoint on the **client**: + +| Endpoint | Method | Called by | Purpose | +|---|---|---|---| +| `/complete` | POST | Provider | Notifies client that SDR+trees are done (includes CIDs + ticket) | + +## Storage Considerations + +The provider needs temporary storage for: + +- **SDR layers**: ~352 GiB per 32 GiB sector (11 layers x 32 GiB) +- **Sealed sector**: 32 GiB +- **Cache**: ~73 MiB (p_aux, t_aux, tree-r-last after finalize) + +After the client fetches the data and C1 is supplied: +- **Finalize** drops the SDR layers (~352 GiB freed) +- **Cleanup** removes everything else (~32 GiB freed) + +The 72-hour cleanup timeout ensures data is available for C1 retries even if +the client is slow to finalize. + +## Monitoring + +Provider pipeline status is visible in the Curio web UI under the Remote Seal +section. Each sector shows its current state (after_sdr, after_tree_d, etc.) +and any failure information. + +The `PipelineGC` task automatically removes completed `rseal_provider_pipeline` +rows 24 hours after the cleanup timeout expires. diff --git a/documentation/en/remote-seal.md b/documentation/en/remote-seal.md new file mode 100644 index 000000000..d5e34fa7f --- /dev/null +++ b/documentation/en/remote-seal.md @@ -0,0 +1,90 @@ +# Remote Seal (Sealing-as-a-Service) + +Remote Seal allows a Curio storage provider (**client**) to delegate the +computationally expensive parts of sector sealing -- SDR, TreeD, TreeC, TreeR -- +to a dedicated **provider** node. After the provider completes the heavy +computation, the client downloads the results and continues the standard +pipeline (precommit, PoRep, commit, move-storage) locally. + +This splits the sealing workload so that expensive GPU/CPU hardware can be +shared across multiple storage providers without requiring each one to maintain +its own sealing infrastructure. + +> **Status**: Experimental. Enable via `EnableRemoteSealProvider` / +> `EnableRemoteSealClient` configuration flags. + +## Architecture + +Both the client and provider Curio nodes share the same **HarmonyDB** +(YugabyteDB) cluster. Communication happens over HTTPS via a set of REST-style +API endpoints under `/remoteseal/delegated/v0/`. + + + +### Roles + +| Role | What it does | Heavy resources needed | +|---|---|---| +| **Client** | Owns the miner actor, submits on-chain messages (precommit, commit), stores sealed sectors, computes PoRep SNARK proof | GPU for SNARK, storage for sealed sectors | +| **Provider** | Performs SDR + tree computation on behalf of the client, serves sealed data and C1 output | GPU/CPU for SDR + trees, temporary storage | + +### What gets delegated + +Only the SDR and tree stages are delegated. Everything else stays on the client: + +| Stage | Where it runs | +|---|---| +| SDR (Stacked DRG encoding) | **Provider** | +| TreeD, TreeC, TreeR | **Provider** | +| Ticket computation | **Provider** (from its own chain node) | +| Precommit message | **Client** | +| PoRep SNARK proof | **Client** (fetches C1/vanilla proof from provider) | +| Commit message | **Client** | +| Finalize + move storage | **Client** | + +## Setup Flow + +1. **Provider operator** creates a partner entry in the UI, specifying the + client's base URL and an allowance (max concurrent sectors). +2. **Provider operator** generates a **connect string** -- a base64-encoded + JSON containing the provider's HTTPS URL and auth token. +3. The connect string is shared with the client operator out-of-band. +4. **Client operator** adds the provider in the UI using the connect string, + selecting which `sp_id` (miner) should use this provider. + +## Sealing Flow (End-to-End) + +1. Client's CC scheduler creates a sector in `sectors_sdr_pipeline`. +2. **RSealDelegate** (client, IAmBored task) detects the sector, checks provider + availability via `POST /available`, and sends `POST /order`. +3. Provider accepts the order, creating a row in `rseal_provider_pipeline`. +4. Provider's **SDR task** computes the ticket from chain randomness and runs SDR. +5. Provider's **TreeD** and **TreeRC** tasks build the Merkle trees. +6. **RSealProvNotify** sends `POST /complete` to the client with CIDs and ticket. +7. Client applies the completion -- marks `sectors_sdr_pipeline` as having SDR, + trees, and synth done, propagates ticket and CIDs. +8. **RSealClientFetch** downloads the sealed sector (32 GiB, supports HTTP Range) + and finalized cache (~73 MiB tar) from the provider. Writes a `c1.url` file. +9. Normal pipeline resumes: precommit message is submitted on-chain. +10. When the seed epoch arrives, **PoRepSnark** runs. It detects `c1.url`, + fetches C1 output from the provider via `POST /commit1` (raw bytes), and + computes the SNARK proof. +11. Commit message is submitted on-chain. +12. **RSealClientCleanup** sends `/finalize` (provider drops layers) then + `/cleanup` (provider removes all sector data). + +## Database Tables + +| Table | Side | Purpose | +|---|---|---| +| `rseal_delegated_partners` | Provider | Authorized partners (clients) with tokens and allowance | +| `rseal_client_providers` | Client | Configured remote seal providers per miner | +| `rseal_client_pipeline` | Client | Tracks delegated sector state (1:1 with `sectors_sdr_pipeline`) | +| `rseal_provider_pipeline` | Provider | Tracks sectors being sealed for remote clients | + +## Further Reading + +- [Remote Seal Provider Guide](remote-seal-provider.md) -- setting up and + operating a provider node +- [Remote Seal Client Guide](remote-seal-client.md) -- setting up and + operating a client node diff --git a/documentation/zh/installation.md b/documentation/zh/installation.md index b6a315e01..7b59c0dd6 100644 --- a/documentation/zh/installation.md +++ b/documentation/zh/installation.md @@ -284,7 +284,7 @@ xcode-select --install 使用命令`brew install`安装以下软件包: ```bash -brew install go bzr jq pkg-config hwloc coreutils +brew install go bzr jq pkg-config hwloc coreutils aria2 ``` 接下来是克隆Lotus仓库并构建可执行文件。 diff --git a/go.mod b/go.mod index 495b45154..f0202648a 100644 --- a/go.mod +++ b/go.mod @@ -83,6 +83,7 @@ require ( github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 github.com/minio/sha256-simd v1.0.1 github.com/mitchellh/go-homedir v1.1.0 + github.com/moby/moby/api v1.54.2 github.com/multiformats/go-base32 v0.1.0 github.com/multiformats/go-multiaddr v0.16.1 github.com/multiformats/go-multibase v0.3.0 @@ -102,6 +103,8 @@ require ( github.com/stretchr/testify v1.11.1 github.com/swaggo/http-swagger/v2 v2.0.2 github.com/swaggo/swag v1.16.6 + github.com/testcontainers/testcontainers-go v0.42.0 + github.com/testcontainers/testcontainers-go/modules/yugabytedb v0.42.0 github.com/triplewz/poseidon v0.0.2 github.com/urfave/cli/v2 v2.27.7 github.com/whyrusleeping/cbor-gen v0.3.2-0.20250409092040-76796969edea @@ -118,11 +121,15 @@ require ( golang.org/x/text v0.36.0 golang.org/x/tools v0.44.0 golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da + google.golang.org/grpc v1.79.3 + google.golang.org/protobuf v1.36.11 ) require ( + dario.cat/mergo v1.0.2 // indirect filippo.io/bigmod v0.1.1-0.20260103110540-f8a47775ebe5 // indirect filippo.io/keygen v0.0.0-20260114151900-8e2790ea4c5b // indirect + github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect github.com/GeertJohan/go.incremental v1.0.0 // indirect github.com/GeertJohan/go.rice v1.0.3 // indirect github.com/Gurpartap/async v0.0.0-20180927173644-4f7f499dd9ee // indirect @@ -135,13 +142,14 @@ require ( github.com/VividCortex/ewma v1.2.0 // indirect github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect github.com/akavel/rsrc v0.10.2 // indirect - github.com/andybalholm/brotli v1.2.1 // indirect + github.com/andybalholm/brotli v1.2.0 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/bahlo/generic-list-go v0.2.0 // indirect github.com/benbjohnson/clock v1.3.5 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.24.4 // indirect github.com/buger/jsonparser v1.1.2 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/charmbracelet/colorprofile v0.4.3 // indirect @@ -155,7 +163,12 @@ require ( github.com/clipperhouse/uax29/v2 v2.7.0 // indirect github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0 // indirect github.com/containerd/cgroups v1.1.0 // indirect + github.com/containerd/errdefs v1.0.0 // indirect + github.com/containerd/errdefs/pkg v0.3.0 // indirect + github.com/containerd/log v0.1.0 // indirect + github.com/containerd/platforms v0.2.1 // indirect github.com/coreos/go-systemd/v22 v22.7.0 // indirect + github.com/cpuguy83/dockercfg v0.3.2 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect github.com/crackcomm/go-gitignore v0.0.0-20241020182519-7843d2ba8fdf // indirect github.com/crate-crypto/go-eth-kzg v1.5.0 // indirect @@ -168,11 +181,14 @@ require ( github.com/dgraph-io/badger/v2 v2.2007.4 // indirect github.com/dgraph-io/ristretto v0.2.0 // indirect github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect + github.com/distribution/reference v0.6.0 // indirect + github.com/docker/go-connections v0.6.0 // indirect github.com/drand/drand/v2 v2.1.4 // indirect github.com/drand/go-clients v0.2.3 // indirect github.com/drand/kyber v1.3.2 // indirect github.com/drand/kyber-bls12381 v0.3.4 // indirect github.com/dunglas/httpsfv v1.1.0 // indirect + github.com/ebitengine/purego v0.10.0 // indirect github.com/elastic/go-elasticsearch/v7 v7.17.10 // indirect github.com/elastic/go-windows v1.0.2 // indirect github.com/elastic/gosigar v0.14.3 // indirect @@ -270,7 +286,9 @@ require ( github.com/libp2p/go-reuseport v0.4.0 // indirect github.com/libp2p/go-yamux/v5 v5.1.0 // indirect github.com/lucasb-eyer/go-colorful v1.4.0 // indirect + github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/magefile/mage v1.17.1 // indirect + github.com/magiconair/properties v1.8.10 // indirect github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.21 // indirect @@ -279,6 +297,14 @@ require ( github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect + github.com/moby/docker-image-spec v1.3.1 // indirect + github.com/moby/go-archive v0.2.0 // indirect + github.com/moby/moby/client v0.4.0 // indirect + github.com/moby/patternmatcher v0.6.1 // indirect + github.com/moby/sys/sequential v0.6.0 // indirect + github.com/moby/sys/user v0.4.0 // indirect + github.com/moby/sys/userns v0.1.0 // indirect + github.com/moby/term v0.5.2 // indirect github.com/mr-tron/base58 v1.3.0 // indirect github.com/muesli/termenv v0.16.0 // indirect github.com/multiformats/go-base36 v0.2.0 // indirect @@ -288,6 +314,8 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/nikkolasg/hexjson v0.1.0 // indirect github.com/nkovacs/streamquote v1.1.0 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.1 // indirect github.com/opencontainers/runtime-spec v1.2.1 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/pb33f/ordered-map/v2 v2.3.1 // indirect @@ -311,6 +339,7 @@ require ( github.com/pion/webrtc/v4 v4.2.11 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/polydawn/refmt v0.89.1-0.20231129105047-37766d95467a // indirect + github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.67.5 // indirect github.com/prometheus/otlptranslator v1.0.0 // indirect @@ -325,12 +354,13 @@ require ( github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/rvagg/go-skellam-pmf v0.0.2 // indirect github.com/shirou/gopsutil v3.21.11+incompatible // indirect + github.com/shirou/gopsutil/v4 v4.26.3 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/supranational/blst v0.3.16 // indirect github.com/swaggo/files/v2 v2.0.2 // indirect github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect - github.com/tklauser/go-sysconf v0.3.15 // indirect - github.com/tklauser/numcpus v0.10.0 // indirect + github.com/tklauser/go-sysconf v0.3.16 // indirect + github.com/tklauser/numcpus v0.11.0 // indirect github.com/twmb/murmur3 v1.1.8 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasttemplate v1.2.2 // indirect @@ -354,6 +384,7 @@ require ( go.dedis.ch/fixbuf v1.0.3 // indirect go.dedis.ch/kyber/v4 v4.0.0-pre2.0.20240924132404-4de33740016e // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 // indirect go.opentelemetry.io/otel v1.43.0 // indirect go.opentelemetry.io/otel/bridge/opencensus v1.40.0 // indirect go.opentelemetry.io/otel/exporters/jaeger v1.14.0 // indirect @@ -375,8 +406,6 @@ require ( golang.org/x/time v0.15.0 // indirect gonum.org/v1/gonum v0.17.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57 // indirect - google.golang.org/grpc v1.79.3 // indirect - google.golang.org/protobuf v1.36.11 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect howett.net/plist v1.0.1 // indirect diff --git a/go.sum b/go.sum index bccc57d8d..ea94f9307 100644 --- a/go.sum +++ b/go.sum @@ -38,12 +38,18 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= contrib.go.opencensus.io/exporter/prometheus v0.4.2 h1:sqfsYl5GIY/L570iT+l93ehxaWJs2/OwXtiWwew3oAg= contrib.go.opencensus.io/exporter/prometheus v0.4.2/go.mod h1:dvEHbiKmgvbr5pjaF9fpw1KeYcjrnC1J8B+JKjsZyRQ= +dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= +dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= filippo.io/bigmod v0.1.1-0.20260103110540-f8a47775ebe5 h1:JA0fFr+kxpqTdxR9LOBiTWpGNchqmkcsgmdeJZRclZ0= filippo.io/bigmod v0.1.1-0.20260103110540-f8a47775ebe5/go.mod h1:OjOXDNlClLblvXdwgFFOQFJEocLhhtai8vGLy0JCZlI= filippo.io/keygen v0.0.0-20260114151900-8e2790ea4c5b h1:REI1FbdW71yO56Are4XAxD+OS/e+BQsB3gE4mZRQEXY= filippo.io/keygen v0.0.0-20260114151900-8e2790ea4c5b/go.mod h1:9nnw1SlYHYuPSo/3wjQzNjSbeHlq2NsKo5iEtfJPWP0= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= +github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= +github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk= github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= @@ -103,8 +109,8 @@ github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8V github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b h1:mimo19zliBX/vSQ6PWWSL9lK8qwHozUj03+zLoEB8O0= github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b/go.mod h1:fvzegU4vN3H1qMT+8wDmzjAcDONcgo2/SZ/TyfdUOFs= github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= -github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eTWro= -github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= +github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ= +github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/ardanlabs/darwin/v2 v2.0.0 h1:XCisQMgQ5EG+ZvSEcADEo+pyfIMKyWAGnn5o2TgriYE= github.com/ardanlabs/darwin/v2 v2.0.0/go.mod h1:MubZ2e9DAYGaym0mClSOi183NYahrrfKxvSy1HMhoes= @@ -151,6 +157,8 @@ github.com/buger/jsonparser v1.1.2 h1:frqHqw7otoVbk5M8LlE/L7HTnIq2v9RX6EJ48i9AxJ github.com/buger/jsonparser v1.1.2/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/canonical/go-sp800.90a-drbg v0.0.0-20210314144037-6eeb1040d6c3 h1:oe6fCvaEpkhyW3qAicT0TnGtyht/UrgvOwMcEgLb7Aw= github.com/canonical/go-sp800.90a-drbg v0.0.0-20210314144037-6eeb1040d6c3/go.mod h1:qdP0gaj0QtgX2RUZhnlVrceJ+Qln8aSlDyJwelLLFeM= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -222,6 +230,14 @@ github.com/consensys/gnark-crypto v0.20.1/go.mod h1:RBWrSgy+IDbGR69RRV313th3M/aZ github.com/containerd/cgroups v0.0.0-20201119153540-4cbc285b3327/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE= github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw= +github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= +github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= +github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= +github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= +github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -231,6 +247,8 @@ github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV github.com/coreos/go-systemd/v22 v22.7.0 h1:LAEzFkke61DFROc7zNLX/WA2i5J8gYqe0rSj9KI28KA= github.com/coreos/go-systemd/v22 v22.7.0/go.mod h1:xNUYtjHu2EDXbsxz1i41wouACIwT7Ybq9o0BQhMwD0w= github.com/corpix/uarand v0.1.1/go.mod h1:SFKZvkcRoLqVRFZ4u25xPmp6m9ktANfbpXZ7SJ0/FNU= +github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA= +github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= @@ -241,6 +259,8 @@ github.com/crackcomm/go-gitignore v0.0.0-20241020182519-7843d2ba8fdf/go.mod h1:p github.com/crate-crypto/go-eth-kzg v1.5.0 h1:FYRiJMJG2iv+2Dy3fi14SVGjcPteZ5HAAUe4YWlJygc= github.com/crate-crypto/go-eth-kzg v1.5.0/go.mod h1:J9/u5sWfznSObptgfa92Jq8rTswn6ahQWEuiLHOjCUI= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s= +github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= github.com/cskr/pubsub v1.0.2 h1:vlOzMhl6PFn60gRlTQQsIfVwaPB/B/8MziK8FhEPt/0= github.com/cskr/pubsub v1.0.2/go.mod h1:/8MzYXk/NJAz782G8RPkFzXTZVu63VotefPnR9TIRis= github.com/curiostorage/go-openrpc-reflect v0.0.0-20260302234910-57f4605f0e7b h1:XRVqUVHJ/wuWHQDfuOm+M2ytKTPbQDtijapW1/xuYbc= @@ -278,6 +298,10 @@ github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUn github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94= +github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= @@ -294,6 +318,8 @@ github.com/dunglas/httpsfv v1.1.0/go.mod h1:zID2mqw9mFsnt7YC3vYQ9/cjq30q41W+1AnD github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/ebitengine/purego v0.10.0 h1:QIw4xfpWT6GWTzaW5XEKy3HXoqrJGx1ijYHzTF0/ISU= +github.com/ebitengine/purego v0.10.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= github.com/elastic/go-elasticsearch/v7 v7.17.10 h1:TCQ8i4PmIJuBunvBS6bwT2ybzVFxxUhhltAs3Gyu1yo= github.com/elastic/go-elasticsearch/v7 v7.17.10/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4= github.com/elastic/go-sysinfo v1.15.4 h1:A3zQcunCxik14MgXu39cXFXcIw2sFXZ0zL886eyiv1Q= @@ -1022,11 +1048,15 @@ github.com/libp2p/go-yamux/v5 v5.1.0/go.mod h1:tgIQ07ObtRR/I0IWsFOyQIL9/dR5UXgc2 github.com/lucasb-eyer/go-colorful v1.0.3/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/lucasb-eyer/go-colorful v1.4.0 h1:UtrWVfLdarDgc44HcS7pYloGHJUjHV/4FwW4TvVgFr4= github.com/lucasb-eyer/go-colorful v1.4.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/magefile/mage v1.9.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/magefile/mage v1.17.1 h1:F1d2lnLSlbQDM0Plq6Ac4NtaHxkxTK8t5nrMY9SkoNA= github.com/magefile/mage v1.17.1/go.mod h1:Yj51kqllmsgFpvvSzgrZPK9WtluG3kUhFaBUVLo4feA= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= +github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE= +github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= @@ -1086,6 +1116,24 @@ github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyua github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/pointerstructure v1.2.0 h1:O+i9nHnXS3l/9Wu7r4NrEdwA2VFTicjUEN1uBnDo34A= github.com/mitchellh/pointerstructure v1.2.0/go.mod h1:BRAsLI5zgXmw97Lf6s25bs8ohIXc3tViBH44KcwB2g4= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/go-archive v0.2.0 h1:zg5QDUM2mi0JIM9fdQZWC7U8+2ZfixfTYoHL7rWUcP8= +github.com/moby/go-archive v0.2.0/go.mod h1:mNeivT14o8xU+5q1YnNrkQVpK+dnNe/K6fHqnTg4qPU= +github.com/moby/moby/api v1.54.2 h1:wiat9QAhnDQjA7wk1kh/TqHz2I1uUA7M7t9SAl/JNXg= +github.com/moby/moby/api v1.54.2/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs= +github.com/moby/moby/client v0.4.0 h1:S+2XegzHQrrvTCvF6s5HFzcrywWQmuVnhOXe2kiWjIw= +github.com/moby/moby/client v0.4.0/go.mod h1:QWPbvWchQbxBNdaLSpoKpCdf5E+WxFAgNHogCWDoa7g= +github.com/moby/patternmatcher v0.6.1 h1:qlhtafmr6kgMIJjKJMDmMWq7WLkKIo23hsrpR3x084U= +github.com/moby/patternmatcher v0.6.1/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= +github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU= +github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko= +github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs= +github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= +github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= +github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= +github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ= +github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= @@ -1173,6 +1221,10 @@ github.com/onsi/gomega v1.36.3 h1:hID7cr8t3Wp26+cYnfcjR6HpJ00fdogN6dqZ1t6IylU= github.com/onsi/gomega v1.36.3/go.mod h1:8D9+Txp43QWKhM24yyOBEdpkzN8FvJyAwecBgsU4KU0= github.com/open-rpc/meta-schema v0.0.0-20250731032156-d8c12c08675d h1:ZNYTz9aScz13ejopZvqRsfgwOtU15OStJjBoNGTZnls= github.com/open-rpc/meta-schema v0.0.0-20250731032156-d8c12c08675d/go.mod h1:Ag6rSXkHIckQmjFBCweJEEt1mrTPBv8b9W4aU/NQWfI= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= +github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.2.1 h1:S4k4ryNgEpxW1dzyqffOmhI1BHYcjzU8lpJfSlR0xww= github.com/opencontainers/runtime-spec v1.2.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= @@ -1251,6 +1303,8 @@ github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a/go.mod h1:uIp+gprXx github.com/polydawn/refmt v0.89.1-0.20231129105047-37766d95467a h1:cgqrm0F3zwf9IPzca7xN4w+Zy6MC9ZkPvAC8QEWa/iQ= github.com/polydawn/refmt v0.89.1-0.20231129105047-37766d95467a/go.mod h1:ocZfO/tLSHqfScRDNTJbAJR1by4D1lewauX9OwTaPuY= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= @@ -1324,6 +1378,8 @@ github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg github.com/shirou/gopsutil v2.18.12+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shirou/gopsutil/v4 v4.26.3 h1:2ESdQt90yU3oXF/CdOlRCJxrP+Am1aBYubTMTfxJ1qc= +github.com/shirou/gopsutil/v4 v4.26.3/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ= github.com/shurcooL/go v0.0.0-20200502201357-93f07166e636/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= @@ -1399,6 +1455,10 @@ github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70 github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/test-go/testify v1.1.4 h1:Tf9lntrKUMHiXQ07qBScBTSA0dhYQlu83hswqelv1iE= github.com/test-go/testify v1.1.4/go.mod h1:rH7cfJo/47vWGdi4GPj16x3/t1xGOj2YxzmNQzk2ghU= +github.com/testcontainers/testcontainers-go v0.42.0 h1:He3IhTzTZOygSXLJPMX7n44XtK+qhjat1nI9cneBbUY= +github.com/testcontainers/testcontainers-go v0.42.0/go.mod h1:vZjdY1YmUA1qEForxOIOazfsrdyORJAbhi0bp8plN30= +github.com/testcontainers/testcontainers-go/modules/yugabytedb v0.42.0 h1:En9QqdG0snracnoY3VMzyDVYWOQtI5yQIeEdZMneXGo= +github.com/testcontainers/testcontainers-go/modules/yugabytedb v0.42.0/go.mod h1:QakiW/UG0atSB8O43wYpZy3EpyVN1H8yu3NvFSm00IU= github.com/tidwall/gjson v1.6.0/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls= github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM= github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= @@ -1406,10 +1466,10 @@ github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc= github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= -github.com/tklauser/go-sysconf v0.3.15 h1:VE89k0criAymJ/Os65CSn1IXaol+1wrsFHEB8Ol49K4= -github.com/tklauser/go-sysconf v0.3.15/go.mod h1:Dmjwr6tYFIseJw7a3dRLJfsHAMXZ3nEnL/aZY+0IuI4= -github.com/tklauser/numcpus v0.10.0 h1:18njr6LDBk1zuna922MgdjQuJFjrdppsZG60sHGfjso= -github.com/tklauser/numcpus v0.10.0/go.mod h1:BiTKazU708GQTYF4mB+cmlpT2Is1gLk7XVuEeem8LsQ= +github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYICU0nA= +github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI= +github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw= +github.com/tklauser/numcpus v0.11.0/go.mod h1:z+LwcLq54uWZTX0u/bGobaV34u6V7KNlTZejzM6/3MQ= github.com/triplewz/poseidon v0.0.2 h1:s5QMVYnUfqvgM1eIqp7O9hHjZLVrKnkhx0E7EQTf9Nk= github.com/triplewz/poseidon v0.0.2/go.mod h1:fmoxtMcbtMUjlSJmpuS3Wk/oKSvdJpIp9YWRbsOu3T0= github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= @@ -1797,6 +1857,7 @@ golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1810,6 +1871,7 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -2074,6 +2136,8 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= +gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -2085,6 +2149,8 @@ howett.net/plist v1.0.1 h1:37GdZ8tP09Q35o9ych3ehygcsL+HqKSwzctveSlarvM= howett.net/plist v1.0.1/go.mod h1:lqaXoTrLY4hg8tnEzNru53gicrbv7rrk+2xJA/7hw9g= lukechampine.com/blake3 v1.4.1 h1:I3Smz7gso8w4/TunLKec6K2fn+kyKtDxr/xcQEN84Wg= lukechampine.com/blake3 v1.4.1/go.mod h1:QFosUxmjB8mnrWFSNwKmvxHpfY72bmD2tQ0kBMM3kwo= +pgregory.net/rapid v1.2.0 h1:keKAYRcjm+e1F0oAuU5F5+YPAWcyxNNRK2wud503Gnk= +pgregory.net/rapid v1.2.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/harmony/harmonydb/downgrade/20260212-remoteseal-delegated.sql b/harmony/harmonydb/downgrade/20260212-remoteseal-delegated.sql new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/harmony/harmonydb/downgrade/20260212-remoteseal-delegated.sql @@ -0,0 +1 @@ + diff --git a/harmony/harmonydb/downgrade/20260216-rseal-allowed-proof-types.sql b/harmony/harmonydb/downgrade/20260216-rseal-allowed-proof-types.sql new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/harmony/harmonydb/downgrade/20260216-rseal-allowed-proof-types.sql @@ -0,0 +1 @@ + diff --git a/harmony/harmonydb/sql/20260212-remoteseal-delegated.sql b/harmony/harmonydb/sql/20260212-remoteseal-delegated.sql new file mode 100644 index 000000000..65f0ac922 --- /dev/null +++ b/harmony/harmonydb/sql/20260212-remoteseal-delegated.sql @@ -0,0 +1,191 @@ + +CREATE TABLE IF NOT EXISTS rseal_delegated_partners ( -- provider side + id BIGSERIAL PRIMARY KEY, + + partner_token TEXT NOT NULL UNIQUE, + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + + allowance_remaining BIGINT NOT NULL, + allowance_total BIGINT NOT NULL, + + partner_name TEXT NOT NULL, + partner_url TEXT NOT NULL +); + +-- rseal_client_providers tracks remote seal providers configured on the client side. +-- Tied to sp_id so that different miners on the same curio cluster can have different +-- provider configurations. A client curio may delegate sealing for multiple miners. +CREATE TABLE IF NOT EXISTS rseal_client_providers ( -- client side + id BIGSERIAL PRIMARY KEY, + + sp_id bigint not null, -- which miner this provider config is for + + provider_url text not null, -- base URL of the remote seal provider API + provider_token text not null, -- auth token for the provider + + provider_name text not null default '', + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + + enabled bool not null default true, + + UNIQUE (sp_id, provider_url) +); + +-- rseal_client_pipeline tracks sectors where SDR+trees are delegated to a remote +-- provider. The sectors_sdr_pipeline row does NOT exist during the remote seal +-- phase — it is created by ApplyRemoteCompletion when the provider finishes +-- SDR+trees and returns ticket/CIDs. +-- +-- Client-side stages: +-- 1. Delegate: send /order to provider (task_id_delegate) +-- 2. Poll: poll /status until complete or failed (task_id_poll) +-- 3. Fetch: download sealed file + cache from provider (task_id_fetch) +-- 4. [sectors_sdr_pipeline takes over: precommit, PoRep, finalize, move-storage, commit] +-- 5. Cleanup: tell provider to drop data (task_id_cleanup) +CREATE TABLE IF NOT EXISTS rseal_client_pipeline ( + sp_id bigint not null, + sector_number bigint not null, + + -- Provider reference + provider_id bigint not null references rseal_client_providers (id), + + -- at request time + create_time timestamptz not null default current_timestamp, + reg_seal_proof int not null, + user_sector_duration_epochs bigint, -- carried through to sectors_sdr_pipeline on completion + + -- Delegate: RSealDelegate sends /order to provider + task_id_delegate bigint, + after_delegate bool not null default false, -- order accepted by provider + + -- Poll: RSealClientPoll polls /status until complete/failed + task_id_poll bigint, + after_sdr bool not null default false, -- set when provider completes SDR+trees + + -- Data fetch: after remote SDR+trees complete, download sealed file (32 GiB) + -- and finalized cache (p_aux, t_aux, tree-r-last) from the provider. + -- Must complete before client precommit/PoRep can run. + task_id_fetch bigint, + after_fetch bool not null default false, + + -- Provider cleanup: after PoRep/finalize, request the provider to + -- release sealed sector data (layers, trees) on its side. + task_id_cleanup bigint, + after_cleanup bool not null default false, + + -- Failure handling + failed bool not null default false, + failed_at timestamptz, + failed_reason varchar(20) not null default '', + failed_reason_msg text not null default '', + + primary key (sp_id, sector_number) + -- No FK to sectors_sdr_pipeline: the sdr_pipeline row is created later + -- (when remote completion is applied) so it may not exist yet. +); + +-- rseal_provider_pipeline tracks sectors being sealed on behalf of a remote client. +-- sp_id/sector_number here is the CLIENT's miner identity - the provider seals under +-- the client's miner actor because ReplicaId is derived from (sp_id, sector_number, ticket). +-- These sectors are always CC (no deal data), so CommD is the static zero-commitment +-- for the sector size (derived from reg_seal_proof). +CREATE TABLE IF NOT EXISTS rseal_provider_pipeline ( + partner_id BIGINT NOT NULL REFERENCES rseal_delegated_partners (id), + + -- client's sp_id and sector_number - used for ReplicaId computation + sp_id bigint not null, + sector_number bigint not null, + + -- at request time + create_time timestamptz not null default current_timestamp, + reg_seal_proof int not null, + + -- sdr + ticket_epoch bigint, + ticket_value bytea, + + task_id_sdr bigint, + after_sdr bool not null default false, + + -- tree D + tree_d_cid text, -- commd from treeD compute, matches zero-comm for sector size + + task_id_tree_d bigint, + after_tree_d bool not null default false, + + -- tree C + task_id_tree_c bigint, + after_tree_c bool not null default false, + + -- tree R + tree_r_cid text, -- commr from treeR compute + + task_id_tree_r bigint, + after_tree_r bool not null default false, + + -- notify client that SDR+trees are done + task_id_notify_client bigint, + after_notify_client bool not null default false, + + -- C1: client supplies seed after precommit, provider computes C1 output + after_c1_supplied bool not null default false, + + -- finalize: after C1 is supplied and client confirms, provider can drop layers + task_id_finalize bigint, + after_finalize bool not null default false, + + -- cleanup: client requests cleanup or timeout triggers it + cleanup_requested bool not null default false, + cleanup_timeout timestamptz, -- non-graceful cleanup timeout, null until SDR+trees complete + + task_id_cleanup bigint, + after_cleanup bool not null default false, + + -- Failure handling + failed bool not null default false, + failed_at timestamptz, + failed_reason varchar(20) not null default '', + failed_reason_msg text not null default '', + + primary key (sp_id, sector_number) +); + +-- batch_sector_refs has a FK to sectors_sdr_pipeline, but SupraSeal batches can now +-- include remote sectors from rseal_provider_pipeline. Drop the FK and add a pipeline +-- source column so the slot manager knows which table to reference. +ALTER TABLE batch_sector_refs DROP CONSTRAINT IF EXISTS batch_sector_refs_sp_id_sector_number_fkey; -- PG naming +ALTER TABLE batch_sector_refs DROP CONSTRAINT IF EXISTS batch_sector_refs_sp_id_fkey; -- YugabyteDB naming +ALTER TABLE batch_sector_refs ADD COLUMN IF NOT EXISTS pipeline_source TEXT NOT NULL DEFAULT 'local'; +-- pipeline_source: 'local' = sectors_sdr_pipeline, 'remote' = rseal_provider_pipeline + +-- Cascade-delete triggers: when a pipeline row is deleted, clean up its batch_sector_refs. +-- This replaces the dropped FK with conditional logic based on pipeline_source. + +CREATE OR REPLACE FUNCTION cascade_delete_batch_refs_local() RETURNS TRIGGER AS $$ +BEGIN + DELETE FROM batch_sector_refs + WHERE sp_id = OLD.sp_id AND sector_number = OLD.sector_number AND pipeline_source = 'local'; + RETURN OLD; +END; +$$ LANGUAGE plpgsql; + +DROP TRIGGER IF EXISTS trg_cascade_batch_refs_local ON sectors_sdr_pipeline; +CREATE TRIGGER trg_cascade_batch_refs_local + BEFORE DELETE ON sectors_sdr_pipeline + FOR EACH ROW EXECUTE FUNCTION cascade_delete_batch_refs_local(); + +CREATE OR REPLACE FUNCTION cascade_delete_batch_refs_remote() RETURNS TRIGGER AS $$ +BEGIN + DELETE FROM batch_sector_refs + WHERE sp_id = OLD.sp_id AND sector_number = OLD.sector_number AND pipeline_source = 'remote'; + RETURN OLD; +END; +$$ LANGUAGE plpgsql; + +DROP TRIGGER IF EXISTS trg_cascade_batch_refs_remote ON rseal_provider_pipeline; +CREATE TRIGGER trg_cascade_batch_refs_remote + BEFORE DELETE ON rseal_provider_pipeline + FOR EACH ROW EXECUTE FUNCTION cascade_delete_batch_refs_remote(); + diff --git a/harmony/harmonydb/sql/20260216-rseal-allowed-proof-types.sql b/harmony/harmonydb/sql/20260216-rseal-allowed-proof-types.sql new file mode 100644 index 000000000..ff5d16e76 --- /dev/null +++ b/harmony/harmonydb/sql/20260216-rseal-allowed-proof-types.sql @@ -0,0 +1,5 @@ + +-- Add allowed_proof_types to rseal_delegated_partners so the provider can +-- restrict which seal proof types each partner is allowed to submit. +-- Empty array means "all proofs accepted" (backward-compatible default). +ALTER TABLE rseal_delegated_partners ADD COLUMN IF NOT EXISTS allowed_proof_types bigint[] NOT NULL DEFAULT '{}'; diff --git a/harmony/harmonydb/sql_test.go b/harmony/harmonydb/sql_test.go new file mode 100644 index 000000000..afcd33b07 --- /dev/null +++ b/harmony/harmonydb/sql_test.go @@ -0,0 +1,35 @@ +package harmonydb + +import ( + "strings" + "testing" +) + +// TestNoDuplicateMigrationDatePrefixes verifies that no two SQL migration files +// share the same YYYYMMDD date prefix. Migrations are applied in lexicographic +// order, so duplicate prefixes create ambiguous ordering. +func TestNoDuplicateMigrationDatePrefixes(t *testing.T) { + entries, err := upgradeFS.ReadDir("sql") + if err != nil { + t.Fatalf("reading embedded sql directory: %v", err) + } + + seen := make(map[string]string) // date prefix → first filename + for _, e := range entries { + if e.IsDir() { + continue + } + name := e.Name() + // Date prefix is everything before the first '-'. + prefix, _, ok := strings.Cut(name, "-") + if !ok { + continue + } + + if prev, exists := seen[prefix]; exists { + t.Errorf("duplicate date prefix %q:\n %s\n %s", prefix, prev, name) + } else { + seen[prefix] = name + } + } +} diff --git a/itests/helpers/env.go b/itests/helpers/env.go index 5d32c152f..c71586a6b 100644 --- a/itests/helpers/env.go +++ b/itests/helpers/env.go @@ -4,6 +4,7 @@ import ( "context" "net" "os" + "strconv" "testing" "github.com/stretchr/testify/require" @@ -26,9 +27,22 @@ func FreeListenAddr(t *testing.T) string { return ln.Addr().String() } +// YBCQLPort returns the YCQL port for test connections. It reads +// CURIO_HARMONYDB_CQL_PORT (set by testcontainers with dynamic port mapping) +// and falls back to the default 9042. +func YBCQLPort() int { + if v := os.Getenv("CURIO_HARMONYDB_CQL_PORT"); v != "" { + p, err := strconv.Atoi(v) + if err == nil { + return p + } + } + return 9042 +} + func NewIndexStore(ctx context.Context, t *testing.T, cfg *config.CurioConfig) *indexstore.IndexStore { hosts := []string{EnvElse("CURIO_HARMONYDB_HOSTS", "127.0.0.1")} - idxStore, err := indexstore.NewIndexStore(hosts, 9042, cfg) + idxStore, err := indexstore.NewIndexStore(hosts, YBCQLPort(), cfg) require.NoError(t, err) err = idxStore.Start(ctx, true) require.NoError(t, err) diff --git a/itests/remoteseal_test.go b/itests/remoteseal_test.go new file mode 100644 index 000000000..c06dc010d --- /dev/null +++ b/itests/remoteseal_test.go @@ -0,0 +1,324 @@ +package itests + +import ( + "context" + "crypto/rand" + "database/sql" + "encoding/hex" + "fmt" + "os" + "testing" + "time" + + "github.com/docker/go-units" + logging "github.com/ipfs/go-log/v2" + "github.com/stretchr/testify/require" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + + "github.com/filecoin-project/curio/deps" + "github.com/filecoin-project/curio/deps/config" + "github.com/filecoin-project/curio/harmony/harmonydb" + "github.com/filecoin-project/curio/itests/helpers" + "github.com/filecoin-project/curio/tasks/seal" + + lapi "github.com/filecoin-project/lotus/api" + miner2 "github.com/filecoin-project/lotus/chain/actors/builtin/miner" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/cli/spcli/createminer" + "github.com/filecoin-project/lotus/itests/kit" +) + +func TestRemoteSealHappyPath(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + _ = logging.SetLogLevel("*", "INFO") + _ = logging.SetLogLevel("harmonytask", "DEBUG") + _ = logging.SetLogLevel("cu/seal", "DEBUG") + _ = logging.SetLogLevel("cu-http", "DEBUG") + _ = logging.SetLogLevel("sealmarket", "DEBUG") + _ = logging.SetLogLevel("remoteseal", "DEBUG") + + full, miner, ensemble := kit.EnsembleMinimal(t, + kit.LatestActorsAt(-1), + kit.PresealSectors(32), + kit.ThroughRPC(), + ) + ensemble.Start() + blockTime := 100 * time.Millisecond + ensemble.BeginMining(blockTime) + + full.WaitTillChain(ctx, kit.HeightAtLeast(15)) + + _ = miner.LogSetLevel(ctx, "*", "ERROR") + _ = full.LogSetLevel(ctx, "*", "ERROR") + + token, err := full.AuthNew(ctx, lapi.AllPermissions) + require.NoError(t, err) + fapi := fmt.Sprintf("%s:%s", string(token), full.ListenAddr) + + sharedITestID := harmonydb.ITestNewID() + t.Logf("sharedITestID: %s", sharedITestID) + + db, err := harmonydb.NewFromConfigWithITestID(t, harmonydb.ItestID(sharedITestID), harmonydb.YugabyteDB(true)) + require.NoError(t, err) + defer db.ITestDeleteAll() + + idxStore := helpers.NewIndexStore(ctx, t, config.DefaultCurioConfig()) + + // Create miner + addr := miner.OwnerKey.Address + sectorSizeInt, err := units.RAMInBytes("2KiB") + require.NoError(t, err) + maddr, err := createminer.CreateStorageMiner(ctx, full, addr, addr, addr, abi.SectorSize(sectorSizeInt), 0, 1.0) + require.NoError(t, err) + + err = deps.CreateMinerConfig(ctx, full, db, []string{maddr.String()}, fapi) + require.NoError(t, err) + + // Load base config from DB (has miner identity, API secrets, etc.) + baseCfg := config.DefaultCurioConfig() + var baseText string + err = db.QueryRow(ctx, "SELECT config FROM harmony_config WHERE title='base'").Scan(&baseText) + require.NoError(t, err) + _, err = deps.LoadConfigWithUpgrades(baseText, baseCfg) + require.NoError(t, err) + + baseCfg.Batching.PreCommit.Timeout = time.Second + baseCfg.Batching.Commit.Timeout = time.Second + + // Provider config: SDR + Trees + Remote Seal Provider + HTTP server + // No DealMarket needed - the HTTP server only needs sealmarket routes. + providerCfg := *baseCfg + providerCfg.Subsystems.EnableSealSDR = true + providerCfg.Subsystems.EnableSealSDRTrees = true + providerCfg.Subsystems.EnableRemoteSealProvider = true + providerCfg.HTTP.Enable = true + providerCfg.HTTP.DelegateTLS = true // plain HTTP for test (no Let's Encrypt) + providerCfg.HTTP.ListenAddress = "127.0.0.1:0" // OS assigns random port + + // Client config: Remote Seal Client + PoRep + commit flow + HTTP server + clientCfg := *baseCfg + clientCfg.Subsystems.EnableRemoteSealClient = true + clientCfg.Subsystems.EnablePoRepProof = true + clientCfg.Subsystems.EnableSendPrecommitMsg = true + clientCfg.Subsystems.EnableSendCommitMsg = true + clientCfg.Subsystems.EnableMoveStorage = true + clientCfg.HTTP.Enable = true + clientCfg.HTTP.DelegateTLS = true + clientCfg.HTTP.ListenAddress = "127.0.0.1:0" + + // Create temp dirs for provider and client + providerDir, err := os.MkdirTemp("", "curio-provider-*") + require.NoError(t, err) + defer func() { _ = os.RemoveAll(providerDir) }() + + clientDir, err := os.MkdirTemp("", "curio-client-*") + require.NoError(t, err) + defer func() { _ = os.RemoveAll(clientDir) }() + + // Start provider instance first so we can discover its HTTP address. + t.Log("Starting provider instance...") + _, providerTerm, providerCloser, _, providerDeps := ConstructCurioTest(ctx, t, providerDir, db, idxStore, full, maddr, &providerCfg) + defer providerTerm() + defer providerCloser() + + providerHTTPAddr := providerDeps.HTTPListenAddr + require.NotEmpty(t, providerHTTPAddr, "provider HTTP server should have started") + t.Logf("Provider HTTP address: %s", providerHTTPAddr) + + // Start client instance. + t.Log("Starting client instance...") + _, clientTerm, clientCloser, _, clientDeps := ConstructCurioTest(ctx, t, clientDir, db, idxStore, full, maddr, &clientCfg) + defer clientTerm() + defer clientCloser() + + clientHTTPAddr := clientDeps.HTTPListenAddr + require.NotEmpty(t, clientHTTPAddr, "client HTTP server should have started") + t.Logf("Client HTTP address: %s", clientHTTPAddr) + + // Wait for both instances to settle (register with harmony_machines). + time.Sleep(3 * time.Second) + + // Generate a shared auth token for the partner/provider relationship. + tokenBytes := make([]byte, 32) + _, err = rand.Read(tokenBytes) + require.NoError(t, err) + testToken := hex.EncodeToString(tokenBytes) + + // Insert partner entry on the provider side. + // partner_url points to the CLIENT's HTTP address (provider calls client for /complete notification). + var partnerID int64 + clientURL := fmt.Sprintf("http://%s", clientHTTPAddr) + err = db.QueryRow(ctx, `INSERT INTO rseal_delegated_partners (partner_name, partner_url, partner_token, allowance_remaining, allowance_total) + VALUES ($1, $2, $3, $4, $4) RETURNING id`, + "test-client", clientURL, testToken, int64(100)).Scan(&partnerID) + require.NoError(t, err) + t.Logf("Created partner ID: %d, partner_url: %s", partnerID, clientURL) + + // Insert provider entry on the client side. + // provider_url points to the PROVIDER's HTTP address (client calls provider for status/fetch/c1/cleanup). + mid, err := address.IDFromAddress(maddr) + require.NoError(t, err) + + providerURL := fmt.Sprintf("http://%s", providerHTTPAddr) + var providerID int64 + err = db.QueryRow(ctx, `INSERT INTO rseal_client_providers (sp_id, provider_url, provider_token, provider_name) + VALUES ($1, $2, $3, $4) RETURNING id`, + int64(mid), providerURL, testToken, "test-provider").Scan(&providerID) + require.NoError(t, err) + t.Logf("Created provider ID: %d, provider_url: %s", providerID, providerURL) + + // Get seal proof type + mi, err := full.StateMinerInfo(ctx, maddr, types.EmptyTSK) + require.NoError(t, err) + nv, err := full.StateNetworkVersion(ctx, types.EmptyTSK) + require.NoError(t, err) + wpt := mi.WindowPoStProofType + spt, err := miner2.PreferredSealProofTypeFromWindowPoStType(nv, wpt, true) + require.NoError(t, err) + + // Allocate a sector and insert into the remote seal pipelines. + // In the real flow, RSealDelegate does this after calling /available + /order. + // For the test we manually insert to skip the delegation HTTP handshake and + // directly test the sealing pipeline. + // + // NOTE: sectors_sdr_pipeline is NOT created here — it will be created by + // ApplyRemoteCompletion when the provider finishes SDR+trees and the + // client receives the /complete notification (or polls /status). + comm, err := db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) { + nums, err := seal.AllocateSectorNumbers(ctx, full, tx, maddr, 1) + if err != nil { + return false, err + } + require.Len(t, nums, 1) + + sectorNum := nums[0] + t.Logf("Allocated sector number: %d", sectorNum) + + // Insert into rseal_client_pipeline (marks sector as remotely sealed) + _, err = tx.Exec(`INSERT INTO rseal_client_pipeline (sp_id, sector_number, provider_id, reg_seal_proof) + VALUES ($1, $2, $3, $4)`, + int64(mid), sectorNum, providerID, spt) + if err != nil { + return false, xerrors.Errorf("inserting into rseal_client_pipeline: %w", err) + } + + // Insert into rseal_provider_pipeline (provider side picks this up) + _, err = tx.Exec(`INSERT INTO rseal_provider_pipeline (partner_id, sp_id, sector_number, reg_seal_proof) + VALUES ($1, $2, $3, $4)`, + partnerID, int64(mid), sectorNum, spt) + if err != nil { + return false, xerrors.Errorf("inserting into rseal_provider_pipeline: %w", err) + } + + return true, nil + }) + require.NoError(t, err) + require.True(t, comm) + + t.Log("Sector pipeline entries created, waiting for sealing to complete...") + + // Poll for completion of the full pipeline. + var pollTask []struct { + SpID int64 `db:"sp_id"` + SectorNumber int64 `db:"sector_number"` + AfterSDR bool `db:"after_sdr"` + AfterTreeD bool `db:"after_tree_d"` + AfterTreeC bool `db:"after_tree_c"` + AfterTreeR bool `db:"after_tree_r"` + AfterSynth bool `db:"after_synth"` + AfterPrecommitMsg bool `db:"after_precommit_msg"` + AfterPrecommitMsgSuccess bool `db:"after_precommit_msg_success"` + AfterPoRep bool `db:"after_porep"` + AfterFinalize bool `db:"after_finalize"` + AfterMoveStorage bool `db:"after_move_storage"` + AfterCommitMsg bool `db:"after_commit_msg"` + AfterCommitMsgSuccess bool `db:"after_commit_msg_success"` + Failed bool `db:"failed"` + FailedReason string `db:"failed_reason"` + StartEpoch sql.NullInt64 `db:"start_epoch"` + } + + require.Eventuallyf(t, func() bool { + h, err := full.ChainHead(ctx) + require.NoError(t, err) + t.Logf("head: %d", h.Height()) + + err = db.Select(ctx, &pollTask, `SELECT sp_id, sector_number, + after_sdr, after_tree_d, after_tree_c, after_tree_r, after_synth, + after_precommit_msg, after_precommit_msg_success, + after_porep, after_finalize, after_move_storage, + after_commit_msg, after_commit_msg_success, + failed, failed_reason, start_epoch + FROM sectors_sdr_pipeline WHERE sp_id = $1`, int64(mid)) + require.NoError(t, err) + + for i, task := range pollTask { + t.Logf("Task %d: sp=%d sector=%d sdr=%t treeD=%t treeC=%t treeR=%t synth=%t precommit=%t precommitOK=%t porep=%t finalize=%t move=%t commit=%t commitOK=%t failed=%t reason=%s", + i, task.SpID, task.SectorNumber, + task.AfterSDR, task.AfterTreeD, task.AfterTreeC, task.AfterTreeR, task.AfterSynth, + task.AfterPrecommitMsg, task.AfterPrecommitMsgSuccess, + task.AfterPoRep, task.AfterFinalize, task.AfterMoveStorage, + task.AfterCommitMsg, task.AfterCommitMsgSuccess, + task.Failed, task.FailedReason) + } + + // Log remote seal pipeline status for debugging + var provPipeline []struct { + SpID int64 `db:"sp_id"` + SectorNumber int64 `db:"sector_number"` + AfterSDR bool `db:"after_sdr"` + AfterTreeD bool `db:"after_tree_d"` + AfterTreeR bool `db:"after_tree_r"` + AfterNotify bool `db:"after_notify_client"` + AfterC1 bool `db:"after_c1_supplied"` + AfterFinalize bool `db:"after_finalize"` + AfterCleanup bool `db:"after_cleanup"` + Failed bool `db:"failed"` + FailedMsg string `db:"failed_reason_msg"` + } + _ = db.Select(ctx, &provPipeline, `SELECT sp_id, sector_number, after_sdr, after_tree_d, after_tree_r, after_notify_client, after_c1_supplied, after_finalize, after_cleanup, failed, failed_reason_msg FROM rseal_provider_pipeline`) + for _, pp := range provPipeline { + t.Logf("ProvPipeline: sp=%d sector=%d sdr=%t treeD=%t treeR=%t notify=%t c1=%t finalize=%t cleanup=%t failed=%t msg=%s", + pp.SpID, pp.SectorNumber, pp.AfterSDR, pp.AfterTreeD, pp.AfterTreeR, pp.AfterNotify, pp.AfterC1, pp.AfterFinalize, pp.AfterCleanup, pp.Failed, pp.FailedMsg) + } + + var clientPipeline []struct { + SpID int64 `db:"sp_id"` + SectorNumber int64 `db:"sector_number"` + AfterSDR bool `db:"after_sdr"` + AfterFetch bool `db:"after_fetch"` + AfterCleanup bool `db:"after_cleanup"` + Failed bool `db:"failed"` + FailedMsg string `db:"failed_reason_msg"` + } + _ = db.Select(ctx, &clientPipeline, `SELECT sp_id, sector_number, after_sdr, after_fetch, after_cleanup, failed, failed_reason_msg FROM rseal_client_pipeline`) + for _, cp := range clientPipeline { + t.Logf("ClientPipeline: sp=%d sector=%d sdr=%t fetch=%t cleanup=%t failed=%t msg=%s", + cp.SpID, cp.SectorNumber, cp.AfterSDR, cp.AfterFetch, cp.AfterCleanup, cp.Failed, cp.FailedMsg) + } + + if len(pollTask) == 0 { + return false + } + + // Check if the sector completed the full pipeline + for _, task := range pollTask { + if task.Failed { + t.Errorf("sector %d failed: %s", task.SectorNumber, task.FailedReason) + return false + } + if !task.AfterCommitMsgSuccess { + return false + } + } + return true + }, 15*time.Minute, 2*time.Second, "remote seal pipeline did not complete in 15 minutes") + + t.Log("Remote seal pipeline completed successfully!") + // Cleanup is handled by defers: providerTerm/clientTerm + providerCloser/clientCloser +} diff --git a/itests/testmain_test.go b/itests/testmain_test.go new file mode 100644 index 000000000..ebee364ac --- /dev/null +++ b/itests/testmain_test.go @@ -0,0 +1,12 @@ +package itests + +import ( + "os" + "testing" + + "github.com/filecoin-project/curio/lib/testutil/dbtest" +) + +func TestMain(m *testing.M) { + os.Exit(dbtest.StartYugabyte(m)) +} diff --git a/lib/ffi/remote_fetch.go b/lib/ffi/remote_fetch.go new file mode 100644 index 000000000..359f73ab0 --- /dev/null +++ b/lib/ffi/remote_fetch.go @@ -0,0 +1,297 @@ +package ffi + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "os/exec" + "path/filepath" + "strconv" + + "golang.org/x/xerrors" + + "github.com/filecoin-project/curio/harmony/harmonytask" + "github.com/filecoin-project/curio/lib/paths" + "github.com/filecoin-project/curio/lib/storiface" + "github.com/filecoin-project/curio/lib/tarutil" +) + +// RemoteSealFetchParams describes everything needed to download sealed sector +// data and cache from a remote seal provider. +type RemoteSealFetchParams struct { + // Full URL to download the sealed sector file (GET, supports Range). + SealedURL string + + // Full URL to download the cache tar (GET, returns tar stream). + CacheURL string + + // C1 metadata to write into the cache directory for PoRep. + C1Info paths.RemoteSealC1Info + + SpID int64 + SectorNumber int64 +} + +// DownloadRemoteSealData acquires storage for a sector's sealed file and cache, +// downloads them from the remote seal provider, writes the c1.url metadata file, +// and ensures only one copy of the data exists in the storage system. +func (sb *SealCalls) DownloadRemoteSealData(ctx context.Context, task *harmonytask.TaskID, sector storiface.SectorRef, params RemoteSealFetchParams) error { + fspaths, pathIDs, release, err := sb.Sectors.AcquireSector(ctx, task, sector, storiface.FTNone, storiface.FTSealed|storiface.FTCache, storiface.PathStorage) + if err != nil { + return xerrors.Errorf("acquiring sector storage: %w", err) + } + defer release() + + if fspaths.Sealed == "" { + return xerrors.Errorf("no sealed path allocated") + } + if fspaths.Cache == "" { + return xerrors.Errorf("no cache path allocated") + } + + // Download sealed sector file (32 GiB) + log.Infow("downloading sealed file from provider", + "sp_id", params.SpID, "sector", params.SectorNumber, + "sealed_path", fspaths.Sealed) + + if err := fetchFile(ctx, fspaths.Sealed, params.SealedURL, params.SpID, params.SectorNumber); err != nil { + return xerrors.Errorf("fetching sealed data: %w", err) + } + + // Download and extract cache tar (p_aux, t_aux, tree-r-last-*) + log.Infow("downloading cache data from provider", + "sp_id", params.SpID, "sector", params.SectorNumber, + "cache_path", fspaths.Cache) + + if err := fetchCacheTar(ctx, fspaths.Cache, params.CacheURL); err != nil { + return xerrors.Errorf("fetching cache data: %w", err) + } + + // Write c1.url file so PoRep can fetch C1 output from the remote provider + c1InfoJSON, err := json.Marshal(params.C1Info) + if err != nil { + return xerrors.Errorf("marshaling c1 info: %w", err) + } + if err := os.WriteFile(filepath.Join(fspaths.Cache, paths.RemoteSealC1UrlFile), c1InfoJSON, 0644); err != nil { + return xerrors.Errorf("writing c1.url file: %w", err) + } + + if err := sb.ensureOneCopy(ctx, sector.ID, pathIDs, storiface.FTSealed|storiface.FTCache); err != nil { + return xerrors.Errorf("ensure one copy: %w", err) + } + + return nil +} + +// fetchFile downloads a file to destPath. If aria2c is installed it is used +// exclusively — we never fall back to Go HTTP after an aria2c failure because +// aria2c may leave a pre-allocated sparse file that would fool the Go HTTP +// resume logic into thinking the download is complete (the file stats as full +// size but contains null-byte holes for unfetched chunks). If aria2c is not +// installed at all, Go HTTP is used as the sole downloader. +func fetchFile(ctx context.Context, destPath, dlURL string, spID, sectorNumber int64) error { + if _, err := exec.LookPath("aria2c"); err == nil { + return fetchWithAria2c(ctx, destPath, dlURL) + } + + log.Warnw("aria2c not found in PATH, using Go HTTP downloader", + "sp_id", spID, "sector", sectorNumber) + return fetchWithGoHTTP(ctx, destPath, dlURL) +} + +// fetchCacheTar downloads a cache tar stream and extracts it to cachePath. +// Extracts to cachePath + ".tmp" first, then renames to cachePath on success. +// Cleans up the temp directory on any error. +func fetchCacheTar(ctx context.Context, cachePath, dlURL string) (err error) { + tmpPath := cachePath + ".tmp" + + // Clean up temp directory on error + defer func() { + if err != nil { + if rmErr := os.RemoveAll(tmpPath); rmErr != nil { + log.Warnw("failed to clean up temp cache dir after error", "path", tmpPath, "error", rmErr) + } + } + }() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, dlURL, nil) + if err != nil { + return xerrors.Errorf("creating request: %w", err) + } + + dlClient := &http.Client{} + resp, err := dlClient.Do(req) + if err != nil { + return xerrors.Errorf("performing request to %s: %w", dlURL, err) + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return xerrors.Errorf("unexpected status %d from %s: %s", resp.StatusCode, dlURL, string(body)) + } + + buf := make([]byte, 1<<20) // 1 MiB buffer + _, err = tarutil.ExtractTar(tarutil.FinCacheFileConstraints, resp.Body, tmpPath, buf) + if err != nil { + return xerrors.Errorf("extracting cache tar to %s: %w", tmpPath, err) + } + + // Remove the cachePath if it exists (should be empty, created by AcquireSector) + // and rename .tmp to final destination + if err := os.RemoveAll(cachePath); err != nil { + return xerrors.Errorf("removing existing cache path %s: %w", cachePath, err) + } + + if err := os.Rename(tmpPath, cachePath); err != nil { + return xerrors.Errorf("renaming temp cache dir to final destination: %w", err) + } + + return nil +} + +// fetchWithAria2c invokes aria2c as a subprocess for multi-connection resumable downloads. +// Downloads to destPath + ".tmp" first, then renames to destPath on success. +// Cleans up temp files and aria2 control files on any error. +func fetchWithAria2c(ctx context.Context, destPath, dlURL string) (err error) { + aria2cPath, err := exec.LookPath("aria2c") + if err != nil { + return xerrors.New("aria2c not found in PATH") + } + + tmpPath := destPath + ".tmp" + aria2ControlPath := tmpPath + ".aria2" + + // Clean up temp files on error + defer func() { + if err != nil { + if rmErr := os.Remove(tmpPath); rmErr != nil && !os.IsNotExist(rmErr) { + log.Warnw("failed to clean up temp file after error", "path", tmpPath, "error", rmErr) + } + if rmErr := os.Remove(aria2ControlPath); rmErr != nil && !os.IsNotExist(rmErr) { + log.Warnw("failed to clean up aria2 control file after error", "path", aria2ControlPath, "error", rmErr) + } + } + }() + + cmd := exec.CommandContext(ctx, aria2cPath, + "--file-allocation=none", // prevent sparse pre-allocation that creates full-size files with null holes + "--lowest-speed-limit=4K", + "--timeout=120", + "--connect-timeout=60", + "--max-tries=0", // infinite retries within the context deadline + "--retry-wait=30", + "--continue=true", + "-x16", + "-s16", + "--auto-file-renaming=false", // don't create .1, .2 copies + "--allow-overwrite=true", + "--dir", filepath.Dir(tmpPath), + "-o", filepath.Base(tmpPath), + dlURL) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + if err := cmd.Run(); err != nil { + return xerrors.Errorf("aria2c failed: %w", err) + } + + // Rename .tmp to final destination only after successful download + if err := os.Rename(tmpPath, destPath); err != nil { + return xerrors.Errorf("renaming temp file to final destination: %w", err) + } + + return nil +} + +// fetchWithGoHTTP downloads a file using a plain Go HTTP client with Range header +// support for resuming partial downloads. Downloads to destPath + ".tmp" first, +// then renames to destPath on success. Cleans up the temp file on any error. +func fetchWithGoHTTP(ctx context.Context, destPath, dlURL string) (err error) { + tmpPath := destPath + ".tmp" + + // Clean up temp file on error + defer func() { + if err != nil { + if rmErr := os.Remove(tmpPath); rmErr != nil && !os.IsNotExist(rmErr) { + log.Warnw("failed to clean up temp file after error", "path", tmpPath, "error", rmErr) + } + } + }() + + f, err := os.OpenFile(tmpPath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) + if err != nil { + return xerrors.Errorf("opening temp file %s: %w", tmpPath, err) + } + defer func() { _ = f.Close() }() + + fStat, err := f.Stat() + if err != nil { + return xerrors.Errorf("stat temp file %s: %w", tmpPath, err) + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, dlURL, nil) + if err != nil { + return xerrors.Errorf("creating request: %w", err) + } + + if fStat.Size() > 0 { + req.Header.Set("Range", fmt.Sprintf("bytes=%s-", strconv.FormatInt(fStat.Size(), 10))) + } + + dlClient := &http.Client{} + resp, err := dlClient.Do(req) + if err != nil { + return xerrors.Errorf("performing request: %w", err) + } + defer func() { _ = resp.Body.Close() }() + + switch resp.StatusCode { + case http.StatusOK: + if fStat.Size() > 0 { + if err := f.Truncate(0); err != nil { + return xerrors.Errorf("truncating temp file for full rewrite: %w", err) + } + if _, err := f.Seek(0, io.SeekStart); err != nil { + return xerrors.Errorf("seeking to start: %w", err) + } + } + case http.StatusPartialContent: + // Server is sending the remaining bytes from our Range offset. + case http.StatusRequestedRangeNotSatisfiable: + // File is already complete. + // Close the temp file before renaming + if err := f.Close(); err != nil { + return xerrors.Errorf("closing temp file: %w", err) + } + // Rename .tmp to final destination + if err := os.Rename(tmpPath, destPath); err != nil { + return xerrors.Errorf("renaming temp file to final destination: %w", err) + } + return nil + default: + body, _ := io.ReadAll(resp.Body) + return xerrors.Errorf("unexpected status %d: %s", resp.StatusCode, string(body)) + } + + buf := make([]byte, 1<<20) // 1 MiB buffer + _, err = io.CopyBuffer(f, resp.Body, buf) + if err != nil { + return xerrors.Errorf("writing data to %s: %w", tmpPath, err) + } + + // Close the file before renaming + if err := f.Close(); err != nil { + return xerrors.Errorf("closing temp file: %w", err) + } + + // Rename .tmp to final destination only after successful download + if err := os.Rename(tmpPath, destPath); err != nil { + return xerrors.Errorf("renaming temp file to final destination: %w", err) + } + + return nil +} diff --git a/lib/ffi/sdr_funcs.go b/lib/ffi/sdr_funcs.go index 26b370098..fbe1b29f0 100644 --- a/lib/ffi/sdr_funcs.go +++ b/lib/ffi/sdr_funcs.go @@ -357,6 +357,13 @@ func (sb *SealCalls) GenerateSynthPoRep() { panic("todo") } +// GeneratePoRepVanillaProof generates a vanilla proof for a sector (C1 output). +// This is the first phase of SealCommit and produces the vanilla proofs that +// are later used in SealCommitPhase2 (C2) to produce the SNARK proof. +func (sb *SealCalls) GeneratePoRepVanillaProof(ctx context.Context, sn storiface.SectorRef, sealed, unsealed cid.Cid, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness) ([]byte, error) { + return sb.Sectors.storage.GeneratePoRepVanillaProof(ctx, sn, sealed, unsealed, ticket, seed) +} + func (sb *SealCalls) PoRepSnark(ctx context.Context, sn storiface.SectorRef, sealed, unsealed cid.Cid, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness) ([]byte, error) { vproof, err := sb.Sectors.storage.GeneratePoRepVanillaProof(ctx, sn, sealed, unsealed, ticket, seed) if err != nil { @@ -796,6 +803,36 @@ func (sb *SealCalls) TreeD(ctx context.Context, sector storiface.SectorRef, unse return nil } +// EnsureSyntheticProofs generates synthetic PoRep vanilla proofs for a sector +// if they don't already exist. Unlike SyntheticProofs, this does NOT clear the +// cache or generate the unsealed copy — it only creates the syn-porep-vanilla-proofs.dat +// file needed by SealCommitPhase1. +// +// This is used by the remote seal provider's C1 handler: the provider runs +// SDR+trees but not the normal Synth task (which also clears layers). The provider +// must keep layers until the client finishes the pipeline, but still needs synthetic +// proofs to serve C1 requests. +func (sb *SealCalls) EnsureSyntheticProofs(ctx context.Context, sector storiface.SectorRef, sealed cid.Cid, unsealed cid.Cid, randomness abi.SealRandomness, pieces []abi.PieceInfo) error { + fspaths, _, releaseSector, err := sb.Sectors.AcquireSector(ctx, nil, sector, storiface.FTCache|storiface.FTSealed, storiface.FTNone, storiface.PathStorage) + if err != nil { + return xerrors.Errorf("acquiring sector paths: %w", err) + } + defer releaseSector() + + // Check if synthetic proofs already exist (idempotent) + synthPath := filepath.Join(fspaths.Cache, "syn-porep-vanilla-proofs.dat") + if _, err := os.Stat(synthPath); err == nil { + return nil // already generated + } + + err = ffi.GenerateSynthProofs(sector.ProofType, sealed, unsealed, fspaths.Cache, fspaths.Sealed, sector.ID.Number, sector.ID.Miner, randomness, pieces) + if err != nil { + return xerrors.Errorf("generating synthetic proofs: %w", err) + } + + return nil +} + func (sb *SealCalls) SyntheticProofs(ctx context.Context, task *harmonytask.TaskID, sector storiface.SectorRef, sealed cid.Cid, unsealed cid.Cid, randomness abi.SealRandomness, pieces []abi.PieceInfo, keepUnsealed bool) error { fspaths, pathIDs, releaseSector, err := sb.Sectors.AcquireSector(ctx, task, sector, storiface.FTCache|storiface.FTSealed, storiface.FTNone, storiface.PathSealing) if err != nil { diff --git a/lib/ffi/task_storage.go b/lib/ffi/task_storage.go index 31450bc5c..14aeae048 100644 --- a/lib/ffi/task_storage.go +++ b/lib/ffi/task_storage.go @@ -62,6 +62,10 @@ type StorageReservation struct { } func (sb *SealCalls) Storage(taskToSectorRef func(taskID harmonytask.TaskID) (SectorRef, error), alloc, existing storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType, MinFreeStoragePercentage float64) *TaskStorage { + oh := storiface.FSOverheadSeal + if pathType == storiface.PathStorage { + oh = storiface.FsOverheadFinalized + } return sb.StorageMulti(func(taskID harmonytask.TaskID) ([]SectorRef, error) { sr, err := taskToSectorRef(taskID) if err != nil { @@ -69,7 +73,7 @@ func (sb *SealCalls) Storage(taskToSectorRef func(taskID harmonytask.TaskID) (Se } return []SectorRef{sr}, nil - }, alloc, existing, ssize, pathType, MinFreeStoragePercentage, storiface.FSOverheadSeal) + }, alloc, existing, ssize, pathType, MinFreeStoragePercentage, oh) } func (sb *SealCalls) StorageMulti(taskToSectorRef func(taskID harmonytask.TaskID) ([]SectorRef, error), alloc, existing storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType, MinFreeStoragePercentage float64, ohs map[storiface.SectorFileType]int) *TaskStorage { diff --git a/lib/paths/fetch.go b/lib/paths/fetch.go index d9a2a956a..8fc99f5c5 100644 --- a/lib/paths/fetch.go +++ b/lib/paths/fetch.go @@ -16,7 +16,7 @@ import ( func init() { tarutil.CacheFileConstraints["batch.json"] = 10_000 - tarutil.CacheFileConstraints["commit-phase1-output"] = 20_000_000 + tarutil.CacheFileConstraints["commit-phase1-output"] = 128_000_000 // ~50 MiB typical for C1 JSON } func fetch(ctx context.Context, url, outname string, header http.Header) (rerr error) { diff --git a/lib/paths/local.go b/lib/paths/local.go index 87b0c1a47..4af61070e 100644 --- a/lib/paths/local.go +++ b/lib/paths/local.go @@ -6,8 +6,10 @@ import ( "encoding/json" "expvar" "fmt" + "io" "math/bits" "math/rand" + "net/http" "os" "path/filepath" "runtime" @@ -71,6 +73,17 @@ const BatchMetaFile = "batch.json" // supraseal const MinFreeStoragePercentage = float64(0) const CommitPhase1OutputFileSupra = "commit-phase1-output" +const RemoteSealC1UrlFile = "c1.url" // remote seal: JSON with commit1 endpoint info + +// RemoteSealC1Info is the JSON structure stored in the c1.url file in a sector's +// cache directory. It tells GeneratePoRepVanillaProof how to fetch C1 output +// from a remote seal provider instead of computing it locally. +type RemoteSealC1Info struct { + C1URL string `json:"c1_url"` // full URL to the provider's /commit1 endpoint + PartnerToken string `json:"partner_token"` // auth token for the provider + SpID int64 `json:"sp_id"` + SectorNumber int64 `json:"sector_number"` +} // used to guard allocation decisions between assignment and reservation var ReservationCtxLock = contextlock.NewContextLock() @@ -1275,6 +1288,14 @@ func (st *Local) GeneratePoRepVanillaProof(ctx context.Context, sr storiface.Sec } } + { + // check if this is a remote-sealed sector with a c1.url file + c1UrlPath := filepath.Join(src.Cache, RemoteSealC1UrlFile) + if _, err := os.Stat(c1UrlPath); err == nil { + return st.remoteSealPoRepVanillaProof(src, sr, seed) + } + } + secPiece := []abi.PieceInfo{{ Size: abi.PaddedPieceSize(ssize), PieceCID: unsealed, @@ -1301,6 +1322,87 @@ func (st *Local) ReadSnapVanillaProof(ctx context.Context, sr storiface.SectorRe return out, nil } +// remoteSealPoRepVanillaProof fetches C1 output from a remote seal provider. +// The c1.url file in the sector cache directory contains the endpoint info. +// The result is saved as commit-phase1-output in the cache directory so that +// the standard PoRepSnark path can use it, consistent with the supra path. +func (st *Local) remoteSealPoRepVanillaProof(src storiface.SectorPaths, sr storiface.SectorRef, seed abi.InteractiveSealRandomness) ([]byte, error) { + // Check if commit-phase1-output already exists (idempotent retry) + commitPhase1OutputPath := filepath.Join(src.Cache, CommitPhase1OutputFileSupra) + if data, err := os.ReadFile(commitPhase1OutputPath); err == nil && len(data) > 0 { + log.Infow("remoteSealPoRepVanillaProof: using cached commit-phase1-output", "sref", sr) + return data, nil + } + + // Read c1.url file + c1UrlPath := filepath.Join(src.Cache, RemoteSealC1UrlFile) + c1InfoData, err := os.ReadFile(c1UrlPath) + if err != nil { + return nil, xerrors.Errorf("read c1.url file: %w", err) + } + + var c1Info RemoteSealC1Info + if err := json.Unmarshal(c1InfoData, &c1Info); err != nil { + return nil, xerrors.Errorf("unmarshal c1.url: %w", err) + } + + // Build commit1 request + reqBody := struct { + PartnerToken string `json:"partner_token"` + SpID int64 `json:"sp_id"` + SectorNumber int64 `json:"sector_number"` + SeedValue []byte `json:"seed_value"` + }{ + PartnerToken: c1Info.PartnerToken, + SpID: c1Info.SpID, + SectorNumber: c1Info.SectorNumber, + SeedValue: seed, + } + + reqJSON, err := json.Marshal(reqBody) + if err != nil { + return nil, xerrors.Errorf("marshal commit1 request: %w", err) + } + + // POST to provider + httpReq, err := http.NewRequest(http.MethodPost, c1Info.C1URL, bytes.NewReader(reqJSON)) + if err != nil { + return nil, xerrors.Errorf("create commit1 request: %w", err) + } + httpReq.Header.Set("Content-Type", "application/json") + + resp, err := http.DefaultClient.Do(httpReq) + if err != nil { + return nil, xerrors.Errorf("commit1 request to %s: %w", c1Info.C1URL, err) + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return nil, xerrors.Errorf("commit1 returned status %d: %s", resp.StatusCode, string(body)) + } + + // Read raw bytes — provider sends application/octet-stream + c1Output, err := io.ReadAll(resp.Body) + if err != nil { + return nil, xerrors.Errorf("read commit1 response body: %w", err) + } + + if len(c1Output) == 0 { + return nil, xerrors.Errorf("provider returned empty C1 output") + } + + // Write to commit-phase1-output for caching / consistency with supra path + if err := os.WriteFile(commitPhase1OutputPath, c1Output, 0644); err != nil { + return nil, xerrors.Errorf("write commit-phase1-output: %w", err) + } + + log.Infow("remoteSealPoRepVanillaProof: fetched C1 from provider", + "sref", sr, "c1_size", len(c1Output), "url", c1Info.C1URL) + + return c1Output, nil +} + var supraC1Token = make(chan struct{}, 1) func (st *Local) supraPoRepVanillaProof(src storiface.SectorPaths, sr storiface.SectorRef, _, unsealed cid.Cid, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness) ([]byte, error) { diff --git a/lib/paths/testmain_test.go b/lib/paths/testmain_test.go new file mode 100644 index 000000000..ca5dbd5bd --- /dev/null +++ b/lib/paths/testmain_test.go @@ -0,0 +1,12 @@ +package paths + +import ( + "os" + "testing" + + "github.com/filecoin-project/curio/lib/testutil/dbtest" +) + +func TestMain(m *testing.M) { + os.Exit(dbtest.StartYugabyte(m)) +} diff --git a/lib/testutil/dbtest/yb.go b/lib/testutil/dbtest/yb.go new file mode 100644 index 000000000..6abb386ff --- /dev/null +++ b/lib/testutil/dbtest/yb.go @@ -0,0 +1,191 @@ +// Package dbtest provides a shared helper that starts a YugabyteDB container +// via testcontainers-go for use in TestMain functions. When the environment +// variable CURIO_HARMONYDB_HOSTS is already set (e.g. by CI or a manually +// started database) the container is skipped and the existing database is used +// instead. +// +// The container uses dynamic port mapping so that multiple packages can each +// start their own YugabyteDB without port conflicts when `go test ./...` runs +// packages in parallel. +// +// Two optimizations are applied to speed up schema migrations in tests: +// +// 1. Tablet count is reduced to 1 per tserver (--yb_num_shards_per_tserver=1, +// --ysql_num_shards_per_tserver=1) instead of the default based on CPU +// count. This drastically reduces the overhead of CREATE TABLE/INDEX. +// +// 2. A colocated database ("curio_test") is created after startup. Colocated +// databases store all tables in a single tablet, which eliminates per-table +// tablet creation overhead entirely. Tests connect to this database via +// the CURIO_HARMONYDB_DB environment variable. +package dbtest + +import ( + "context" + "fmt" + "io" + "os" + "strings" + "testing" + "time" + + "github.com/moby/moby/api/types/container" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/modules/yugabytedb" + "github.com/testcontainers/testcontainers-go/wait" +) + +const ybImage = "yugabytedb/yugabyte:2024.1.2.0-b77" + +// colocatedDBName is the name of the colocated database created for tests. +const colocatedDBName = "curio_test" + +// StartYugabyte starts a YugabyteDB container (unless CURIO_HARMONYDB_HOSTS is +// already set), runs the test suite, terminates the container, and returns the +// exit code from m.Run(). Callers should use it as: +// +// func TestMain(m *testing.M) { os.Exit(dbtest.StartYugabyte(m)) } +func StartYugabyte(m *testing.M) int { + // If the env var is already set, an external database is available. + if os.Getenv("CURIO_HARMONYDB_HOSTS") != "" { + return m.Run() + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + fmt.Println("dbtest: no CURIO_HARMONYDB_HOSTS set, starting YugabyteDB via testcontainers...") + + ctr, err := yugabytedb.Run(ctx, ybImage, + // Do NOT bind to fixed host ports — let Docker pick free ports so + // that multiple packages can each run their own container in + // parallel without conflicts. + + // Reduce tablet count to 1 per tserver for both YCQL and YSQL. + // YugabyteDB normally creates multiple tablets per table based on + // CPU count, which makes CREATE TABLE very slow. With 1 shard + // per tserver, each DDL statement creates just one tablet. + // See: https://docs.yugabyte.com/v2024.1/best-practices-operations/administration/#settings-for-ci-and-cd-integration-tests + testcontainers.CustomizeRequest(testcontainers.GenericContainerRequest{ + ContainerRequest: testcontainers.ContainerRequest{ + Cmd: []string{ + "--tserver_flags=yb_num_shards_per_tserver=1,ysql_num_shards_per_tserver=1", + }, + }, + }), + // Strip all YSQL_*/YCQL_* env vars so YugabyteDB starts with + // default trust authentication (no passwords, same as the bare + // `docker run` used historically in CI). The testcontainers + // YugabyteDB module sets user/password env vars by default, which + // causes yugabyted to enable md5/PasswordAuthenticator. + testcontainers.WithConfigModifier(func(cfg *container.Config) { + filtered := cfg.Env[:0] + for _, e := range cfg.Env { + if !strings.HasPrefix(e, "YSQL_") && !strings.HasPrefix(e, "YCQL_") { + filtered = append(filtered, e) + } + } + cfg.Env = filtered + }), + // Generous deadline — YugabyteDB can take 30-60s to become ready. + testcontainers.WithWaitStrategyAndDeadline(3*time.Minute, + wait.ForLog("YugabyteDB Started").WithOccurrence(1), + wait.ForLog("Data placement constraint successfully verified").WithOccurrence(1), + wait.ForListeningPort("5433/tcp"), + wait.ForListeningPort("9042/tcp"), + ), + ) + if err != nil { + fmt.Fprintf(os.Stderr, "dbtest: failed to start YugabyteDB container: %v\n", err) + fmt.Fprintf(os.Stderr, "dbtest:\n") + fmt.Fprintf(os.Stderr, "dbtest: Possible causes:\n") + fmt.Fprintf(os.Stderr, "dbtest: - Docker is not running\n") + fmt.Fprintf(os.Stderr, "dbtest: - Image pull failure (check network)\n") + fmt.Fprintf(os.Stderr, "dbtest:\n") + fmt.Fprintf(os.Stderr, "dbtest: To skip the container and use an existing DB, set CURIO_HARMONYDB_HOSTS=127.0.0.1\n") + if ctr != nil { + _ = testcontainers.TerminateContainer(ctr) + } + return 1 + } + + // Retrieve dynamically mapped host and ports. + host, err := ctr.Host(ctx) + if err != nil { + fmt.Fprintf(os.Stderr, "dbtest: failed to get container host: %v\n", err) + _ = testcontainers.TerminateContainer(ctr) + return 1 + } + + ysqlPort, err := ctr.MappedPort(ctx, "5433/tcp") + if err != nil { + fmt.Fprintf(os.Stderr, "dbtest: failed to get YSQL mapped port: %v\n", err) + _ = testcontainers.TerminateContainer(ctr) + return 1 + } + + ycqlPort, err := ctr.MappedPort(ctx, "9042/tcp") + if err != nil { + fmt.Fprintf(os.Stderr, "dbtest: failed to get YCQL mapped port: %v\n", err) + _ = testcontainers.TerminateContainer(ctr) + return 1 + } + + fmt.Printf("dbtest: YugabyteDB ready (YSQL=%s:%s, YCQL=%s:%s)\n", + host, ysqlPort.Port(), host, ycqlPort.Port()) + + // Create a colocated database for tests. In a colocated database all + // tables share a single tablet, which eliminates the per-table tablet + // creation overhead that makes schema migrations slow in YugabyteDB. + // yugabyted binds YSQL to the container's assigned IP (not 0.0.0.0 + // or 127.0.0.1), so we need the container IP for the exec command. + containerIP, err := ctr.ContainerIP(ctx) + if err != nil { + fmt.Fprintf(os.Stderr, "dbtest: failed to get container IP: %v\n", err) + _ = testcontainers.TerminateContainer(ctr) + return 1 + } + + exitCode, execReader, err := ctr.Exec(ctx, []string{ + "ysqlsh", "-h", containerIP, "-p", "5433", "-U", "yugabyte", + "-c", fmt.Sprintf("CREATE DATABASE %s WITH COLOCATION = true", colocatedDBName), + }) + if err != nil || exitCode != 0 { + var execOutput string + if execReader != nil { + if b, readErr := io.ReadAll(execReader); readErr == nil { + execOutput = string(b) + } + } + fmt.Fprintf(os.Stderr, "dbtest: failed to create colocated database %q (exit=%d): %v\nOutput: %s\n", + colocatedDBName, exitCode, err, execOutput) + _ = testcontainers.TerminateContainer(ctr) + return 1 + } + fmt.Printf("dbtest: created colocated database %q\n", colocatedDBName) + + // Publish connection info via environment variables so that + // harmonydb.NewFromConfigWithITestID (reads CURIO_HARMONYDB_HOSTS, + // CURIO_HARMONYDB_PORT, and CURIO_HARMONYDB_DB) and indexstore tests + // (reads CURIO_HARMONYDB_HOSTS and CURIO_HARMONYDB_CQL_PORT) can find + // the container. + for _, kv := range [][2]string{ + {"CURIO_HARMONYDB_HOSTS", host}, + {"CURIO_HARMONYDB_PORT", ysqlPort.Port()}, + {"CURIO_HARMONYDB_CQL_PORT", ycqlPort.Port()}, + {"CURIO_HARMONYDB_DB", colocatedDBName}, + } { + if err := os.Setenv(kv[0], kv[1]); err != nil { + fmt.Fprintf(os.Stderr, "dbtest: failed to set %s: %v\n", kv[0], err) + _ = testcontainers.TerminateContainer(ctr) + return 1 + } + } + + code := m.Run() + + fmt.Println("dbtest: stopping YugabyteDB container...") + _ = testcontainers.TerminateContainer(ctr) + + return code +} diff --git a/market/indexstore/indexstore_test.go b/market/indexstore/indexstore_test.go index 44ece2c0c..fba958027 100644 --- a/market/indexstore/indexstore_test.go +++ b/market/indexstore/indexstore_test.go @@ -5,6 +5,7 @@ import ( "io" "math/rand" "os" + "strconv" "testing" carv2 "github.com/ipld/go-car/v2" @@ -33,7 +34,12 @@ func TestNewIndexStore(t *testing.T) { ctx := context.Background() cfg := config.DefaultCurioConfig() - idxStore, err := NewIndexStore([]string{envElse("CURIO_HARMONYDB_HOSTS", "127.0.0.1")}, 9042, cfg) + cqlPort := 9042 + if v := os.Getenv("CURIO_HARMONYDB_CQL_PORT"); v != "" { + cqlPort, _ = strconv.Atoi(v) + } + + idxStore, err := NewIndexStore([]string{envElse("CURIO_HARMONYDB_HOSTS", "127.0.0.1")}, cqlPort, cfg) require.NoError(t, err) err = idxStore.Start(ctx, true) diff --git a/market/indexstore/testmain_test.go b/market/indexstore/testmain_test.go new file mode 100644 index 000000000..897f0603a --- /dev/null +++ b/market/indexstore/testmain_test.go @@ -0,0 +1,12 @@ +package indexstore + +import ( + "os" + "testing" + + "github.com/filecoin-project/curio/lib/testutil/dbtest" +) + +func TestMain(m *testing.M) { + os.Exit(dbtest.StartYugabyte(m)) +} diff --git a/market/sealmarket/metrics.go b/market/sealmarket/metrics.go new file mode 100644 index 000000000..f630d0cb5 --- /dev/null +++ b/market/sealmarket/metrics.go @@ -0,0 +1,265 @@ +package sealmarket + +import ( + "context" + "net/http" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + "go.opencensus.io/stats" + "go.opencensus.io/stats/view" + "go.opencensus.io/tag" +) + +// Tag keys for remote seal metrics. +var ( + RsealEndpointKey, _ = tag.NewKey("endpoint") + RsealMethodKey, _ = tag.NewKey("method") + RsealStatusCodeKey, _ = tag.NewKey("status_code") + RsealAcceptedKey, _ = tag.NewKey("accepted") +) + +// Duration distribution tuned for remote seal workloads. +// Range: 100ms to 30 minutes. Sealed-data transfers (32 GiB) and commit1 +// FFI computation can take minutes; lighter endpoints finish in <1s. +var rsealDurationDistribution = view.Distribution( + 100, // 100ms + 500, // 500ms + 1000, // 1s + 2000, // 2s + 5000, // 5s + 10000, // 10s + 30000, // 30s + 60000, // 1min + 120000, // 2min + 300000, // 5min + 600000, // 10min + 1800000, // 30min +) + +// Commit1 FFI compute distribution, tuned for CPU/GPU-bound work. +// Range: 1s to 10 minutes. +var rsealComputeDistribution = view.Distribution( + 1000, // 1s + 5000, // 5s + 10000, // 10s + 30000, // 30s + 60000, // 1min + 120000, // 2min + 300000, // 5min + 600000, // 10min +) + +// Tier 1: Cross-cutting middleware measures. +var ( + // RsealRequestCount counts HTTP requests to remote seal endpoints. + RsealRequestCount = stats.Int64("rseal/request_count", "Counter of remote seal HTTP requests", stats.UnitDimensionless) + + // RsealResponseStatusCount counts HTTP response status codes. + RsealResponseStatusCount = stats.Int64("rseal/response_status_count", "Counter of remote seal HTTP response status codes", stats.UnitDimensionless) + + // RsealResponseBytesCount sums HTTP response bytes written. + RsealResponseBytesCount = stats.Int64("rseal/response_bytes_count", "Sum of remote seal HTTP response bytes", stats.UnitBytes) + + // RsealActiveRequests tracks in-flight requests per endpoint. + RsealActiveRequests = stats.Int64("rseal/active_requests", "Number of active remote seal HTTP requests", stats.UnitDimensionless) + + // RsealRequestDuration records request latency in milliseconds. + RsealRequestDuration = stats.Float64("rseal/request_duration_ms", "Remote seal HTTP request duration", stats.UnitMilliseconds) +) + +// Tier 3: Business-level measures. +var ( + // RsealOrdersTotal counts order attempts, labeled by accepted (true/false). + RsealOrdersTotal = stats.Int64("rseal/orders_total", "Counter of remote seal order attempts", stats.UnitDimensionless) + + // RsealSlotsIssued counts slot tokens issued by /available. + RsealSlotsIssued = stats.Int64("rseal/slots_issued_total", "Counter of remote seal slot tokens issued", stats.UnitDimensionless) + + // RsealCommit1ComputeDuration records the C1 FFI computation time in milliseconds + // (excluding network I/O, DB queries, etc.). + RsealCommit1ComputeDuration = stats.Float64("rseal/commit1_compute_duration_ms", "Remote seal commit1 FFI compute duration", stats.UnitMilliseconds) +) + +// Views define how the measures are aggregated and exported to Prometheus. +var ( + RsealRequestCountView = &view.View{ + Measure: RsealRequestCount, + Aggregation: view.Count(), + TagKeys: []tag.Key{RsealEndpointKey, RsealMethodKey}, + } + RsealResponseStatusCountView = &view.View{ + Measure: RsealResponseStatusCount, + Aggregation: view.Count(), + TagKeys: []tag.Key{RsealEndpointKey, RsealMethodKey, RsealStatusCodeKey}, + } + RsealResponseBytesCountView = &view.View{ + Measure: RsealResponseBytesCount, + Aggregation: view.Sum(), + TagKeys: []tag.Key{RsealEndpointKey, RsealStatusCodeKey}, + } + RsealActiveRequestsView = &view.View{ + Measure: RsealActiveRequests, + Aggregation: view.LastValue(), + TagKeys: []tag.Key{RsealEndpointKey, RsealMethodKey}, + } + RsealRequestDurationView = &view.View{ + Measure: RsealRequestDuration, + Aggregation: rsealDurationDistribution, + TagKeys: []tag.Key{RsealEndpointKey, RsealMethodKey}, + } + + RsealOrdersTotalView = &view.View{ + Measure: RsealOrdersTotal, + Aggregation: view.Count(), + TagKeys: []tag.Key{RsealAcceptedKey}, + } + RsealSlotsIssuedView = &view.View{ + Measure: RsealSlotsIssued, + Aggregation: view.Count(), + } + RsealCommit1ComputeDurationView = &view.View{ + Measure: RsealCommit1ComputeDuration, + Aggregation: rsealComputeDistribution, + } +) + +func init() { + err := view.Register( + RsealRequestCountView, + RsealResponseStatusCountView, + RsealResponseBytesCountView, + RsealActiveRequestsView, + RsealRequestDurationView, + RsealOrdersTotalView, + RsealSlotsIssuedView, + RsealCommit1ComputeDurationView, + ) + if err != nil { + panic(err) + } +} + +// --- Metrics middleware --- + +// rsealActiveCounters maps "endpoint:method" → *atomic.Int64 for the gauge. +var rsealActiveCounters sync.Map + +func rsealIncrementActive(endpoint, method string) *atomic.Int64 { + key := endpoint + ":" + method + if v, ok := rsealActiveCounters.Load(key); ok { + counter := v.(*atomic.Int64) + counter.Add(1) + return counter + } + counter := &atomic.Int64{} + actual, _ := rsealActiveCounters.LoadOrStore(key, counter) + c := actual.(*atomic.Int64) + c.Add(1) + return c +} + +func rsealDecrementActive(ctx context.Context, counter *atomic.Int64, endpoint, method string) { + val := counter.Add(-1) + _ = stats.RecordWithTags(ctx, []tag.Mutator{ + tag.Upsert(RsealEndpointKey, endpoint), + tag.Upsert(RsealMethodKey, method), + }, RsealActiveRequests.M(val)) +} + +// rsealResponseWriter wraps http.ResponseWriter to capture status code and bytes written. +type rsealResponseWriter struct { + http.ResponseWriter + statusCode int + bytesWritten int64 +} + +func (rw *rsealResponseWriter) WriteHeader(statusCode int) { + rw.statusCode = statusCode + rw.ResponseWriter.WriteHeader(statusCode) +} + +func (rw *rsealResponseWriter) Write(b []byte) (int, error) { + n, err := rw.ResponseWriter.Write(b) + rw.bytesWritten += int64(n) + return n, err +} + +// Unwrap exposes the underlying ResponseWriter for http.ServeFile to detect +// io.ReadFrom support (sendfile). +func (rw *rsealResponseWriter) Unwrap() http.ResponseWriter { + return rw.ResponseWriter +} + +// rsealEndpointName extracts a short, low-cardinality endpoint label from a request path. +// The path is expected to start with DelegatedSealPath ("/remoteseal/delegated/v0/"). +// Examples: +// +// "/remoteseal/delegated/v0/capabilities" → "capabilities" +// "/remoteseal/delegated/v0/sealed-data/1234/5" → "sealed-data" +// "/remoteseal/delegated/v0/cache-data/1234/5" → "cache-data" +// "/remoteseal/delegated/v0/commit1" → "commit1" +func rsealEndpointName(urlPath string) string { + // Strip the base prefix + rest := strings.TrimPrefix(urlPath, DelegatedSealPath) + if rest == urlPath { + // Path didn't have the expected prefix — fallback + return "unknown" + } + + // Take the first segment (before any '/') + if idx := strings.IndexByte(rest, '/'); idx >= 0 { + rest = rest[:idx] + } + + if rest == "" { + return "unknown" + } + + return rest +} + +// rsealMetricsMiddleware is a chi middleware that records OpenCensus metrics +// for all remote seal endpoints. It mirrors the retrieval metricsMiddleware +// pattern: request count, response status/bytes, in-flight gauge, and latency. +func rsealMetricsMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + endpoint := rsealEndpointName(r.URL.Path) + start := time.Now() + + // Record request count and set up context with tags + ctx, _ := tag.New(r.Context(), + tag.Upsert(RsealEndpointKey, endpoint), + tag.Upsert(RsealMethodKey, r.Method), + ) + stats.Record(ctx, RsealRequestCount.M(1)) + + // Track in-flight requests + counter := rsealIncrementActive(endpoint, r.Method) + defer rsealDecrementActive(ctx, counter, endpoint, r.Method) + + // Wrap response writer to capture status and bytes + wrapper := &rsealResponseWriter{ + ResponseWriter: w, + statusCode: http.StatusOK, // default if WriteHeader is not called + } + + // Serve the request + next.ServeHTTP(wrapper, r.WithContext(ctx)) + + // Record response metrics + elapsed := float64(time.Since(start).Milliseconds()) + statusStr := strconv.Itoa(wrapper.statusCode) + + _ = stats.RecordWithTags(ctx, []tag.Mutator{ + tag.Upsert(RsealStatusCodeKey, statusStr), + }, + RsealResponseStatusCount.M(1), + RsealResponseBytesCount.M(wrapper.bytesWritten), + ) + stats.Record(ctx, RsealRequestDuration.M(elapsed)) + }) +} diff --git a/market/sealmarket/sealapi.go b/market/sealmarket/sealapi.go new file mode 100644 index 000000000..b9bf88c13 --- /dev/null +++ b/market/sealmarket/sealapi.go @@ -0,0 +1,1113 @@ +package sealmarket + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "strconv" + "sync" + "time" + + "github.com/go-chi/chi/v5" + "github.com/ipfs/go-cid" + logging "github.com/ipfs/go-log/v2" + "go.opencensus.io/stats" + "go.opencensus.io/tag" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + + "github.com/filecoin-project/curio/harmony/harmonydb" + "github.com/filecoin-project/curio/lib/paths" + "github.com/filecoin-project/curio/lib/storiface" + "github.com/filecoin-project/curio/tasks/seal" +) + +var log = logging.Logger("sealmarket") + +// slotEntry is an in-memory slot reservation with a deadline. +type slotEntry struct { + partnerID int64 + deadline time.Time +} + +type SealMarket struct { + db *harmonydb.DB + paths *paths.Remote + api seal.TicketNodeAPI // chain API for ticket generation (client-side) + + slotsMu sync.Mutex + slots map[string]*slotEntry +} + +func NewSealMarket(db *harmonydb.DB, paths *paths.Remote, api seal.TicketNodeAPI) *SealMarket { + return &SealMarket{ + db: db, + paths: paths, + api: api, + slots: make(map[string]*slotEntry), + } +} + +// readerPieceReadSeeker wraps a ReaderPiece factory function into an io.ReadSeeker +// so that http.ServeContent can handle Range headers automatically. +type readerPieceReadSeeker struct { + factory func(startOffset, endOffset int64) (io.ReadCloser, error) + size int64 + pos int64 + cur io.ReadCloser +} + +func (r *readerPieceReadSeeker) Read(p []byte) (int, error) { + if r.pos >= r.size { + return 0, io.EOF + } + if r.cur == nil { + // Open reader from current position to end + rc, err := r.factory(r.pos, r.size) + if err != nil { + return 0, err + } + r.cur = rc + } + n, err := r.cur.Read(p) + r.pos += int64(n) + return n, err +} + +func (r *readerPieceReadSeeker) Seek(offset int64, whence int) (int64, error) { + var newPos int64 + switch whence { + case io.SeekStart: + newPos = offset + case io.SeekCurrent: + newPos = r.pos + offset + case io.SeekEnd: + newPos = r.size + offset + default: + return 0, xerrors.Errorf("invalid whence: %d", whence) + } + if newPos < 0 { + return 0, xerrors.Errorf("negative seek position: %d", newPos) + } + // If seeking to a different position, close the current reader + if r.cur != nil && newPos != r.pos { + _ = r.cur.Close() + r.cur = nil + } + r.pos = newPos + return r.pos, nil +} + +func (r *readerPieceReadSeeker) Close() error { + if r.cur != nil { + return r.cur.Close() + } + return nil +} + +const SealMarketRoutePath = "/remoteseal/" +const DelegatedSealPath = SealMarketRoutePath + "delegated/v0/" + +/* +/remoteseal/delegated/v0 -> v0 trusted delegated sealing + +// setup flow +1. Client provides partner_url (base cuhttp url) +2. Provider in UI sets the URL, partner_name, total allowance +3. Providers curio contacts partner_url to check remoteseal capabilities +4. Provider UI returns a connect-string (token+cuhttp endpoint b64 encoded) +5. Client sets connect-string in UI, and clicks "connect" +6. Client curio -> provider /remoteseal/delegated/v0/capabilities +7. Client curio -> provider /remoteseal/delegated/v0/authorize (stateless auth confirm) +8. Client curio adds provider to DB + +-- +// sealing flow + +1. Client starts cc sector as usual, e.g with cc scheduler +2. Sealing pipeline creates sectors_sdr_pipeline entry +3. RSealDelegate task picks up the task, similar to supra batch seal task, creates rseal_client_pipeline entry + 3.1. IFF we have providers that are available + 3.2. With 5s timeout contact provider to check /available -> true/false + 30s available slot token +4. If provider is available, rseal_client_pipeline entry is created +5. RSealDelegate task starts client side, sends /remoteseal/delegated/v0/order to provider with sector details +6. Provider spawns matching pipeline +7. Provider SDR task computes ticket from chain locally and runs SDR +8. Trees run and finish provider side +9. Provider sends /remoteseal/delegated/v0/complete to client (includes ticket), client also polls /remoteseal/delegated/v0/status +10.1. Client sends precommit through the normal precommit pipeline +10.2. Client fetches sealed file: GET /remoteseal/delegated/v0/sealed-data/{sp_id}/{sector_number}?token=... (32 GiB, Range, aria2c) +10.3. Client fetches fincache: GET /remoteseal/delegated/v0/cache-data/{sp_id}/{sector_number}?token=... (tar, ~73 MiB) +11. At C1 client contacts provider /remoteseal/delegated/v0/commit1 to supply C1 seed and get C1 output +12. After client records C1 output, client sends /remoteseal/delegated/v0/finalize to provider letting the provider know that layers can be dropped +13. After all data is client-side, client sends /remoteseal/delegated/v0/cleanup to provider letting the provider know that cleanup can begin +*/ + +// --- Request/Response types --- + +// CapabilitiesResponse is returned by the provider to describe what it can do. +type CapabilitiesResponse struct { + // SupportedProofs lists the registered seal proof types this provider supports. + SupportedProofs []int64 `json:"supported_proofs"` + + // MaxBatchSize is the maximum batch size the provider can handle (e.g. 128 for supraseal). + MaxBatchSize int `json:"max_batch_size"` + + // SupportsRangeRequests indicates sealed-data endpoint supports HTTP Range headers. + SupportsRangeRequests bool `json:"supports_range_requests"` +} + +// AuthorizeRequest is sent by the client to confirm auth works. +type AuthorizeRequest struct { + PartnerToken string `json:"partner_token"` +} + +// AuthorizeResponse confirms the partner is recognized and has allowance. +type AuthorizeResponse struct { + Authorized bool `json:"authorized"` + PartnerName string `json:"partner_name"` + AllowanceRemaining int64 `json:"allowance_remaining"` +} + +// AvailableResponse is returned when checking provider slot availability. +type AvailableResponse struct { + Available bool `json:"available"` + SlotToken string `json:"slot_token,omitempty"` // 30s reservation token +} + +// OrderRequest is sent by the client to create a remote seal order. +// Delegated sectors are always CC (no deal data). CommD is the static +// zero-commitment derived from the sector size (reg_seal_proof). +type OrderRequest struct { + PartnerToken string `json:"partner_token"` + SlotToken string `json:"slot_token"` // from /available + + SpID int64 `json:"sp_id"` + SectorNumber int64 `json:"sector_number"` + RegSealProof int `json:"reg_seal_proof"` +} + +// OrderResponse confirms the order was accepted by the provider. +// The provider seals under the client's sp_id/sector_number (no separate provider identity). +type OrderResponse struct { + Accepted bool `json:"accepted"` + RejectReason string `json:"reject_reason,omitempty"` +} + +// StatusRequest is used by the client to poll completion status. +type StatusRequest struct { + PartnerToken string `json:"partner_token"` + SpID int64 `json:"sp_id"` + SectorNumber int64 `json:"sector_number"` +} + +// StatusResponse describes the current state of a remote seal job. +type StatusResponse struct { + State string `json:"state"` // "pending", "sdr", "trees", "complete", "failed", "gone" + TreeDCid string `json:"tree_d_cid,omitempty"` + TreeRCid string `json:"tree_r_cid,omitempty"` + TicketEpoch int64 `json:"ticket_epoch,omitempty"` + TicketValue []byte `json:"ticket_value,omitempty"` + FailReason string `json:"fail_reason,omitempty"` +} + +// CompleteNotification is sent by the provider to the client when SDR+trees finish. +type CompleteNotification struct { + PartnerToken string `json:"partner_token"` + SpID int64 `json:"sp_id"` + SectorNumber int64 `json:"sector_number"` + TreeDCid string `json:"tree_d_cid"` + TreeRCid string `json:"tree_r_cid"` + TicketEpoch int64 `json:"ticket_epoch"` + TicketValue []byte `json:"ticket_value"` +} + +// Commit1Request is sent by the client to the provider to exchange C1 seed for C1 output. +type Commit1Request struct { + PartnerToken string `json:"partner_token"` + SpID int64 `json:"sp_id"` + SectorNumber int64 `json:"sector_number"` + SeedEpoch int64 `json:"seed_epoch"` + SeedValue []byte `json:"seed_value"` +} + +// FinalizeRequest is sent by the client to tell the provider layers can be dropped. +type FinalizeRequest struct { + PartnerToken string `json:"partner_token"` + SpID int64 `json:"sp_id"` + SectorNumber int64 `json:"sector_number"` +} + +// CleanupRequest is sent by the client to tell the provider to begin full cleanup. +type CleanupRequest struct { + PartnerToken string `json:"partner_token"` + SpID int64 `json:"sp_id"` + SectorNumber int64 `json:"sector_number"` +} + +// TicketResponse is returned by the client's /ticket endpoint with seal randomness +// from the client's chain. The provider calls this instead of using its own chain +// when sealing remote sectors (which may be on a different network). +type TicketResponse struct { + Ticket []byte `json:"ticket"` // abi.SealRandomness + Epoch int64 `json:"epoch"` // abi.ChainEpoch +} + +// --- Routes --- + +func Routes(r *chi.Mux, sm *SealMarket) { + r.Route(DelegatedSealPath, func(r chi.Router) { + r.Use(rsealMetricsMiddleware) + + // Setup flow endpoints (called by client) + r.Get("/capabilities", sm.handleCapabilities) + r.Post("/authorize", sm.handleAuthorize) + + // Availability check (called by client) + r.Post("/available", sm.handleAvailable) + + // Sealing flow - provider-side endpoints (called by client) + r.Post("/order", sm.handleOrder) + r.Post("/status", sm.handleStatus) + r.Get("/sealed-data/{sp_id}/{sector_number}", sm.handleSealedData) // serves sealed file; supports Range header + r.Get("/cache-data/{sp_id}/{sector_number}", sm.handleCacheData) // serves fincache tar (p_aux, t_aux, tree-r-last) + r.Post("/commit1", sm.handleCommit1) + r.Post("/finalize", sm.handleFinalize) + r.Post("/cleanup", sm.handleCleanup) + + // Sealing flow - client-side endpoints (called by provider) + r.Post("/complete", sm.handleComplete) + r.Get("/ticket", sm.handleTicket) // stateless: provider fetches seal randomness from client's chain + }) +} + +// --- Handler implementations --- + +// handleCapabilities returns what this provider supports. +// GET /remoteseal/delegated/v0/capabilities +func (sm *SealMarket) handleCapabilities(w http.ResponseWriter, r *http.Request) { + resp := CapabilitiesResponse{ + SupportedProofs: []int64{ + int64(abi.RegisteredSealProof_StackedDrg32GiBV1_1), + int64(abi.RegisteredSealProof_StackedDrg64GiBV1_1), + int64(abi.RegisteredSealProof_StackedDrg32GiBV1_1_Feat_SyntheticPoRep), + int64(abi.RegisteredSealProof_StackedDrg64GiBV1_1_Feat_SyntheticPoRep), + }, + MaxBatchSize: 128, + SupportsRangeRequests: true, + } + writeJSON(w, http.StatusOK, resp) +} + +// handleAuthorize confirms the partner token is valid and has allowance. +// POST /remoteseal/delegated/v0/authorize +func (sm *SealMarket) handleAuthorize(w http.ResponseWriter, r *http.Request) { + var req AuthorizeRequest + if !readJSON(w, r, &req) { + return + } + + var partners []struct { + PartnerName string `db:"partner_name"` + AllowanceRemaining int64 `db:"allowance_remaining"` + } + + err := sm.db.Select(r.Context(), &partners, `SELECT partner_name, allowance_remaining FROM rseal_delegated_partners WHERE partner_token = $1`, req.PartnerToken) + if err != nil { + log.Errorw("authorize: db query failed", "error", err) + http.Error(w, "internal error", http.StatusInternalServerError) + return + } + + if len(partners) == 0 { + writeJSON(w, http.StatusOK, AuthorizeResponse{Authorized: false}) + return + } + + resp := AuthorizeResponse{ + Authorized: true, + PartnerName: partners[0].PartnerName, + AllowanceRemaining: partners[0].AllowanceRemaining, + } + writeJSON(w, http.StatusOK, resp) +} + +// handleAvailable checks if the provider has capacity for a new seal job. +// POST /remoteseal/delegated/v0/available +func (sm *SealMarket) handleAvailable(w http.ResponseWriter, r *http.Request) { + var req AuthorizeRequest + if !readJSON(w, r, &req) { + return + } + + // Look up partner + var partners []struct { + ID int64 `db:"id"` + AllowanceTotal int64 `db:"allowance_total"` + AllowanceRemaining int64 `db:"allowance_remaining"` + } + + err := sm.db.Select(r.Context(), &partners, `SELECT id, allowance_total, allowance_remaining FROM rseal_delegated_partners WHERE partner_token = $1`, req.PartnerToken) + if err != nil { + log.Errorw("available: db query failed", "error", err) + http.Error(w, "internal error", http.StatusInternalServerError) + return + } + + if len(partners) == 0 { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + + partner := partners[0] + + // Check lifetime allowance (hard cap on total sectors ever accepted) + if partner.AllowanceRemaining <= 0 { + writeJSON(w, http.StatusOK, AvailableResponse{Available: false}) + return + } + + // Count active (non-cleaned-up) sectors for this partner + var counts []struct { + Count int64 `db:"count"` + } + + err = sm.db.Select(r.Context(), &counts, `SELECT COUNT(*) AS count FROM rseal_provider_pipeline WHERE partner_id = $1 AND after_cleanup != TRUE`, partner.ID) + if err != nil { + log.Errorw("available: count query failed", "error", err) + http.Error(w, "internal error", http.StatusInternalServerError) + return + } + + activeCount := int64(0) + if len(counts) > 0 { + activeCount = counts[0].Count + } + + // Check concurrent slots cap + if activeCount >= partner.AllowanceTotal { + writeJSON(w, http.StatusOK, AvailableResponse{Available: false}) + return + } + + // Generate a slot token with 30s expiry + tokenBytes := make([]byte, 16) + if _, err := rand.Read(tokenBytes); err != nil { + log.Errorw("available: failed to generate slot token", "error", err) + http.Error(w, "internal error", http.StatusInternalServerError) + return + } + slotToken := hex.EncodeToString(tokenBytes) + + sm.slotsMu.Lock() + sm.slots[slotToken] = &slotEntry{ + partnerID: partner.ID, + deadline: time.Now().Add(30 * time.Second), + } + sm.slotsMu.Unlock() + + stats.Record(r.Context(), RsealSlotsIssued.M(1)) + + writeJSON(w, http.StatusOK, AvailableResponse{ + Available: true, + SlotToken: slotToken, + }) +} + +// handleOrder accepts a remote seal order from a client. +// Idempotent: re-submitting the same (sp_id, sector_number) returns the existing order. +// POST /remoteseal/delegated/v0/order +func (sm *SealMarket) handleOrder(w http.ResponseWriter, r *http.Request) { + var req OrderRequest + if !readJSON(w, r, &req) { + return + } + + // Validate partner token + partnerID, err := sm.validatePartnerToken(r.Context(), req.PartnerToken) + if err != nil { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + + // Check allowed proof types for this partner + { + var partners []struct { + AllowedProofTypes []int64 `db:"allowed_proof_types"` + } + if err := sm.db.Select(r.Context(), &partners, `SELECT allowed_proof_types FROM rseal_delegated_partners WHERE id = $1`, partnerID); err != nil { + log.Errorw("order: failed to query allowed proof types", "error", err) + http.Error(w, "internal error", http.StatusInternalServerError) + return + } + if len(partners) > 0 && len(partners[0].AllowedProofTypes) > 0 { + allowed := false + for _, pt := range partners[0].AllowedProofTypes { + if pt == int64(req.RegSealProof) { + allowed = true + break + } + } + if !allowed { + writeJSON(w, http.StatusOK, OrderResponse{ + Accepted: false, + RejectReason: fmt.Sprintf("proof type %d not allowed for this partner", req.RegSealProof), + }) + return + } + } + } + + // Validate slot token (if present) + if req.SlotToken != "" { + sm.slotsMu.Lock() + entry, ok := sm.slots[req.SlotToken] + if ok { + if time.Now().After(entry.deadline) || entry.partnerID != partnerID { + ok = false + } + delete(sm.slots, req.SlotToken) + } + sm.slotsMu.Unlock() + + if !ok { + _ = stats.RecordWithTags(r.Context(), []tag.Mutator{ + tag.Upsert(RsealAcceptedKey, "false"), + }, RsealOrdersTotal.M(1)) + + writeJSON(w, http.StatusOK, OrderResponse{ + Accepted: false, + RejectReason: "invalid or expired slot token", + }) + return + } + } + + // Insert the pipeline entry (idempotent via ON CONFLICT DO NOTHING) + n, err := sm.db.Exec(r.Context(), `INSERT INTO rseal_provider_pipeline (partner_id, sp_id, sector_number, reg_seal_proof) VALUES ($1, $2, $3, $4) ON CONFLICT (sp_id, sector_number) DO NOTHING`, + partnerID, req.SpID, req.SectorNumber, req.RegSealProof) + if err != nil { + log.Errorw("order: insert failed", "error", err) + http.Error(w, "internal error", http.StatusInternalServerError) + return + } + + // If a new row was inserted, decrement the allowance + if n == 1 { + _, err = sm.db.Exec(r.Context(), `UPDATE rseal_delegated_partners SET allowance_remaining = allowance_remaining - 1 WHERE id = $1 AND allowance_remaining > 0`, partnerID) + if err != nil { + log.Errorw("order: decrement allowance failed", "error", err) + // The pipeline entry was created, so we still return accepted + } + } + + _ = stats.RecordWithTags(r.Context(), []tag.Mutator{ + tag.Upsert(RsealAcceptedKey, "true"), + }, RsealOrdersTotal.M(1)) + + writeJSON(w, http.StatusOK, OrderResponse{Accepted: true}) +} + +// handleStatus returns the current state of a remote seal job. +// POST /remoteseal/delegated/v0/status +func (sm *SealMarket) handleStatus(w http.ResponseWriter, r *http.Request) { + var req StatusRequest + if !readJSON(w, r, &req) { + return + } + + // Validate partner token + partnerID, err := sm.validatePartnerToken(r.Context(), req.PartnerToken) + if err != nil { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + + var rows []struct { + TaskIDSdr *int64 `db:"task_id_sdr"` + TicketEpoch *int64 `db:"ticket_epoch"` + TicketValue []byte `db:"ticket_value"` + AfterSDR bool `db:"after_sdr"` + AfterTreeC bool `db:"after_tree_c"` + AfterTreeR bool `db:"after_tree_r"` + TreeDCid *string `db:"tree_d_cid"` + TreeRCid *string `db:"tree_r_cid"` + Failed bool `db:"failed"` + FailedReasonMsg string `db:"failed_reason_msg"` + } + + err = sm.db.Select(r.Context(), &rows, `SELECT task_id_sdr, ticket_epoch, ticket_value, after_sdr, after_tree_c, after_tree_r, tree_d_cid, tree_r_cid, failed, failed_reason_msg FROM rseal_provider_pipeline WHERE sp_id = $1 AND sector_number = $2 AND partner_id = $3`, + req.SpID, req.SectorNumber, partnerID) + if err != nil { + log.Errorw("status: db query failed", "error", err) + http.Error(w, "internal error", http.StatusInternalServerError) + return + } + + if len(rows) == 0 { + // Sector not found on the provider — return a structured "gone" state + // so the client poll task can detect this and fail the sector gracefully. + writeJSON(w, http.StatusOK, StatusResponse{State: "gone"}) + return + } + + row := rows[0] + resp := StatusResponse{} + + if row.Failed { + resp.State = "failed" + resp.FailReason = row.FailedReasonMsg + } else if row.AfterTreeR && row.AfterTreeC { + resp.State = "complete" + if row.TreeDCid != nil { + resp.TreeDCid = *row.TreeDCid + } + if row.TreeRCid != nil { + resp.TreeRCid = *row.TreeRCid + } + if row.TicketEpoch != nil { + resp.TicketEpoch = *row.TicketEpoch + resp.TicketValue = row.TicketValue + } + } else if row.AfterSDR { + resp.State = "trees" + } else if row.TaskIDSdr != nil { + resp.State = "sdr" + } else { + resp.State = "pending" + } + + writeJSON(w, http.StatusOK, resp) +} + +// handleComplete is called by the provider to notify the client that SDR+trees are done. +// Idempotent: if already marked complete, returns 200 OK without error. +// POST /remoteseal/delegated/v0/complete +func (sm *SealMarket) handleComplete(w http.ResponseWriter, r *http.Request) { + var req CompleteNotification + if !readJSON(w, r, &req) { + return + } + + // Validate partner token by looking up in rseal_client_providers + var providers []struct { + ID int64 `db:"id"` + } + + err := sm.db.Select(r.Context(), &providers, `SELECT id FROM rseal_client_providers WHERE provider_token = $1`, req.PartnerToken) + if err != nil { + log.Errorw("complete: db query failed", "error", err) + http.Error(w, "internal error", http.StatusInternalServerError) + return + } + + if len(providers) == 0 { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + + providerID := providers[0].ID + + // Check if already complete + var existing []struct { + AfterSDR bool `db:"after_sdr"` + } + + err = sm.db.Select(r.Context(), &existing, `SELECT after_sdr FROM rseal_client_pipeline WHERE sp_id = $1 AND sector_number = $2 AND provider_id = $3`, req.SpID, req.SectorNumber, providerID) + if err != nil { + log.Errorw("complete: db query failed", "error", err) + http.Error(w, "internal error", http.StatusInternalServerError) + return + } + + if len(existing) == 0 { + http.Error(w, "sector not found", http.StatusNotFound) + return + } + + // Already complete - idempotent + if existing[0].AfterSDR { + w.WriteHeader(http.StatusOK) + return + } + + // Apply the completion using shared function (ticket comes from the provider notification) + err = ApplyRemoteCompletion(r.Context(), sm.db, req.SpID, req.SectorNumber, providerID, + req.TreeDCid, req.TreeRCid, req.TicketEpoch, req.TicketValue) + if err != nil { + log.Errorw("complete: transaction failed", "error", err) + http.Error(w, "internal error", http.StatusInternalServerError) + return + } + + log.Infow("remote seal complete notification applied", + "sp_id", req.SpID, "sector", req.SectorNumber, + "tree_d_cid", req.TreeDCid, "tree_r_cid", req.TreeRCid) + + w.WriteHeader(http.StatusOK) +} + +// handleTicket returns seal randomness from the client's chain node. +// The provider calls this when sealing a remote sector so it uses the correct +// chain's randomness (the client may be on a different network than the provider). +// Stateless, no auth required — the randomness is public chain data. +// GET /remoteseal/delegated/v0/ticket?maddr=f0XXXX +func (sm *SealMarket) handleTicket(w http.ResponseWriter, r *http.Request) { + if sm.api == nil { + http.Error(w, "ticket endpoint not available (no chain API)", http.StatusServiceUnavailable) + return + } + + maddrStr := r.URL.Query().Get("maddr") + if maddrStr == "" { + http.Error(w, "missing maddr query parameter", http.StatusBadRequest) + return + } + + maddr, err := address.NewFromString(maddrStr) + if err != nil { + http.Error(w, fmt.Sprintf("invalid maddr: %v", err), http.StatusBadRequest) + return + } + + ticket, epoch, err := seal.GetTicket(r.Context(), sm.api, maddr) + if err != nil { + log.Errorw("ticket: GetTicket failed", "error", err, "maddr", maddrStr) + http.Error(w, "failed to get ticket from chain", http.StatusInternalServerError) + return + } + + resp := TicketResponse{ + Ticket: ticket, + Epoch: int64(epoch), + } + + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(resp); err != nil { + log.Errorw("ticket: encoding response failed", "error", err) + } +} + +// handleSealedData streams the sealed sector file (32 GiB) to the client. +// Supports HTTP Range headers for resumable downloads (aria2c compatible). +// Auth via ?token= query param (GET can't have body for proper Range support). +// GET /remoteseal/delegated/v0/sealed-data/{sp_id}/{sector_number}?token=... +func (sm *SealMarket) handleSealedData(w http.ResponseWriter, r *http.Request) { + spID, sectorNumber, ok := parseSectorPathParams(w, r) + if !ok { + return + } + token := r.URL.Query().Get("token") + + // Validate partner token + partnerID, err := sm.validatePartnerToken(r.Context(), token) + if err != nil { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + + // Look up the sector in rseal_provider_pipeline to get the proof type + var sectors []struct { + RegSealProof int `db:"reg_seal_proof"` + } + + err = sm.db.Select(r.Context(), §ors, `SELECT reg_seal_proof FROM rseal_provider_pipeline WHERE sp_id = $1 AND sector_number = $2 AND partner_id = $3`, spID, sectorNumber, partnerID) + if err != nil { + log.Errorw("sealed-data: db query failed", "error", err) + http.Error(w, "internal error", http.StatusInternalServerError) + return + } + + if len(sectors) == 0 { + http.Error(w, "sector not found", http.StatusNotFound) + return + } + + // Build SectorRef + sref := storiface.SectorRef{ + ID: abi.SectorID{ + Miner: abi.ActorID(spID), + Number: abi.SectorNumber(sectorNumber), + }, + ProofType: abi.RegisteredSealProof(sectors[0].RegSealProof), + } + + ssize, err := sref.ProofType.SectorSize() + if err != nil { + log.Errorw("sealed-data: get sector size failed", "error", err) + http.Error(w, "internal error", http.StatusInternalServerError) + return + } + + readerFactory, err := sm.paths.ReaderPiece(r.Context(), sref, storiface.FTSealed, 0, int64(ssize)) + if err != nil { + log.Errorw("sealed-data: get reader failed", "error", err) + http.Error(w, "sector data not available", http.StatusInternalServerError) + return + } + if readerFactory == nil { + http.Error(w, "sector data not found in storage", http.StatusNotFound) + return + } + + rs := &readerPieceReadSeeker{factory: readerFactory, size: int64(ssize)} + defer func() { _ = rs.Close() }() + + // http.ServeContent handles Range headers automatically + http.ServeContent(w, r, "", time.Time{}, rs) +} + +// handleCacheData streams the finalized cache (p_aux, t_aux, tree-r-last) as a tar archive. +// Uses FinCacheFileConstraints from tarutil (~73 MiB total). +// Auth via ?token= query param. +// GET /remoteseal/delegated/v0/cache-data/{sp_id}/{sector_number}?token=... +func (sm *SealMarket) handleCacheData(w http.ResponseWriter, r *http.Request) { + spID, sectorNumber, ok := parseSectorPathParams(w, r) + if !ok { + return + } + token := r.URL.Query().Get("token") + + // Validate partner token + partnerID, err := sm.validatePartnerToken(r.Context(), token) + if err != nil { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + + // Look up the sector in rseal_provider_pipeline to get the proof type + var sectors []struct { + RegSealProof int `db:"reg_seal_proof"` + } + + err = sm.db.Select(r.Context(), §ors, `SELECT reg_seal_proof FROM rseal_provider_pipeline WHERE sp_id = $1 AND sector_number = $2 AND partner_id = $3`, spID, sectorNumber, partnerID) + if err != nil { + log.Errorw("cache-data: db query failed", "error", err) + http.Error(w, "internal error", http.StatusInternalServerError) + return + } + + if len(sectors) == 0 { + http.Error(w, "sector not found", http.StatusNotFound) + return + } + + // Build SectorRef + sref := storiface.SectorRef{ + ID: abi.SectorID{ + Miner: abi.ActorID(spID), + Number: abi.SectorNumber(sectorNumber), + }, + ProofType: abi.RegisteredSealProof(sectors[0].RegSealProof), + } + + w.Header().Set("Content-Type", "application/x-tar") + w.WriteHeader(http.StatusOK) // no seek + + err = sm.paths.ReadMinCacheInto(r.Context(), sref, storiface.FTCache, w) + if err != nil { + log.Errorw("cache-data: read min cache into failed", "error", err) + return + } +} + +// handleCommit1 accepts the C1 seed from the client and returns C1 output. +// POST /remoteseal/delegated/v0/commit1 +func (sm *SealMarket) handleCommit1(w http.ResponseWriter, r *http.Request) { + var req Commit1Request + if !readJSON(w, r, &req) { + return + } + + // Validate partner token + partnerID, err := sm.validatePartnerToken(r.Context(), req.PartnerToken) + if err != nil { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + + // Look up sector in rseal_provider_pipeline + var sectors []struct { + RegSealProof int `db:"reg_seal_proof"` + TicketEpoch *int64 `db:"ticket_epoch"` + TicketValue []byte `db:"ticket_value"` + TreeDCid *string `db:"tree_d_cid"` + TreeRCid *string `db:"tree_r_cid"` + AfterTreeR bool `db:"after_tree_r"` + } + + err = sm.db.Select(r.Context(), §ors, `SELECT reg_seal_proof, ticket_epoch, ticket_value, tree_d_cid, tree_r_cid, after_tree_r FROM rseal_provider_pipeline WHERE sp_id = $1 AND sector_number = $2 AND partner_id = $3`, + req.SpID, req.SectorNumber, partnerID) + if err != nil { + log.Errorw("commit1: db query failed", "error", err) + http.Error(w, "internal error", http.StatusInternalServerError) + return + } + + if len(sectors) == 0 { + http.Error(w, "sector not found", http.StatusNotFound) + return + } + + sector := sectors[0] + + if !sector.AfterTreeR { + http.Error(w, "sector trees not yet complete", http.StatusPreconditionFailed) + return + } + + if sector.TreeDCid == nil || sector.TreeRCid == nil { + http.Error(w, "sector CIDs not available", http.StatusPreconditionFailed) + return + } + + // Parse CIDs + sealedCID, err := cid.Decode(*sector.TreeRCid) + if err != nil { + log.Errorw("commit1: invalid tree_r_cid", "error", err, "cid", *sector.TreeRCid) + http.Error(w, "invalid tree_r_cid", http.StatusInternalServerError) + return + } + + unsealedCID, err := cid.Decode(*sector.TreeDCid) + if err != nil { + log.Errorw("commit1: invalid tree_d_cid", "error", err, "cid", *sector.TreeDCid) + http.Error(w, "invalid tree_d_cid", http.StatusInternalServerError) + return + } + + // Build SectorRef + sref := storiface.SectorRef{ + ID: abi.SectorID{ + Miner: abi.ActorID(req.SpID), + Number: abi.SectorNumber(req.SectorNumber), + }, + ProofType: abi.RegisteredSealProof(sector.RegSealProof), + } + + computeStart := time.Now() + + // Compute the vanilla proof (C1) + vanillaProof, err := sm.paths.GeneratePoRepVanillaProof( + r.Context(), + sref, + sealedCID, + unsealedCID, + abi.SealRandomness(sector.TicketValue), + abi.InteractiveSealRandomness(req.SeedValue), + ) + if err != nil { + log.Errorw("commit1: GeneratePoRepVanillaProof failed", "error", err) + http.Error(w, "failed to compute C1", http.StatusInternalServerError) + return + } + + stats.Record(r.Context(), RsealCommit1ComputeDuration.M(float64(time.Since(computeStart).Milliseconds()))) + + // Mark after_c1_supplied = TRUE + _, err = sm.db.Exec(r.Context(), `UPDATE rseal_provider_pipeline SET after_c1_supplied = TRUE WHERE sp_id = $1 AND sector_number = $2 AND partner_id = $3`, + req.SpID, req.SectorNumber, partnerID) + if err != nil { + log.Errorw("commit1: failed to update after_c1_supplied", "error", err) + http.Error(w, "internal error", http.StatusInternalServerError) + return + } + + // Write raw bytes — C1 output is large (~50-128 MiB), JSON+base64 adds ~33% overhead + w.Header().Set("Content-Type", "application/octet-stream") + w.WriteHeader(http.StatusOK) + if _, err := w.Write(vanillaProof); err != nil { + log.Errorw("commit1: failed to write response", "error", err) + } +} + +// handleFinalize tells the provider that layers can be dropped (sealed data fetched). +// POST /remoteseal/delegated/v0/finalize +func (sm *SealMarket) handleFinalize(w http.ResponseWriter, r *http.Request) { + var req FinalizeRequest + if !readJSON(w, r, &req) { + return + } + + // Validate partner token + partnerID, err := sm.validatePartnerToken(r.Context(), req.PartnerToken) + if err != nil { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + + // Mark after_c1_supplied = TRUE if not already set. + // The finalize task in the provider poller starts when after_c1_supplied is TRUE. + // If /commit1 was already called, this is a no-op. + _, err = sm.db.Exec(r.Context(), `UPDATE rseal_provider_pipeline SET after_c1_supplied = TRUE WHERE sp_id = $1 AND sector_number = $2 AND partner_id = $3 AND after_c1_supplied = FALSE`, + req.SpID, req.SectorNumber, partnerID) + if err != nil { + log.Errorw("finalize: failed to update", "error", err) + http.Error(w, "internal error", http.StatusInternalServerError) + return + } + + w.WriteHeader(http.StatusOK) +} + +// handleCleanup tells the provider to fully clean up all sector data. +// Idempotent: if already cleaned up or cleanup already requested, returns 200 OK. +// POST /remoteseal/delegated/v0/cleanup +func (sm *SealMarket) handleCleanup(w http.ResponseWriter, r *http.Request) { + var req CleanupRequest + if !readJSON(w, r, &req) { + return + } + + // Validate partner token + partnerID, err := sm.validatePartnerToken(r.Context(), req.PartnerToken) + if err != nil { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + + // Set cleanup_requested = true (no-op if already set) + _, err = sm.db.Exec(r.Context(), `UPDATE rseal_provider_pipeline SET cleanup_requested = TRUE WHERE sp_id = $1 AND sector_number = $2 AND partner_id = $3`, + req.SpID, req.SectorNumber, partnerID) + if err != nil { + log.Errorw("cleanup: failed to update", "error", err) + http.Error(w, "internal error", http.StatusInternalServerError) + return + } + + w.WriteHeader(http.StatusOK) +} + +// --- Shared functions --- + +// ApplyRemoteCompletion updates rseal_client_pipeline and creates/updates the +// sectors_sdr_pipeline row when a remote provider completes SDR+trees. Called +// by both the poll task (in remoteseal package) and the /complete callback. +// The ticket data comes from the provider (via notification or status poll). +func ApplyRemoteCompletion(ctx context.Context, db *harmonydb.DB, spID, sectorNumber, providerID int64, treeDCid, treeRCid string, ticketEpoch int64, ticketValue []byte) error { + _, err := db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (bool, error) { + // Update rseal_client_pipeline: mark remote SDR+trees as done + n, err := tx.Exec(` + UPDATE rseal_client_pipeline + SET after_sdr = TRUE, task_id_poll = NULL + WHERE sp_id = $1 AND sector_number = $2`, + spID, sectorNumber) + if err != nil { + return false, xerrors.Errorf("updating rseal_client_pipeline: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("expected to update 1 rseal_client_pipeline row, updated %d", n) + } + + // Read reg_seal_proof and user_sector_duration_epochs from rseal_client_pipeline + // to carry through to sectors_sdr_pipeline. + var clientInfo struct { + RegSealProof int `db:"reg_seal_proof"` + UserSectorDurationEpochs *int64 `db:"user_sector_duration_epochs"` + } + err = tx.QueryRow(`SELECT reg_seal_proof, user_sector_duration_epochs + FROM rseal_client_pipeline WHERE sp_id = $1 AND sector_number = $2`, + spID, sectorNumber).Scan(&clientInfo.RegSealProof, &clientInfo.UserSectorDurationEpochs) + if err != nil { + return false, xerrors.Errorf("reading rseal_client_pipeline for completion: %w", err) + } + + // Create or update sectors_sdr_pipeline: mark SDR, trees, and synth as done. + // For new remote CC sectors, this INSERT creates the row for the first time. + // For existing sectors (delegated from the local pipeline), this updates the row. + // after_synth = TRUE because remote-sealed sectors skip the local synth step. + n, err = tx.Exec(` + INSERT INTO sectors_sdr_pipeline (sp_id, sector_number, reg_seal_proof, user_sector_duration_epochs, + after_sdr, after_tree_d, after_tree_c, after_tree_r, after_synth, + tree_d_cid, tree_r_cid, ticket_epoch, ticket_value) + VALUES ($1, $2, $3, $4, TRUE, TRUE, TRUE, TRUE, TRUE, $5, $6, $7, $8) + ON CONFLICT (sp_id, sector_number) DO UPDATE SET + after_sdr = TRUE, after_tree_d = TRUE, after_tree_c = TRUE, after_tree_r = TRUE, + after_synth = TRUE, + tree_d_cid = $5, tree_r_cid = $6, + ticket_epoch = $7, ticket_value = $8, + task_id_sdr = NULL, task_id_tree_d = NULL, task_id_tree_c = NULL, task_id_tree_r = NULL`, + spID, sectorNumber, clientInfo.RegSealProof, clientInfo.UserSectorDurationEpochs, + treeDCid, treeRCid, ticketEpoch, ticketValue) + if err != nil { + return false, xerrors.Errorf("upserting sectors_sdr_pipeline: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("expected to upsert 1 sectors_sdr_pipeline row, affected %d", n) + } + + return true, nil + }, harmonydb.OptionRetry()) + if err != nil { + return xerrors.Errorf("applying remote completion transaction: %w", err) + } + + return nil +} + +// --- Helpers --- + +// validatePartnerToken checks the partner_token against rseal_delegated_partners +// and returns the partner ID if valid. +func (sm *SealMarket) validatePartnerToken(ctx context.Context, token string) (int64, error) { + var partners []struct { + ID int64 `db:"id"` + } + + err := sm.db.Select(ctx, &partners, `SELECT id FROM rseal_delegated_partners WHERE partner_token = $1`, token) + if err != nil { + return 0, xerrors.Errorf("querying partner token: %w", err) + } + + if len(partners) == 0 { + return 0, xerrors.Errorf("partner token not found") + } + + return partners[0].ID, nil +} + +func parseSectorPathParams(w http.ResponseWriter, r *http.Request) (spID int64, sectorNumber int64, ok bool) { + spIDStr := chi.URLParam(r, "sp_id") + sectorNumberStr := chi.URLParam(r, "sector_number") + + spID, err := strconv.ParseInt(spIDStr, 10, 64) + if err != nil { + http.Error(w, "invalid sp_id", http.StatusBadRequest) + return 0, 0, false + } + + sectorNumber, err = strconv.ParseInt(sectorNumberStr, 10, 64) + if err != nil { + http.Error(w, "invalid sector_number", http.StatusBadRequest) + return 0, 0, false + } + + return spID, sectorNumber, true +} + +func readJSON(w http.ResponseWriter, r *http.Request, v interface{}) bool { + if r.Header.Get("Content-Type") != "application/json" { + http.Error(w, "Content-Type must be application/json", http.StatusBadRequest) + return false + } + dec := json.NewDecoder(r.Body) + if err := dec.Decode(v); err != nil { + http.Error(w, "invalid request body: "+err.Error(), http.StatusBadRequest) + return false + } + return true +} + +func writeJSON(w http.ResponseWriter, status int, v interface{}) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + if err := json.NewEncoder(w).Encode(v); err != nil { + log.Errorw("failed to write JSON response", "error", err) + } +} diff --git a/market/sealmarket/sealapi_test.go b/market/sealmarket/sealapi_test.go new file mode 100644 index 000000000..b0a8ab1ad --- /dev/null +++ b/market/sealmarket/sealapi_test.go @@ -0,0 +1,592 @@ +package sealmarket + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "testing" + "time" + + "github.com/go-chi/chi/v5" + "github.com/stretchr/testify/require" + + "github.com/filecoin-project/curio/harmony/harmonydb" + "github.com/filecoin-project/curio/lib/testutil/dbtest" +) + +// sharedDB is initialized once in TestMain and reused by all tests, +// avoiding the ~65s schema-migration cost per test. +var sharedDB *harmonydb.DB + +func TestMain(m *testing.M) { + code := dbtest.StartYugabyte(m) + // Note: StartYugabyte calls m.Run() internally, so 'code' is the test result. + // sharedDB cleanup happens inside StartYugabyte's scope (DB is closed on exit). + os.Exit(code) +} + +// initSharedDB lazily creates the shared DB on first use. It is called +// by setupHarness. A sync.Once is not needed because Go tests run +// sequentially by default within a package. +func initSharedDB(t *testing.T) { + t.Helper() + if sharedDB != nil { + return + } + + envOr := func(env, fallback string) string { + if v := os.Getenv(env); v != "" { + return v + } + return fallback + } + + iTestID := harmonydb.ITestNewID() + + db, err := harmonydb.NewFromConfig(harmonydb.Config{ + Hosts: []string{envOr("CURIO_HARMONYDB_HOSTS", "127.0.0.1")}, + Database: envOr("CURIO_HARMONYDB_DB", "yugabyte"), + Username: "yugabyte", + Password: "yugabyte", + Port: envOr("CURIO_HARMONYDB_PORT", "5433"), + ITestID: iTestID, + }) + if err != nil { + fmt.Fprintf(os.Stderr, "sealapi_test: failed to create shared DB: %v\n", err) + t.Fatalf("failed to create shared DB: %v", err) + } + + sharedDB = db + + // Clean up only when the entire test binary finishes. + // We cannot use t.Cleanup because the first test's cleanup + // would close the pool before subsequent tests run. + // Instead we rely on the process exiting to close connections. + // ITestDeleteAll is nice-to-have but the container is torn down anyway. +} + +// testHarness bundles a SealMarket, chi router, and DB handle for API tests. +type testHarness struct { + t *testing.T + db *harmonydb.DB + sm *SealMarket + router *chi.Mux +} + +// setupHarness creates a test SealMarket backed by a real YugabyteDB. +// SealCalls (sc) is nil — none of the auth/constraint endpoints need it. +// All tests share one DB connection; each gets a fresh SealMarket (fresh +// in-memory slot map). Tests use unique partner tokens to avoid collisions. +func setupHarness(t *testing.T) *testHarness { + t.Helper() + initSharedDB(t) + + sm := NewSealMarket(sharedDB, nil, nil) + r := chi.NewMux() + Routes(r, sm) + + return &testHarness{t: t, db: sharedDB, sm: sm, router: r} +} + +// seedPartner inserts a partner row and returns its auto-generated id. +func (h *testHarness) seedPartner(token, name string, allowance int64) int64 { + h.t.Helper() + + var ids []struct { + ID int64 `db:"id"` + } + err := h.db.Select(context.Background(), &ids, + `INSERT INTO rseal_delegated_partners (partner_token, partner_name, partner_url, allowance_remaining, allowance_total) + VALUES ($1, $2, 'http://test', $3, $3) RETURNING id`, + token, name, allowance) + require.NoError(h.t, err) + require.Len(h.t, ids, 1) + return ids[0].ID +} + +// getAllowance reads the current allowance_remaining for a partner. +func (h *testHarness) getAllowance(partnerID int64) int64 { + h.t.Helper() + + var rows []struct { + Remaining int64 `db:"allowance_remaining"` + } + err := h.db.Select(context.Background(), &rows, + `SELECT allowance_remaining FROM rseal_delegated_partners WHERE id = $1`, partnerID) + require.NoError(h.t, err) + require.Len(h.t, rows, 1) + return rows[0].Remaining +} + +// postJSON sends a POST with JSON body to the router and returns the recorder. +func (h *testHarness) postJSON(path string, body interface{}) *httptest.ResponseRecorder { + h.t.Helper() + + b, err := json.Marshal(body) + require.NoError(h.t, err) + + req := httptest.NewRequest(http.MethodPost, path, bytes.NewReader(b)) + req.Header.Set("Content-Type", "application/json") + rec := httptest.NewRecorder() + h.router.ServeHTTP(rec, req) + return rec +} + +// decodeJSON unmarshals a recorder's body into v. +func decodeJSON(t *testing.T, rec *httptest.ResponseRecorder, v interface{}) { + t.Helper() + require.NoError(t, json.NewDecoder(rec.Body).Decode(v)) +} + +// --- Authorize tests --- + +func TestAuthorize_ValidToken(t *testing.T) { + h := setupHarness(t) + h.seedPartner("tok-valid", "partner-A", 10) + + rec := h.postJSON(DelegatedSealPath+"authorize", AuthorizeRequest{PartnerToken: "tok-valid"}) + require.Equal(t, http.StatusOK, rec.Code) + + var resp AuthorizeResponse + decodeJSON(t, rec, &resp) + require.True(t, resp.Authorized) + require.Equal(t, "partner-A", resp.PartnerName) + require.Equal(t, int64(10), resp.AllowanceRemaining) +} + +func TestAuthorize_InvalidToken(t *testing.T) { + h := setupHarness(t) + + rec := h.postJSON(DelegatedSealPath+"authorize", AuthorizeRequest{PartnerToken: "tok-bogus"}) + require.Equal(t, http.StatusOK, rec.Code) + + var resp AuthorizeResponse + decodeJSON(t, rec, &resp) + require.False(t, resp.Authorized) +} + +// --- Auth rejection (401) on endpoints that call validatePartnerToken --- + +func TestAvailable_InvalidToken_401(t *testing.T) { + h := setupHarness(t) + + rec := h.postJSON(DelegatedSealPath+"available", AuthorizeRequest{PartnerToken: "bad"}) + require.Equal(t, http.StatusUnauthorized, rec.Code) +} + +func TestOrder_InvalidToken_401(t *testing.T) { + h := setupHarness(t) + + rec := h.postJSON(DelegatedSealPath+"order", OrderRequest{PartnerToken: "bad", SlotToken: "x", SpID: 1, SectorNumber: 1, RegSealProof: 8}) + require.Equal(t, http.StatusUnauthorized, rec.Code) +} + +func TestStatus_InvalidToken_401(t *testing.T) { + h := setupHarness(t) + + rec := h.postJSON(DelegatedSealPath+"status", StatusRequest{PartnerToken: "bad", SpID: 1, SectorNumber: 1}) + require.Equal(t, http.StatusUnauthorized, rec.Code) +} + +func TestFinalize_InvalidToken_401(t *testing.T) { + h := setupHarness(t) + + rec := h.postJSON(DelegatedSealPath+"finalize", FinalizeRequest{PartnerToken: "bad", SpID: 1, SectorNumber: 1}) + require.Equal(t, http.StatusUnauthorized, rec.Code) +} + +func TestCleanup_InvalidToken_401(t *testing.T) { + h := setupHarness(t) + + rec := h.postJSON(DelegatedSealPath+"cleanup", CleanupRequest{PartnerToken: "bad", SpID: 1, SectorNumber: 1}) + require.Equal(t, http.StatusUnauthorized, rec.Code) +} + +func TestCommit1_InvalidToken_401(t *testing.T) { + h := setupHarness(t) + + rec := h.postJSON(DelegatedSealPath+"commit1", Commit1Request{PartnerToken: "bad", SpID: 1, SectorNumber: 1}) + require.Equal(t, http.StatusUnauthorized, rec.Code) +} + +// --- Available / quota tests --- + +func TestAvailable_WithCapacity(t *testing.T) { + h := setupHarness(t) + h.seedPartner("tok-cap", "partner-B", 5) + + rec := h.postJSON(DelegatedSealPath+"available", AuthorizeRequest{PartnerToken: "tok-cap"}) + require.Equal(t, http.StatusOK, rec.Code) + + var resp AvailableResponse + decodeJSON(t, rec, &resp) + require.True(t, resp.Available) + require.NotEmpty(t, resp.SlotToken) +} + +func TestAvailable_QuotaExhausted(t *testing.T) { + h := setupHarness(t) + // allowance_remaining = 0 means no capacity + h.seedPartner("tok-full", "partner-C", 0) + + rec := h.postJSON(DelegatedSealPath+"available", AuthorizeRequest{PartnerToken: "tok-full"}) + require.Equal(t, http.StatusOK, rec.Code) + + var resp AvailableResponse + decodeJSON(t, rec, &resp) + require.False(t, resp.Available) + require.Empty(t, resp.SlotToken) +} + +func TestAvailable_QuotaExhaustedByActiveSectors(t *testing.T) { + h := setupHarness(t) + // allowance_remaining = 1, but we'll insert one active pipeline row + pid := h.seedPartner("tok-active", "partner-D", 1) + + // Insert an active (non-cleaned-up) sector into rseal_provider_pipeline + _, err := h.db.Exec(context.Background(), + `INSERT INTO rseal_provider_pipeline (partner_id, sp_id, sector_number, reg_seal_proof) VALUES ($1, 100, 1, 8)`, + pid) + require.NoError(t, err) + + rec := h.postJSON(DelegatedSealPath+"available", AuthorizeRequest{PartnerToken: "tok-active"}) + require.Equal(t, http.StatusOK, rec.Code) + + var resp AvailableResponse + decodeJSON(t, rec, &resp) + require.False(t, resp.Available) +} + +// --- Order + quota decrement tests --- + +func TestOrder_AcceptedAndAllowanceDecremented(t *testing.T) { + h := setupHarness(t) + pid := h.seedPartner("tok-order", "partner-E", 5) + + // Get a slot token first + rec := h.postJSON(DelegatedSealPath+"available", AuthorizeRequest{PartnerToken: "tok-order"}) + require.Equal(t, http.StatusOK, rec.Code) + + var avail AvailableResponse + decodeJSON(t, rec, &avail) + require.True(t, avail.Available) + + // Place the order + rec = h.postJSON(DelegatedSealPath+"order", OrderRequest{ + PartnerToken: "tok-order", + SlotToken: avail.SlotToken, + SpID: 200, + SectorNumber: 1, + RegSealProof: 8, + }) + require.Equal(t, http.StatusOK, rec.Code) + + var orderResp OrderResponse + decodeJSON(t, rec, &orderResp) + require.True(t, orderResp.Accepted) + + // Verify allowance went from 5 to 4 + require.Equal(t, int64(4), h.getAllowance(pid)) +} + +func TestOrder_Idempotent_NoDuplicateDecrement(t *testing.T) { + h := setupHarness(t) + pid := h.seedPartner("tok-idem", "partner-F", 5) + + // First order + rec := h.postJSON(DelegatedSealPath+"available", AuthorizeRequest{PartnerToken: "tok-idem"}) + var avail1 AvailableResponse + decodeJSON(t, rec, &avail1) + require.True(t, avail1.Available) + + rec = h.postJSON(DelegatedSealPath+"order", OrderRequest{ + PartnerToken: "tok-idem", + SlotToken: avail1.SlotToken, + SpID: 300, + SectorNumber: 1, + RegSealProof: 8, + }) + require.Equal(t, http.StatusOK, rec.Code) + var resp1 OrderResponse + decodeJSON(t, rec, &resp1) + require.True(t, resp1.Accepted) + require.Equal(t, int64(4), h.getAllowance(pid)) + + // Second order with same (sp_id, sector_number) — should be idempotent. + // Get a new slot token. + rec = h.postJSON(DelegatedSealPath+"available", AuthorizeRequest{PartnerToken: "tok-idem"}) + var avail2 AvailableResponse + decodeJSON(t, rec, &avail2) + require.True(t, avail2.Available) + + rec = h.postJSON(DelegatedSealPath+"order", OrderRequest{ + PartnerToken: "tok-idem", + SlotToken: avail2.SlotToken, + SpID: 300, + SectorNumber: 1, + RegSealProof: 8, + }) + require.Equal(t, http.StatusOK, rec.Code) + var resp2 OrderResponse + decodeJSON(t, rec, &resp2) + require.True(t, resp2.Accepted) + + // Allowance should still be 4, not 3 + require.Equal(t, int64(4), h.getAllowance(pid)) +} + +func TestOrder_MultipleOrders_AllowanceDecrementsCorrectly(t *testing.T) { + h := setupHarness(t) + pid := h.seedPartner("tok-multi", "partner-G", 10) + + for i := int64(1); i <= 3; i++ { + rec := h.postJSON(DelegatedSealPath+"available", AuthorizeRequest{PartnerToken: "tok-multi"}) + var avail AvailableResponse + decodeJSON(t, rec, &avail) + require.True(t, avail.Available) + + rec = h.postJSON(DelegatedSealPath+"order", OrderRequest{ + PartnerToken: "tok-multi", + SlotToken: avail.SlotToken, + SpID: 400, + SectorNumber: i, + RegSealProof: 8, + }) + require.Equal(t, http.StatusOK, rec.Code) + + var orderResp OrderResponse + decodeJSON(t, rec, &orderResp) + require.True(t, orderResp.Accepted) + } + + // 10 - 3 = 7 + require.Equal(t, int64(7), h.getAllowance(pid)) +} + +// --- Slot token validation tests --- + +func TestOrder_ExpiredSlotToken(t *testing.T) { + h := setupHarness(t) + h.seedPartner("tok-exp", "partner-H", 5) + + // Get a slot token + rec := h.postJSON(DelegatedSealPath+"available", AuthorizeRequest{PartnerToken: "tok-exp"}) + var avail AvailableResponse + decodeJSON(t, rec, &avail) + require.True(t, avail.Available) + + // Manually expire the slot by setting its deadline in the past + h.sm.slotsMu.Lock() + entry, ok := h.sm.slots[avail.SlotToken] + require.True(t, ok) + entry.deadline = time.Now().Add(-1 * time.Second) + h.sm.slotsMu.Unlock() + + rec = h.postJSON(DelegatedSealPath+"order", OrderRequest{ + PartnerToken: "tok-exp", + SlotToken: avail.SlotToken, + SpID: 500, + SectorNumber: 1, + RegSealProof: 8, + }) + require.Equal(t, http.StatusOK, rec.Code) + + var orderResp OrderResponse + decodeJSON(t, rec, &orderResp) + require.False(t, orderResp.Accepted) + require.Contains(t, orderResp.RejectReason, "invalid or expired slot token") +} + +func TestOrder_MismatchedPartnerSlotToken(t *testing.T) { + h := setupHarness(t) + h.seedPartner("tok-mm-A", "partner-mm-A", 5) + h.seedPartner("tok-mm-B", "partner-mm-B", 5) + + // Get slot token for partner A + rec := h.postJSON(DelegatedSealPath+"available", AuthorizeRequest{PartnerToken: "tok-mm-A"}) + var avail AvailableResponse + decodeJSON(t, rec, &avail) + require.True(t, avail.Available) + + // Try to use partner A's slot token with partner B's auth token + rec = h.postJSON(DelegatedSealPath+"order", OrderRequest{ + PartnerToken: "tok-mm-B", + SlotToken: avail.SlotToken, + SpID: 600, + SectorNumber: 1, + RegSealProof: 8, + }) + require.Equal(t, http.StatusOK, rec.Code) + + var orderResp OrderResponse + decodeJSON(t, rec, &orderResp) + require.False(t, orderResp.Accepted) + require.Contains(t, orderResp.RejectReason, "invalid or expired slot token") +} + +func TestOrder_ReusedSlotToken(t *testing.T) { + h := setupHarness(t) + h.seedPartner("tok-reuse", "partner-I", 5) + + // Get a slot token + rec := h.postJSON(DelegatedSealPath+"available", AuthorizeRequest{PartnerToken: "tok-reuse"}) + var avail AvailableResponse + decodeJSON(t, rec, &avail) + require.True(t, avail.Available) + + // First order — consumes the slot token + rec = h.postJSON(DelegatedSealPath+"order", OrderRequest{ + PartnerToken: "tok-reuse", + SlotToken: avail.SlotToken, + SpID: 700, + SectorNumber: 1, + RegSealProof: 8, + }) + require.Equal(t, http.StatusOK, rec.Code) + var resp1 OrderResponse + decodeJSON(t, rec, &resp1) + require.True(t, resp1.Accepted) + + // Second order with same slot token — should be rejected (single-use) + rec = h.postJSON(DelegatedSealPath+"order", OrderRequest{ + PartnerToken: "tok-reuse", + SlotToken: avail.SlotToken, + SpID: 700, + SectorNumber: 2, + RegSealProof: 8, + }) + require.Equal(t, http.StatusOK, rec.Code) + var resp2 OrderResponse + decodeJSON(t, rec, &resp2) + require.False(t, resp2.Accepted) + require.Contains(t, resp2.RejectReason, "invalid or expired slot token") +} + +func TestOrder_InvalidSlotToken(t *testing.T) { + h := setupHarness(t) + h.seedPartner("tok-inv", "partner-J", 5) + + rec := h.postJSON(DelegatedSealPath+"order", OrderRequest{ + PartnerToken: "tok-inv", + SlotToken: "totally-made-up", + SpID: 800, + SectorNumber: 1, + RegSealProof: 8, + }) + require.Equal(t, http.StatusOK, rec.Code) + + var orderResp OrderResponse + decodeJSON(t, rec, &orderResp) + require.False(t, orderResp.Accepted) + require.Contains(t, orderResp.RejectReason, "invalid or expired slot token") +} + +// --- Status returns "gone" state for unknown sector --- + +func TestStatus_UnknownSector_Gone(t *testing.T) { + h := setupHarness(t) + h.seedPartner("tok-stat", "partner-K", 5) + + rec := h.postJSON(DelegatedSealPath+"status", StatusRequest{ + PartnerToken: "tok-stat", + SpID: 999, + SectorNumber: 999, + }) + require.Equal(t, http.StatusOK, rec.Code) + + var status StatusResponse + decodeJSON(t, rec, &status) + require.Equal(t, "gone", status.State) +} + +// --- Status returns pending for newly ordered sector --- + +func TestStatus_PendingSector(t *testing.T) { + h := setupHarness(t) + h.seedPartner("tok-pend", "partner-L", 5) + + // Get slot + order + rec := h.postJSON(DelegatedSealPath+"available", AuthorizeRequest{PartnerToken: "tok-pend"}) + var avail AvailableResponse + decodeJSON(t, rec, &avail) + require.True(t, avail.Available) + + rec = h.postJSON(DelegatedSealPath+"order", OrderRequest{ + PartnerToken: "tok-pend", + SlotToken: avail.SlotToken, + SpID: 900, + SectorNumber: 1, + RegSealProof: 8, + }) + var orderResp OrderResponse + decodeJSON(t, rec, &orderResp) + require.True(t, orderResp.Accepted) + + // Poll status + rec = h.postJSON(DelegatedSealPath+"status", StatusRequest{ + PartnerToken: "tok-pend", + SpID: 900, + SectorNumber: 1, + }) + require.Equal(t, http.StatusOK, rec.Code) + + var status StatusResponse + decodeJSON(t, rec, &status) + require.Equal(t, "pending", status.State) +} + +// --- Capabilities (no auth required) --- + +func TestCapabilities(t *testing.T) { + h := setupHarness(t) + + req := httptest.NewRequest(http.MethodGet, DelegatedSealPath+"capabilities", nil) + rec := httptest.NewRecorder() + h.router.ServeHTTP(rec, req) + require.Equal(t, http.StatusOK, rec.Code) + + var resp CapabilitiesResponse + decodeJSON(t, rec, &resp) + require.True(t, len(resp.SupportedProofs) > 0) + require.True(t, resp.SupportsRangeRequests) +} + +// --- Cross-partner isolation: partner A cannot see partner B's sectors --- + +func TestStatus_CrossPartnerIsolation(t *testing.T) { + h := setupHarness(t) + h.seedPartner("tok-iso-a", "partner-iso-A", 5) + h.seedPartner("tok-iso-b", "partner-iso-B", 5) + + // Partner A orders a sector + rec := h.postJSON(DelegatedSealPath+"available", AuthorizeRequest{PartnerToken: "tok-iso-a"}) + var avail AvailableResponse + decodeJSON(t, rec, &avail) + + rec = h.postJSON(DelegatedSealPath+"order", OrderRequest{ + PartnerToken: "tok-iso-a", + SlotToken: avail.SlotToken, + SpID: 1000, + SectorNumber: 1, + RegSealProof: 8, + }) + var orderResp OrderResponse + decodeJSON(t, rec, &orderResp) + require.True(t, orderResp.Accepted) + + // Partner B tries to check status of partner A's sector — should get "gone" state + rec = h.postJSON(DelegatedSealPath+"status", StatusRequest{ + PartnerToken: "tok-iso-b", + SpID: 1000, + SectorNumber: 1, + }) + require.Equal(t, http.StatusOK, rec.Code) + + var status StatusResponse + decodeJSON(t, rec, &status) + require.Equal(t, "gone", status.State) +} diff --git a/tasks/gc/pipeline_meta_gc.go b/tasks/gc/pipeline_meta_gc.go index fc2b6d6f9..efcbc3569 100644 --- a/tasks/gc/pipeline_meta_gc.go +++ b/tasks/gc/pipeline_meta_gc.go @@ -44,6 +44,9 @@ func (s *PipelineGC) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done if err := s.cleanupPDPPipeline(); err != nil { return false, xerrors.Errorf("cleanupPDPPipeline: %w", err) } + if err := s.cleanupRemoteSealProvider(); err != nil { + return false, xerrors.Errorf("cleanupRemoteSealProvider: %w", err) + } return true, nil } @@ -239,5 +242,19 @@ func (s *PipelineGC) cleanupPDPPipeline() error { return nil } +func (s *PipelineGC) cleanupRemoteSealProvider() error { + // Remove rseal_provider_pipeline entries where cleanup is done and the row + // has been idle for at least 24 hours (gives time for any pending queries). + ctx := context.Background() + _, err := s.db.Exec(ctx, `DELETE FROM rseal_provider_pipeline + WHERE after_cleanup = TRUE + AND cleanup_timeout IS NOT NULL + AND cleanup_timeout < NOW() - INTERVAL '24 hours'`) + if err != nil { + return xerrors.Errorf("failed to clean up remote seal provider entries: %w", err) + } + return nil +} + var _ harmonytask.TaskInterface = &PipelineGC{} var _ = harmonytask.Reg(&PipelineGC{}) diff --git a/tasks/gc/storage_gc_mark.go b/tasks/gc/storage_gc_mark.go index 089e148ce..881dde0d4 100644 --- a/tasks/gc/storage_gc_mark.go +++ b/tasks/gc/storage_gc_mark.go @@ -182,6 +182,25 @@ func (s *StorageGCMark) Do(taskID harmonytask.TaskID, stillOwned func() bool) (d toRemove[abi.ActorID(sector.SpID)].Unset(uint64(sector.SectorNum)) } + + // Also exclude sectors from remote seal provider pipeline + var remotePipelineSectors []struct { + SpID int64 `db:"sp_id"` + SectorNum int64 `db:"sector_number"` + } + + err = tx.Select(&remotePipelineSectors, `SELECT sp_id, sector_number FROM rseal_provider_pipeline WHERE after_cleanup != TRUE`) + if err != nil { + return false, xerrors.Errorf("select remote provider pipeline: %w", err) + } + + for _, sector := range remotePipelineSectors { + if toRemove[abi.ActorID(sector.SpID)] == nil { + continue + } + + toRemove[abi.ActorID(sector.SpID)].Unset(uint64(sector.SectorNum)) + } } if len(toRemove) > 0 { // precommits diff --git a/tasks/remoteseal/client.go b/tasks/remoteseal/client.go new file mode 100644 index 000000000..f4d10e19d --- /dev/null +++ b/tasks/remoteseal/client.go @@ -0,0 +1,154 @@ +package remoteseal + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net/http" + "net/url" + "time" + + "golang.org/x/xerrors" + + "github.com/filecoin-project/curio/market/sealmarket" +) + +// RSealClient is an HTTP client for calling remote seal provider endpoints. +type RSealClient struct { + httpClient *http.Client +} + +// NewRSealClient creates a new RSealClient with default timeout. +func NewRSealClient() *RSealClient { + return &RSealClient{ + httpClient: &http.Client{ + Timeout: 30 * time.Second, + }, + } +} + +// doPost performs a POST request to the given URL with a JSON body and decodes the JSON response. +func (c *RSealClient) doPost(ctx context.Context, url string, reqBody interface{}, respBody interface{}) error { + bodyBytes, err := json.Marshal(reqBody) + if err != nil { + return xerrors.Errorf("marshaling request body: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(bodyBytes)) + if err != nil { + return xerrors.Errorf("creating request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + + resp, err := c.httpClient.Do(req) + if err != nil { + return xerrors.Errorf("performing request to %s: %w", url, err) + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return xerrors.Errorf("unexpected status %d from %s: %s", resp.StatusCode, url, string(body)) + } + + if respBody != nil { + if err := json.NewDecoder(resp.Body).Decode(respBody); err != nil { + return xerrors.Errorf("decoding response from %s: %w", url, err) + } + } + + return nil +} + +// doPostNoResponse performs a POST request expecting only a status code (no JSON body). +func (c *RSealClient) doPostNoResponse(ctx context.Context, url string, reqBody interface{}) error { + bodyBytes, err := json.Marshal(reqBody) + if err != nil { + return xerrors.Errorf("marshaling request body: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(bodyBytes)) + if err != nil { + return xerrors.Errorf("creating request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + + resp, err := c.httpClient.Do(req) + if err != nil { + return xerrors.Errorf("performing request to %s: %w", url, err) + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return xerrors.Errorf("unexpected status %d from %s: %s", resp.StatusCode, url, string(body)) + } + + return nil +} + +// endpoint constructs a full URL from the provider base URL, the delegated seal path, and the endpoint name. +func endpoint(providerURL, ep string) string { + u, err := url.JoinPath(providerURL, sealmarket.DelegatedSealPath, ep) + if err != nil { + // Should never happen with valid URLs; fall back to simple concat. + return providerURL + sealmarket.DelegatedSealPath + ep + } + return u +} + +// CheckAvailable checks if the provider has an available slot. +// POST /remoteseal/delegated/v0/available +func (c *RSealClient) CheckAvailable(ctx context.Context, providerURL, token string) (*sealmarket.AvailableResponse, error) { + reqBody := sealmarket.AuthorizeRequest{ + PartnerToken: token, + } + var resp sealmarket.AvailableResponse + if err := c.doPost(ctx, endpoint(providerURL, "available"), &reqBody, &resp); err != nil { + return nil, xerrors.Errorf("checking available: %w", err) + } + return &resp, nil +} + +// SendOrder sends a seal order to the provider. +// POST /remoteseal/delegated/v0/order +func (c *RSealClient) SendOrder(ctx context.Context, providerURL, token string, req *sealmarket.OrderRequest) (*sealmarket.OrderResponse, error) { + req.PartnerToken = token + var resp sealmarket.OrderResponse + if err := c.doPost(ctx, endpoint(providerURL, "order"), req, &resp); err != nil { + return nil, xerrors.Errorf("sending order: %w", err) + } + return &resp, nil +} + +// GetStatus polls the provider for the status of a remote seal job. +// POST /remoteseal/delegated/v0/status +func (c *RSealClient) GetStatus(ctx context.Context, providerURL, token string, req *sealmarket.StatusRequest) (*sealmarket.StatusResponse, error) { + req.PartnerToken = token + var resp sealmarket.StatusResponse + if err := c.doPost(ctx, endpoint(providerURL, "status"), req, &resp); err != nil { + return nil, xerrors.Errorf("getting status: %w", err) + } + return &resp, nil +} + +// SendFinalize tells the provider that layers can be dropped. +// POST /remoteseal/delegated/v0/finalize +func (c *RSealClient) SendFinalize(ctx context.Context, providerURL, token string, req *sealmarket.FinalizeRequest) error { + req.PartnerToken = token + if err := c.doPostNoResponse(ctx, endpoint(providerURL, "finalize"), req); err != nil { + return xerrors.Errorf("sending finalize: %w", err) + } + return nil +} + +// SendCleanup tells the provider to begin full cleanup of all sector data. +// POST /remoteseal/delegated/v0/cleanup +func (c *RSealClient) SendCleanup(ctx context.Context, providerURL, token string, req *sealmarket.CleanupRequest) error { + req.PartnerToken = token + if err := c.doPostNoResponse(ctx, endpoint(providerURL, "cleanup"), req); err != nil { + return xerrors.Errorf("sending cleanup: %w", err) + } + return nil +} diff --git a/tasks/remoteseal/client_poller.go b/tasks/remoteseal/client_poller.go new file mode 100644 index 000000000..b275fcb6d --- /dev/null +++ b/tasks/remoteseal/client_poller.go @@ -0,0 +1,174 @@ +package remoteseal + +import ( + "context" + "time" + + "golang.org/x/xerrors" + + "github.com/filecoin-project/curio/harmony/harmonydb" + "github.com/filecoin-project/curio/harmony/harmonytask" + "github.com/filecoin-project/curio/lib/promise" +) + +const ( + pollerClientPoll = iota + pollerClientFetch + pollerClientCleanup + + numClientPollers +) + +const rsealClientPollerInterval = 10 * time.Second + +// RSealClientPoller watches rseal_client_pipeline and creates tasks +// for poll, C1 exchange, and cleanup stages. +type RSealClientPoller struct { + db *harmonydb.DB + + pollers [numClientPollers]promise.Promise[harmonytask.AddTaskFunc] +} + +func NewRSealClientPoller(db *harmonydb.DB) *RSealClientPoller { + return &RSealClientPoller{ + db: db, + } +} + +type clientPollTask struct { + SpID int64 `db:"sp_id"` + SectorNumber int64 `db:"sector_number"` + + // client pipeline state + AfterDelegate bool `db:"after_delegate"` + AfterSDR bool `db:"after_sdr"` + AfterFetch bool `db:"after_fetch"` + AfterCleanup bool `db:"after_cleanup"` + Failed bool `db:"failed"` + + TaskIDDelegate *int64 `db:"task_id_delegate"` + TaskIDPoll *int64 `db:"task_id_poll"` + TaskIDFetch *int64 `db:"task_id_fetch"` + TaskIDCleanup *int64 `db:"task_id_cleanup"` + + // from sectors_sdr_pipeline + AfterPoRep bool `db:"after_porep"` +} + +// RunPoller starts the polling loop for the client-side remote seal pipeline. +func (p *RSealClientPoller) RunPoller(ctx context.Context) { + ticker := time.NewTicker(rsealClientPollerInterval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if err := p.poll(ctx); err != nil { + log.Errorw("rseal client polling failed", "error", err) + } + } + } +} + +func (p *RSealClientPoller) poll(ctx context.Context) error { + var tasks []clientPollTask + + err := p.db.Select(ctx, &tasks, ` + SELECT + c.sp_id, + c.sector_number, + c.after_delegate, + c.after_sdr, + c.after_fetch, + c.after_cleanup, + c.failed, + c.task_id_delegate, + c.task_id_poll, + c.task_id_fetch, + c.task_id_cleanup, + COALESCE(s.after_porep, FALSE) AS after_porep + FROM rseal_client_pipeline c + LEFT JOIN sectors_sdr_pipeline s ON c.sp_id = s.sp_id AND c.sector_number = s.sector_number + WHERE c.after_cleanup != TRUE OR c.after_fetch != TRUE`) + if err != nil { + return xerrors.Errorf("querying rseal_client_pipeline: %w", err) + } + + for _, task := range tasks { + if task.Failed { + continue + } + + p.pollClientPoll(ctx, task) + p.pollClientFetch(ctx, task) + p.pollClientCleanup(ctx, task) + } + + return nil +} + +// pollClientPoll creates RSealClientPoll tasks for sectors where delegation is done +// (after_delegate = TRUE) but remote SDR+trees have not yet completed, and no poll +// task is currently running. +func (p *RSealClientPoller) pollClientPoll(ctx context.Context, task clientPollTask) { + if task.AfterDelegate && !task.AfterSDR && task.TaskIDPoll == nil && p.pollers[pollerClientPoll].IsSet() { + p.pollers[pollerClientPoll].Val(ctx)(func(id harmonytask.TaskID, tx *harmonydb.Tx) (bool, error) { + n, err := tx.Exec(`UPDATE rseal_client_pipeline SET task_id_poll = $1 + WHERE sp_id = $2 AND sector_number = $3 AND after_delegate = TRUE AND after_sdr = FALSE AND task_id_poll IS NULL`, + id, task.SpID, task.SectorNumber) + if err != nil { + return false, xerrors.Errorf("updating rseal_client_pipeline for poll: %w", err) + } + if n != 1 { + return false, nil + } + return true, nil + }) + } +} + +// pollClientFetch creates fetch tasks for sectors where SDR+trees have completed remotely +// but the sealed data and cache have not yet been downloaded to local storage. +func (p *RSealClientPoller) pollClientFetch(ctx context.Context, task clientPollTask) { + if task.AfterSDR && !task.AfterFetch && task.TaskIDFetch == nil && + p.pollers[pollerClientFetch].IsSet() { + + p.pollers[pollerClientFetch].Val(ctx)(func(id harmonytask.TaskID, tx *harmonydb.Tx) (bool, error) { + n, err := tx.Exec(`UPDATE rseal_client_pipeline SET task_id_fetch = $1 + WHERE sp_id = $2 AND sector_number = $3 + AND after_sdr = TRUE AND after_fetch = FALSE AND task_id_fetch IS NULL`, + id, task.SpID, task.SectorNumber) + if err != nil { + return false, xerrors.Errorf("updating rseal_client_pipeline for fetch: %w", err) + } + if n != 1 { + return false, nil + } + return true, nil + }) + } +} + +// pollClientCleanup creates cleanup tasks for sectors where PoRep is done +// and the provider has not yet been told to clean up. +func (p *RSealClientPoller) pollClientCleanup(ctx context.Context, task clientPollTask) { + if task.AfterPoRep && !task.AfterCleanup && task.TaskIDCleanup == nil && + p.pollers[pollerClientCleanup].IsSet() { + + p.pollers[pollerClientCleanup].Val(ctx)(func(id harmonytask.TaskID, tx *harmonydb.Tx) (bool, error) { + n, err := tx.Exec(`UPDATE rseal_client_pipeline SET task_id_cleanup = $1 + WHERE sp_id = $2 AND sector_number = $3 + AND after_cleanup = FALSE AND task_id_cleanup IS NULL`, + id, task.SpID, task.SectorNumber) + if err != nil { + return false, xerrors.Errorf("updating rseal_client_pipeline for cleanup: %w", err) + } + if n != 1 { + return false, nil + } + return true, nil + }) + } +} diff --git a/tasks/remoteseal/provider_poller.go b/tasks/remoteseal/provider_poller.go new file mode 100644 index 000000000..aba5fddd4 --- /dev/null +++ b/tasks/remoteseal/provider_poller.go @@ -0,0 +1,321 @@ +package remoteseal + +import ( + "context" + "time" + + logging "github.com/ipfs/go-log/v2" + "golang.org/x/xerrors" + + "github.com/filecoin-project/curio/harmony/harmonydb" + "github.com/filecoin-project/curio/harmony/harmonytask" + "github.com/filecoin-project/curio/lib/promise" +) + +var log = logging.Logger("remoteseal") + +const ( + pollerProvSDR = iota + pollerProvTreeD + pollerProvTreeRC + pollerProvNotifyClient + pollerProvFinalize + pollerProvCleanup + + numProviderPollers +) + +const providerPollerInterval = 10 * time.Second + +type RSealProviderPoller struct { + db *harmonydb.DB + pollers [numProviderPollers]promise.Promise[harmonytask.AddTaskFunc] +} + +func NewProviderPoller(db *harmonydb.DB) *RSealProviderPoller { + return &RSealProviderPoller{ + db: db, + } +} + +// SetPollerSDR allows the SDR task to register its AddTaskFunc with the +// provider poller so that rseal_provider_pipeline rows can be scheduled. +func (sp *RSealProviderPoller) SetPollerSDR(f harmonytask.AddTaskFunc) { + sp.pollers[pollerProvSDR].Set(f) +} + +// SetPollerTreeD allows the TreeD task to register its AddTaskFunc with the +// provider poller so that rseal_provider_pipeline rows can be scheduled. +func (sp *RSealProviderPoller) SetPollerTreeD(f harmonytask.AddTaskFunc) { + sp.pollers[pollerProvTreeD].Set(f) +} + +// SetPollerTreeRC allows the TreeRC task to register its AddTaskFunc with the +// provider poller so that rseal_provider_pipeline rows can be scheduled. +func (sp *RSealProviderPoller) SetPollerTreeRC(f harmonytask.AddTaskFunc) { + sp.pollers[pollerProvTreeRC].Set(f) +} + +type pollProviderTask struct { + SpID int64 `db:"sp_id"` + SectorNumber int64 `db:"sector_number"` + RegSealProof int `db:"reg_seal_proof"` + PartnerID int64 `db:"partner_id"` + + // task IDs + TaskIDSdr *int64 `db:"task_id_sdr"` + TaskIDTreeD *int64 `db:"task_id_tree_d"` + TaskIDTreeC *int64 `db:"task_id_tree_c"` + TaskIDTreeR *int64 `db:"task_id_tree_r"` + TaskIDNotifyClient *int64 `db:"task_id_notify_client"` + TaskIDFinalize *int64 `db:"task_id_finalize"` + TaskIDCleanup *int64 `db:"task_id_cleanup"` + + // after flags + AfterSDR bool `db:"after_sdr"` + AfterTreeD bool `db:"after_tree_d"` + AfterTreeC bool `db:"after_tree_c"` + AfterTreeR bool `db:"after_tree_r"` + AfterNotifyClient bool `db:"after_notify_client"` + AfterC1Supplied bool `db:"after_c1_supplied"` + AfterFinalize bool `db:"after_finalize"` + AfterCleanup bool `db:"after_cleanup"` + + // cleanup + CleanupRequested bool `db:"cleanup_requested"` + CleanupTimeout *time.Time `db:"cleanup_timeout"` + + // failure + Failed bool `db:"failed"` +} + +func (sp *RSealProviderPoller) RunPoller(ctx context.Context) { + ticker := time.NewTicker(providerPollerInterval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if err := sp.poll(ctx); err != nil { + log.Errorw("provider poller failed", "error", err) + } + } + } +} + +func (sp *RSealProviderPoller) poll(ctx context.Context) error { + var tasks []pollProviderTask + + err := sp.db.Select(ctx, &tasks, `SELECT + sp_id, + sector_number, + reg_seal_proof, + partner_id, + task_id_sdr, + after_sdr, + task_id_tree_d, + after_tree_d, + task_id_tree_c, + after_tree_c, + task_id_tree_r, + after_tree_r, + task_id_notify_client, + after_notify_client, + after_c1_supplied, + task_id_finalize, + after_finalize, + cleanup_requested, + cleanup_timeout, + task_id_cleanup, + after_cleanup, + failed + FROM rseal_provider_pipeline + WHERE after_cleanup != TRUE AND failed != TRUE`) + if err != nil { + return xerrors.Errorf("selecting provider pipeline tasks: %w", err) + } + + for _, task := range tasks { + task := task + + if task.Failed { + continue + } + + // 1. SDR: not done, no SDR task running + // The SDR task computes its own ticket from the chain - no separate ticket fetch needed. + if !task.AfterSDR && task.TaskIDSdr == nil { + sp.pollStartSDR(ctx, task) + continue + } + + // 2. TreeD: SDR done, TreeD not done, no TreeD task running + if task.AfterSDR && !task.AfterTreeD && task.TaskIDTreeD == nil { + sp.pollStartTreeD(ctx, task) + continue + } + + // 3. TreeRC: TreeD done, TreeC/TreeR not done, no tasks running + if task.AfterTreeD && !task.AfterTreeC && !task.AfterTreeR && task.TaskIDTreeC == nil && task.TaskIDTreeR == nil { + sp.pollStartTreeRC(ctx, task) + continue + } + + // 4. NotifyClient: TreeR done, not yet notified, no notify task running + if task.AfterTreeR && !task.AfterNotifyClient && task.TaskIDNotifyClient == nil { + sp.pollStartNotifyClient(ctx, task) + continue + } + + // 5. Finalize: C1 supplied by client, not yet finalized, no finalize task running + if task.AfterC1Supplied && !task.AfterFinalize && task.TaskIDFinalize == nil { + sp.pollStartFinalize(ctx, task) + continue + } + + // 6. Cleanup: cleanup requested (or timeout reached), not yet cleaned, no cleanup task running + if !task.AfterCleanup && task.TaskIDCleanup == nil { + shouldCleanup := task.CleanupRequested || + (task.CleanupTimeout != nil && time.Now().After(*task.CleanupTimeout)) + if shouldCleanup { + sp.pollStartCleanup(ctx, task) + continue + } + } + } + + return nil +} + +// pollStartSDR assigns an SDR task to a sector. +// This uses the same SDR task type as the regular seal pipeline; the existing +// SDR task's Do() queries rseal_provider_pipeline via UNION ALL. +// The SDR task computes its own ticket from the chain. +func (sp *RSealProviderPoller) pollStartSDR(ctx context.Context, task pollProviderTask) { + if !sp.pollers[pollerProvSDR].IsSet() { + return + } + + sp.pollers[pollerProvSDR].Val(ctx)(func(id harmonytask.TaskID, tx *harmonydb.Tx) (shouldCommit bool, seriousError error) { + n, err := tx.Exec(`UPDATE rseal_provider_pipeline SET task_id_sdr = $1 + WHERE sp_id = $2 AND sector_number = $3 AND task_id_sdr IS NULL AND after_sdr = FALSE`, + id, task.SpID, task.SectorNumber) + if err != nil { + return false, xerrors.Errorf("update sdr task: %w", err) + } + if n != 1 { + return false, nil + } + + return true, nil + }) +} + +// pollStartTreeD assigns a TreeD task. Uses the same TreeD task type as the regular pipeline. +func (sp *RSealProviderPoller) pollStartTreeD(ctx context.Context, task pollProviderTask) { + if !sp.pollers[pollerProvTreeD].IsSet() { + return + } + + sp.pollers[pollerProvTreeD].Val(ctx)(func(id harmonytask.TaskID, tx *harmonydb.Tx) (shouldCommit bool, seriousError error) { + n, err := tx.Exec(`UPDATE rseal_provider_pipeline SET task_id_tree_d = $1 + WHERE sp_id = $2 AND sector_number = $3 AND after_sdr = TRUE AND task_id_tree_d IS NULL`, + id, task.SpID, task.SectorNumber) + if err != nil { + return false, xerrors.Errorf("update tree_d task: %w", err) + } + if n != 1 { + return false, nil + } + + return true, nil + }) +} + +// pollStartTreeRC assigns TreeC and TreeR tasks (same task handles both). +// Uses the same TreeRC task type as the regular pipeline. +func (sp *RSealProviderPoller) pollStartTreeRC(ctx context.Context, task pollProviderTask) { + if !sp.pollers[pollerProvTreeRC].IsSet() { + return + } + + sp.pollers[pollerProvTreeRC].Val(ctx)(func(id harmonytask.TaskID, tx *harmonydb.Tx) (shouldCommit bool, seriousError error) { + n, err := tx.Exec(`UPDATE rseal_provider_pipeline SET task_id_tree_c = $1, task_id_tree_r = $1 + WHERE sp_id = $2 AND sector_number = $3 AND after_tree_d = TRUE AND task_id_tree_c IS NULL AND task_id_tree_r IS NULL`, + id, task.SpID, task.SectorNumber) + if err != nil { + return false, xerrors.Errorf("update tree_rc task: %w", err) + } + if n != 1 { + return false, nil + } + + return true, nil + }) +} + +// pollStartNotifyClient creates a task to notify the client that SDR+trees are done. +func (sp *RSealProviderPoller) pollStartNotifyClient(ctx context.Context, task pollProviderTask) { + if !sp.pollers[pollerProvNotifyClient].IsSet() { + return + } + + sp.pollers[pollerProvNotifyClient].Val(ctx)(func(id harmonytask.TaskID, tx *harmonydb.Tx) (shouldCommit bool, seriousError error) { + n, err := tx.Exec(`UPDATE rseal_provider_pipeline SET task_id_notify_client = $1 + WHERE sp_id = $2 AND sector_number = $3 AND after_tree_r = TRUE AND task_id_notify_client IS NULL AND after_notify_client = FALSE`, + id, task.SpID, task.SectorNumber) + if err != nil { + return false, xerrors.Errorf("update notify client task: %w", err) + } + if n != 1 { + return false, nil + } + + return true, nil + }) +} + +// pollStartFinalize creates a task to drop SDR layers after the client has supplied C1. +func (sp *RSealProviderPoller) pollStartFinalize(ctx context.Context, task pollProviderTask) { + if !sp.pollers[pollerProvFinalize].IsSet() { + return + } + + sp.pollers[pollerProvFinalize].Val(ctx)(func(id harmonytask.TaskID, tx *harmonydb.Tx) (shouldCommit bool, seriousError error) { + n, err := tx.Exec(`UPDATE rseal_provider_pipeline SET task_id_finalize = $1 + WHERE sp_id = $2 AND sector_number = $3 AND after_c1_supplied = TRUE AND task_id_finalize IS NULL AND after_finalize = FALSE`, + id, task.SpID, task.SectorNumber) + if err != nil { + return false, xerrors.Errorf("update finalize task: %w", err) + } + if n != 1 { + return false, nil + } + + return true, nil + }) +} + +// pollStartCleanup creates a task to remove all sector data from storage. +func (sp *RSealProviderPoller) pollStartCleanup(ctx context.Context, task pollProviderTask) { + if !sp.pollers[pollerProvCleanup].IsSet() { + return + } + + sp.pollers[pollerProvCleanup].Val(ctx)(func(id harmonytask.TaskID, tx *harmonydb.Tx) (shouldCommit bool, seriousError error) { + n, err := tx.Exec(`UPDATE rseal_provider_pipeline SET task_id_cleanup = $1 + WHERE sp_id = $2 AND sector_number = $3 AND task_id_cleanup IS NULL AND after_cleanup = FALSE + AND (cleanup_requested = TRUE OR (cleanup_timeout IS NOT NULL AND NOW() > cleanup_timeout))`, + id, task.SpID, task.SectorNumber) + if err != nil { + return false, xerrors.Errorf("update cleanup task: %w", err) + } + if n != 1 { + return false, nil + } + + return true, nil + }) +} diff --git a/tasks/remoteseal/task_client_cleanup.go b/tasks/remoteseal/task_client_cleanup.go new file mode 100644 index 000000000..163409c8d --- /dev/null +++ b/tasks/remoteseal/task_client_cleanup.go @@ -0,0 +1,142 @@ +package remoteseal + +import ( + "context" + "time" + + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-state-types/abi" + + "github.com/filecoin-project/curio/harmony/harmonydb" + "github.com/filecoin-project/curio/harmony/harmonytask" + "github.com/filecoin-project/curio/harmony/resources" + "github.com/filecoin-project/curio/harmony/taskhelp" + "github.com/filecoin-project/curio/market/sealmarket" +) + +// RSealClientCleanup tells the remote provider to clean up after PoRep is done. +// First sends a finalize request (drop layers), then sends a cleanup request +// (full data removal). +type RSealClientCleanup struct { + db *harmonydb.DB + client *RSealClient + sp *RSealClientPoller +} + +func NewRSealClientCleanup(db *harmonydb.DB, client *RSealClient, sp *RSealClientPoller) *RSealClientCleanup { + return &RSealClientCleanup{ + db: db, + client: client, + sp: sp, + } +} + +func (t *RSealClientCleanup) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done bool, err error) { + ctx := context.Background() + + // Find the sector assigned to this cleanup task + var sectors []struct { + SpID int64 `db:"sp_id"` + SectorNumber int64 `db:"sector_number"` + ProviderURL string `db:"provider_url"` + ProviderToken string `db:"provider_token"` + } + + err = t.db.Select(ctx, §ors, ` + SELECT c.sp_id, c.sector_number, pr.provider_url, pr.provider_token + FROM rseal_client_pipeline c + JOIN rseal_client_providers pr ON c.provider_id = pr.id + WHERE c.task_id_cleanup = $1`, taskID) + if err != nil { + return false, xerrors.Errorf("querying sector for cleanup: %w", err) + } + + if len(sectors) != 1 { + return false, xerrors.Errorf("expected 1 sector for cleanup, got %d", len(sectors)) + } + sector := sectors[0] + + // Send finalize to provider (drop layers, keep sealed+cache for potential C1 retries) + err = t.client.SendFinalize(ctx, sector.ProviderURL, sector.ProviderToken, &sealmarket.FinalizeRequest{ + SpID: sector.SpID, + SectorNumber: sector.SectorNumber, + }) + if err != nil { + return false, xerrors.Errorf("sending finalize to provider: %w", err) + } + + // Send cleanup to provider (full data removal) + err = t.client.SendCleanup(ctx, sector.ProviderURL, sector.ProviderToken, &sealmarket.CleanupRequest{ + SpID: sector.SpID, + SectorNumber: sector.SectorNumber, + }) + if err != nil { + return false, xerrors.Errorf("sending cleanup to provider: %w", err) + } + + if !stillOwned() { + return false, xerrors.Errorf("task no longer owned") + } + + // Mark cleanup as done + _, err = t.db.Exec(ctx, ` + UPDATE rseal_client_pipeline + SET after_cleanup = TRUE, task_id_cleanup = NULL + WHERE sp_id = $1 AND sector_number = $2`, + sector.SpID, sector.SectorNumber) + if err != nil { + return false, xerrors.Errorf("updating rseal_client_pipeline: %w", err) + } + + log.Infow("remote seal cleanup completed", + "sp_id", sector.SpID, "sector", sector.SectorNumber) + + return true, nil +} + +func (t *RSealClientCleanup) CanAccept(ids []harmonytask.TaskID, _ *harmonytask.TaskEngine) ([]harmonytask.TaskID, error) { + return ids, nil +} + +func (t *RSealClientCleanup) TypeDetails() harmonytask.TaskTypeDetails { + return harmonytask.TaskTypeDetails{ + Name: "RSealCleanup", + Cost: resources.Resources{ + Cpu: 0, + Gpu: 0, + Ram: 16 << 20, // 16 MiB - just HTTP calls + }, + MaxFailures: 50, + RetryWait: taskhelp.RetryWaitLinear(60*time.Second, 60*time.Second), + } +} + +func (t *RSealClientCleanup) Adder(taskFunc harmonytask.AddTaskFunc) { + t.sp.pollers[pollerClientCleanup].Set(taskFunc) +} + +func (t *RSealClientCleanup) GetSpid(db *harmonydb.DB, taskID int64) string { + sid, err := t.GetSectorID(db, taskID) + if err != nil { + log.Errorf("getting sector id: %s", err) + return "" + } + return sid.Miner.String() +} + +func (t *RSealClientCleanup) GetSectorID(db *harmonydb.DB, taskID int64) (*abi.SectorID, error) { + var spId, sectorNumber uint64 + err := db.QueryRow(context.Background(), + `SELECT sp_id, sector_number FROM rseal_client_pipeline WHERE task_id_cleanup = $1`, taskID).Scan(&spId, §orNumber) + if err != nil { + return nil, err + } + return &abi.SectorID{ + Miner: abi.ActorID(spId), + Number: abi.SectorNumber(sectorNumber), + }, nil +} + +var _ = harmonytask.Reg(&RSealClientCleanup{}) +var _ harmonytask.TaskInterface = &RSealClientCleanup{} diff --git a/tasks/remoteseal/task_client_delegate.go b/tasks/remoteseal/task_client_delegate.go new file mode 100644 index 000000000..f892e64a3 --- /dev/null +++ b/tasks/remoteseal/task_client_delegate.go @@ -0,0 +1,395 @@ +package remoteseal + +import ( + "context" + "sync" + "time" + + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-bitfield" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/builtin" + miner12 "github.com/filecoin-project/go-state-types/builtin/v12/miner" + + "github.com/filecoin-project/curio/harmony/harmonydb" + "github.com/filecoin-project/curio/harmony/harmonytask" + "github.com/filecoin-project/curio/harmony/resources" + "github.com/filecoin-project/curio/harmony/taskhelp" + "github.com/filecoin-project/curio/market/sealmarket" + "github.com/filecoin-project/curio/tasks/seal" + + lotusapi "github.com/filecoin-project/lotus/api" + apitypes "github.com/filecoin-project/lotus/api/types" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" + "github.com/filecoin-project/lotus/chain/types" +) + +// RSealDelegateAPI provides chain state access needed for CC sector scheduling. +type RSealDelegateAPI interface { + StateMinerAllocated(context.Context, address.Address, types.TipSetKey) (*bitfield.BitField, error) + StateMinerInfo(context.Context, address.Address, types.TipSetKey) (lotusapi.MinerInfo, error) + StateNetworkVersion(context.Context, types.TipSetKey) (apitypes.NetworkVersion, error) +} + +// RSealDelegate intercepts sectors before normal SDR processing and delegates +// them to remote providers. Uses the IAmBored pattern like SupraSeal's schedule(). +// +// The schedule() callback only does fast DB operations to claim sectors. +// The expensive HTTP dance (CheckAvailable + SendOrder) happens in Do() so +// the scheduling loop is not blocked. +// +// When no existing unclaimed sectors are found, schedule() also creates new CC +// sectors from the sectors_cc_scheduler table for SPs that have enabled remote +// providers. +type RSealDelegate struct { + db *harmonydb.DB + api RSealDelegateAPI // optional, nil disables CC scheduling + client *RSealClient + + lastScheduledWork bool // true if the last schedule() call found/created work +} + +func NewRSealDelegate(db *harmonydb.DB, api RSealDelegateAPI, client *RSealClient) *RSealDelegate { + return &RSealDelegate{ + db: db, + api: api, + client: client, + } +} + +// schedule is the IAmBored callback. It finds unclaimed sectors that have enabled +// providers and atomically claims them in the DB. No HTTP calls happen here — +// the expensive provider interaction is deferred to Do(). +// +// When no existing unclaimed sectors are found, it also creates new CC sectors +// from the sectors_cc_scheduler table for SPs that have enabled remote providers. +func (d *RSealDelegate) schedule(taskFunc harmonytask.AddTaskFunc) error { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + d.lastScheduledWork = false + + // Find sectors ready for SDR that are not yet claimed by any task and + // have no existing rseal_client_pipeline entry, but DO have an enabled provider. + var sectors []struct { + SpID int64 `db:"sp_id"` + SectorNumber int64 `db:"sector_number"` + RegSealProof int `db:"reg_seal_proof"` + ProviderID int64 `db:"provider_id"` + UserSectorDurationEpochs *int64 `db:"user_sector_duration_epochs"` + } + err := d.db.Select(ctx, §ors, ` + SELECT s.sp_id, s.sector_number, s.reg_seal_proof, s.user_sector_duration_epochs, p.id AS provider_id + FROM sectors_sdr_pipeline s + JOIN rseal_client_providers p ON p.sp_id = s.sp_id AND p.enabled = TRUE + WHERE s.after_sdr = FALSE + AND s.task_id_sdr IS NULL + AND NOT EXISTS ( + SELECT 1 FROM rseal_client_pipeline c + WHERE c.sp_id = s.sp_id + AND c.sector_number = s.sector_number + ) + LIMIT 1`) + if err != nil { + return xerrors.Errorf("finding candidate sectors: %w", err) + } + + if len(sectors) == 0 { + // No existing unclaimed sectors — try to create CC sectors for remote delegation + return d.scheduleCCRemote(ctx, taskFunc) + } + + sector := sectors[0] + + // Atomically claim the sector in the DB. Do() will handle the HTTP calls. + d.claimSectorForDelegation(taskFunc, sector.SpID, sector.SectorNumber, sector.ProviderID, sector.RegSealProof, sector.UserSectorDurationEpochs) + d.lastScheduledWork = true + + return nil +} + +// claimSectorForDelegation atomically creates the rseal_client_pipeline entry. +// The rseal_client_pipeline row is what prevents the local SDR poller and +// SupraSeal batch scheduler from claiming this sector (via LEFT JOIN / NOT EXISTS checks). +func (d *RSealDelegate) claimSectorForDelegation(taskFunc harmonytask.AddTaskFunc, spID, sectorNumber int64, providerID int64, regSealProof int, userDuration *int64) { + taskFunc(func(id harmonytask.TaskID, tx *harmonydb.Tx) (bool, error) { + n, err := tx.Exec(` + INSERT INTO rseal_client_pipeline (sp_id, sector_number, provider_id, reg_seal_proof, user_sector_duration_epochs, task_id_delegate) + VALUES ($1, $2, $3, $4, $5, $6) + ON CONFLICT (sp_id, sector_number) DO NOTHING`, + spID, sectorNumber, providerID, regSealProof, userDuration, id) + if err != nil { + return false, xerrors.Errorf("inserting rseal_client_pipeline: %w", err) + } + if n == 0 { + return false, nil // already claimed + } + + return true, nil + }) +} + +// scheduleCCRemote creates new CC sectors from the sectors_cc_scheduler table +// for SPs that have enabled remote providers. It allocates a sector number, +// inserts into sectors_sdr_pipeline, creates the rseal_client_pipeline entry, +// and claims the sector — all in one transaction. +func (d *RSealDelegate) scheduleCCRemote(ctx context.Context, taskFunc harmonytask.AddTaskFunc) error { + if d.api == nil { + return nil // CC scheduling not configured + } + + // Find enabled CC schedules for SPs that HAVE an enabled remote provider. + var schedules []struct { + SpID int64 `db:"sp_id"` + ToSeal int64 `db:"to_seal"` + DurationDays int64 `db:"duration_days"` + ProviderID int64 `db:"provider_id"` + } + err := d.db.Select(ctx, &schedules, ` + SELECT cs.sp_id, cs.to_seal, cs.duration_days, p.id AS provider_id + FROM sectors_cc_scheduler cs + JOIN rseal_client_providers p ON p.sp_id = cs.sp_id AND p.enabled = TRUE + WHERE cs.enabled = TRUE + AND cs.to_seal > 0 + ORDER BY cs.sp_id + LIMIT 1`) + if err != nil { + return xerrors.Errorf("querying cc_scheduler for remote: %w", err) + } + + if len(schedules) == 0 { + return nil + } + + schedule := schedules[0] + + nv, err := d.api.StateNetworkVersion(ctx, types.EmptyTSK) + if err != nil { + return xerrors.Errorf("getting network version: %w", err) + } + + maddr, err := address.NewIDAddress(uint64(schedule.SpID)) + if err != nil { + return xerrors.Errorf("creating miner address: %w", err) + } + + mi, err := d.api.StateMinerInfo(ctx, maddr, types.EmptyTSK) + if err != nil { + return xerrors.Errorf("getting miner info for %s: %w", maddr, err) + } + + spt, err := miner.PreferredSealProofTypeFromWindowPoStType(nv, mi.WindowPoStProofType, false) + if err != nil { + return xerrors.Errorf("getting seal proof type: %w", err) + } + + userDuration := schedule.DurationDays * builtin.EpochsInDay + if miner12.MaxSectorExpirationExtension < userDuration { + return xerrors.Errorf("duration exceeds max allowed: %d > %d", userDuration, miner12.MaxSectorExpirationExtension) + } + if miner12.MinSectorExpiration > userDuration { + return xerrors.Errorf("duration is too short: %d < %d", userDuration, miner12.MinSectorExpiration) + } + + taskFunc(func(id harmonytask.TaskID, tx *harmonydb.Tx) (bool, error) { + // Allocate one sector number + sectorNumbers, err := seal.AllocateSectorNumbers(ctx, d.api, tx, maddr, 1) + if err != nil { + return false, xerrors.Errorf("allocating sector number: %w", err) + } + if len(sectorNumbers) != 1 { + return false, xerrors.Errorf("expected 1 sector number, got %d", len(sectorNumbers)) + } + sectorNum := sectorNumbers[0] + + // Only insert into rseal_client_pipeline — the sectors_sdr_pipeline row + // will be created later by ApplyRemoteCompletion when the provider finishes + // SDR+trees and the sealed data is ready for precommit. + n, err := tx.Exec(` + INSERT INTO rseal_client_pipeline (sp_id, sector_number, provider_id, reg_seal_proof, user_sector_duration_epochs, task_id_delegate) + VALUES ($1, $2, $3, $4, $5, $6) + ON CONFLICT (sp_id, sector_number) DO NOTHING`, + schedule.SpID, int64(sectorNum), schedule.ProviderID, int(spt), userDuration, id) + if err != nil { + return false, xerrors.Errorf("inserting rseal_client_pipeline: %w", err) + } + if n == 0 { + return false, nil // shouldn't happen for a freshly allocated sector + } + + // Decrement to_seal + _, err = tx.Exec(`UPDATE sectors_cc_scheduler SET to_seal = to_seal - 1 WHERE sp_id = $1 AND to_seal > 0`, schedule.SpID) + if err != nil { + return false, xerrors.Errorf("decrementing to_seal: %w", err) + } + + log.Infow("CC scheduler: created remote CC sector", + "sp_id", schedule.SpID, + "sector", sectorNum, + "proof", spt, + "provider_id", schedule.ProviderID, + "duration_days", schedule.DurationDays) + + return true, nil + }) + + d.lastScheduledWork = true + return nil +} + +func (d *RSealDelegate) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done bool, err error) { + ctx := context.Background() + + // Read the claimed sector and provider info + var sectors []struct { + SpID int64 `db:"sp_id"` + SectorNumber int64 `db:"sector_number"` + RegSealProof int `db:"reg_seal_proof"` + ProviderURL string `db:"provider_url"` + ProviderToken string `db:"provider_token"` + } + + err = d.db.Select(ctx, §ors, ` + SELECT c.sp_id, c.sector_number, c.reg_seal_proof, + p.provider_url, p.provider_token + FROM rseal_client_pipeline c + JOIN rseal_client_providers p ON c.provider_id = p.id + WHERE c.task_id_delegate = $1`, taskID) + if err != nil { + return false, xerrors.Errorf("querying sector for delegate task: %w", err) + } + + if len(sectors) != 1 { + return false, xerrors.Errorf("expected 1 sector for delegate task, got %d", len(sectors)) + } + sector := sectors[0] + + // Check provider availability + availCtx, availCancel := context.WithTimeout(ctx, 10*time.Second) + availResp, err := d.client.CheckAvailable(availCtx, sector.ProviderURL, sector.ProviderToken) + availCancel() + if err != nil { + return false, xerrors.Errorf("checking provider availability: %w", err) + } + + if !availResp.Available { + // Provider not available right now — retry later + return false, xerrors.Errorf("provider %s not available", sector.ProviderURL) + } + + // Send order to provider + orderResp, err := d.client.SendOrder(ctx, sector.ProviderURL, sector.ProviderToken, &sealmarket.OrderRequest{ + SlotToken: availResp.SlotToken, + SpID: sector.SpID, + SectorNumber: sector.SectorNumber, + RegSealProof: sector.RegSealProof, + }) + if err != nil { + return false, xerrors.Errorf("sending order to provider: %w", err) + } + + if !orderResp.Accepted { + // Provider rejected the order — fail permanently so the sector can be + // re-assigned (the poller will clear task_id_delegate on failure) + log.Warnw("provider rejected order", + "provider", sector.ProviderURL, + "reason", orderResp.RejectReason, + "sp_id", sector.SpID, + "sector", sector.SectorNumber) + return false, xerrors.Errorf("provider rejected order: %s", orderResp.RejectReason) + } + + // Order accepted. Mark delegate as done so the poller can create a + // RSealClientPoll task to monitor provider progress. + _, err = d.db.Exec(ctx, `UPDATE rseal_client_pipeline + SET after_delegate = TRUE, task_id_delegate = NULL + WHERE sp_id = $1 AND sector_number = $2 AND task_id_delegate = $3`, + sector.SpID, sector.SectorNumber, taskID) + if err != nil { + return false, xerrors.Errorf("marking delegate complete: %w", err) + } + + log.Infow("delegated sector to remote provider", + "sp_id", sector.SpID, + "sector", sector.SectorNumber, + "provider", sector.ProviderURL) + + return true, nil +} + +func (d *RSealDelegate) CanAccept(ids []harmonytask.TaskID, _ *harmonytask.TaskEngine) ([]harmonytask.TaskID, error) { + return ids, nil +} + +func (d *RSealDelegate) TypeDetails() harmonytask.TaskTypeDetails { + return harmonytask.TaskTypeDetails{ + Name: "RSealDelegate", + Cost: resources.Resources{ + Cpu: 0, + Gpu: 0, + Ram: 16 << 20, // 16 MiB - minimal, just HTTP calls + }, + MaxFailures: 100, + RetryWait: taskhelp.RetryWaitLinear(20*time.Second, 0), + IAmBored: d.adaptiveSchedule(), + } +} + +// adaptiveSchedule returns a rate-limited schedule function that runs more +// frequently (1s) when work was found on the last call, and backs off to 15s +// when idle. This allows rapid CC sector creation when the scheduler is active. +func (d *RSealDelegate) adaptiveSchedule() func(harmonytask.AddTaskFunc) error { + var lastCall time.Time + var lk sync.Mutex + + return func(taskFunc harmonytask.AddTaskFunc) error { + lk.Lock() + defer lk.Unlock() + + interval := 15 * time.Second + if d.lastScheduledWork { + interval = 1 * time.Second + } + + if time.Since(lastCall) < interval { + return nil + } + + defer func() { + lastCall = time.Now() + }() + + return d.schedule(taskFunc) + } +} + +func (d *RSealDelegate) Adder(taskFunc harmonytask.AddTaskFunc) { + // IAmBored tasks don't use the Adder pattern +} + +func (d *RSealDelegate) GetSpid(db *harmonydb.DB, taskID int64) string { + sid, err := d.GetSectorID(db, taskID) + if err != nil { + log.Errorf("getting sector id: %s", err) + return "" + } + return sid.Miner.String() +} + +func (d *RSealDelegate) GetSectorID(db *harmonydb.DB, taskID int64) (*abi.SectorID, error) { + var spId, sectorNumber uint64 + err := db.QueryRow(context.Background(), + `SELECT sp_id, sector_number FROM rseal_client_pipeline WHERE task_id_delegate = $1`, taskID).Scan(&spId, §orNumber) + if err != nil { + return nil, err + } + return &abi.SectorID{ + Miner: abi.ActorID(spId), + Number: abi.SectorNumber(sectorNumber), + }, nil +} + +var _ = harmonytask.Reg(&RSealDelegate{}) +var _ harmonytask.TaskInterface = &RSealDelegate{} diff --git a/tasks/remoteseal/task_client_fetch.go b/tasks/remoteseal/task_client_fetch.go new file mode 100644 index 000000000..bb078425f --- /dev/null +++ b/tasks/remoteseal/task_client_fetch.go @@ -0,0 +1,180 @@ +package remoteseal + +import ( + "context" + "fmt" + "time" + + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-state-types/abi" + + "github.com/filecoin-project/curio/harmony/harmonydb" + "github.com/filecoin-project/curio/harmony/harmonytask" + "github.com/filecoin-project/curio/harmony/resources" + "github.com/filecoin-project/curio/harmony/taskhelp" + ffi "github.com/filecoin-project/curio/lib/ffi" + "github.com/filecoin-project/curio/lib/paths" + "github.com/filecoin-project/curio/lib/storiface" +) + +// RSealClientFetch downloads the sealed sector file and finalized cache tar +// from the remote provider after SDR+trees complete remotely. The sealed file +// is streamed directly to disk (32 GiB), and the cache tar is extracted into +// the local cache directory. +type RSealClientFetch struct { + db *harmonydb.DB + sc *ffi.SealCalls + sp *RSealClientPoller + max int +} + +func NewRSealClientFetch(db *harmonydb.DB, client *RSealClient, sc *ffi.SealCalls, sp *RSealClientPoller, max int) *RSealClientFetch { + return &RSealClientFetch{ + db: db, + sc: sc, + sp: sp, + max: max, + } +} + +func (f *RSealClientFetch) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done bool, err error) { + ctx := context.Background() + + // Find the sector assigned to this fetch task + var sectors []struct { + SpID int64 `db:"sp_id"` + SectorNumber int64 `db:"sector_number"` + RegSealProof int `db:"reg_seal_proof"` + ProviderURL string `db:"provider_url"` + ProviderToken string `db:"provider_token"` + } + + err = f.db.Select(ctx, §ors, ` + SELECT c.sp_id, c.sector_number, c.reg_seal_proof, + pr.provider_url, pr.provider_token + FROM rseal_client_pipeline c + JOIN rseal_client_providers pr ON c.provider_id = pr.id + WHERE c.task_id_fetch = $1`, taskID) + if err != nil { + return false, xerrors.Errorf("querying sector for fetch task: %w", err) + } + + if len(sectors) != 1 { + return false, xerrors.Errorf("expected 1 sector for fetch task, got %d", len(sectors)) + } + sector := sectors[0] + + sref := storiface.SectorRef{ + ID: abi.SectorID{ + Miner: abi.ActorID(sector.SpID), + Number: abi.SectorNumber(sector.SectorNumber), + }, + ProofType: abi.RegisteredSealProof(sector.RegSealProof), + } + + log.Infow("downloading remote seal data", + "sp_id", sector.SpID, "sector", sector.SectorNumber, + "provider", sector.ProviderURL) + + err = f.sc.DownloadRemoteSealData(ctx, &taskID, sref, ffi.RemoteSealFetchParams{ + SealedURL: endpoint(sector.ProviderURL, fmt.Sprintf("sealed-data/%d/%d", sector.SpID, sector.SectorNumber)) + "?token=" + sector.ProviderToken, + CacheURL: endpoint(sector.ProviderURL, fmt.Sprintf("cache-data/%d/%d", sector.SpID, sector.SectorNumber)) + "?token=" + sector.ProviderToken, + C1Info: paths.RemoteSealC1Info{ + C1URL: endpoint(sector.ProviderURL, "commit1"), + PartnerToken: sector.ProviderToken, + SpID: sector.SpID, + SectorNumber: sector.SectorNumber, + }, + SpID: sector.SpID, + SectorNumber: sector.SectorNumber, + }) + if err != nil { + return false, xerrors.Errorf("downloading remote seal data: %w", err) + } + + if !stillOwned() { + return false, xerrors.Errorf("task no longer owned") + } + + // Mark fetch as done — sealed data is now in local storage so the sector + // can proceed through the normal pipeline (precommit, PoRep, etc). + _, err = f.db.Exec(ctx, ` + UPDATE rseal_client_pipeline + SET after_fetch = TRUE, task_id_fetch = NULL + WHERE sp_id = $1 AND sector_number = $2`, + sector.SpID, sector.SectorNumber) + if err != nil { + return false, xerrors.Errorf("updating rseal_client_pipeline: %w", err) + } + + log.Infow("remote seal fetch completed", + "sp_id", sector.SpID, "sector", sector.SectorNumber) + + return true, nil +} + +func (f *RSealClientFetch) CanAccept(ids []harmonytask.TaskID, _ *harmonytask.TaskEngine) ([]harmonytask.TaskID, error) { + return ids, nil +} + +func (f *RSealClientFetch) TypeDetails() harmonytask.TaskTypeDetails { + ssize := abi.SectorSize(32 << 30) // todo task details needs taskID to get correct sector size + + return harmonytask.TaskTypeDetails{ + Max: taskhelp.Max(f.max), + Name: "RSealClientFetch", + Cost: resources.Resources{ + Cpu: 0, + Gpu: 0, + Ram: 64 << 20, // 64 MiB - streaming to disk + Storage: f.sc.Storage(f.taskToSector, storiface.FTCache|storiface.FTSealed, storiface.FTNone, ssize, storiface.PathStorage, paths.MinFreeStoragePercentage), + }, + MaxFailures: 20, + RetryWait: taskhelp.RetryWaitLinear(5*time.Minute, 5*time.Minute), + } +} + +func (f *RSealClientFetch) taskToSector(id harmonytask.TaskID) (ffi.SectorRef, error) { + var refs []ffi.SectorRef + + err := f.db.Select(context.Background(), &refs, `SELECT sp_id, sector_number, reg_seal_proof FROM rseal_client_pipeline WHERE task_id_fetch = $1`, id) + if err != nil { + return ffi.SectorRef{}, xerrors.Errorf("getting sector ref: %w", err) + } + + if len(refs) != 1 { + return ffi.SectorRef{}, xerrors.Errorf("expected 1 sector ref, got %d", len(refs)) + } + + return refs[0], nil +} + +func (f *RSealClientFetch) Adder(taskFunc harmonytask.AddTaskFunc) { + f.sp.pollers[pollerClientFetch].Set(taskFunc) +} + +func (f *RSealClientFetch) GetSpid(db *harmonydb.DB, taskID int64) string { + sid, err := f.GetSectorID(db, taskID) + if err != nil { + log.Errorf("getting sector id: %s", err) + return "" + } + return sid.Miner.String() +} + +func (f *RSealClientFetch) GetSectorID(db *harmonydb.DB, taskID int64) (*abi.SectorID, error) { + var spId, sectorNumber uint64 + err := db.QueryRow(context.Background(), + `SELECT sp_id, sector_number FROM rseal_client_pipeline WHERE task_id_fetch = $1`, taskID).Scan(&spId, §orNumber) + if err != nil { + return nil, err + } + return &abi.SectorID{ + Miner: abi.ActorID(spId), + Number: abi.SectorNumber(sectorNumber), + }, nil +} + +var _ = harmonytask.Reg(&RSealClientFetch{}) +var _ harmonytask.TaskInterface = &RSealClientFetch{} diff --git a/tasks/remoteseal/task_client_poll.go b/tasks/remoteseal/task_client_poll.go new file mode 100644 index 000000000..b5248dff8 --- /dev/null +++ b/tasks/remoteseal/task_client_poll.go @@ -0,0 +1,181 @@ +package remoteseal + +import ( + "context" + "time" + + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-state-types/abi" + + "github.com/filecoin-project/curio/harmony/harmonydb" + "github.com/filecoin-project/curio/harmony/harmonytask" + "github.com/filecoin-project/curio/harmony/resources" + "github.com/filecoin-project/curio/harmony/taskhelp" + "github.com/filecoin-project/curio/market/sealmarket" +) + +// RSealClientPoll polls remote providers for completion status. +// This is a fallback mechanism - the primary path is the /complete callback +// from the provider. The poll task runs periodically to catch cases where +// the callback was missed. +type RSealClientPoll struct { + db *harmonydb.DB + client *RSealClient + sp *RSealClientPoller +} + +func NewRSealClientPoll(db *harmonydb.DB, client *RSealClient, sp *RSealClientPoller) *RSealClientPoll { + return &RSealClientPoll{ + db: db, + client: client, + sp: sp, + } +} + +func (p *RSealClientPoll) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done bool, err error) { + ctx := context.Background() + + const pollInterval = 30 * time.Second + + // Find the sector assigned to this poll task + var sectors []struct { + SpID int64 `db:"sp_id"` + SectorNumber int64 `db:"sector_number"` + RegSealProof int `db:"reg_seal_proof"` + ProviderID int64 `db:"provider_id"` + ProviderURL string `db:"provider_url"` + ProviderToken string `db:"provider_token"` + } + + err = p.db.Select(ctx, §ors, ` + SELECT c.sp_id, c.sector_number, c.reg_seal_proof, c.provider_id, pr.provider_url, pr.provider_token + FROM rseal_client_pipeline c + JOIN rseal_client_providers pr ON c.provider_id = pr.id + WHERE c.task_id_poll = $1 AND c.after_sdr = FALSE`, taskID) + if err != nil { + return false, xerrors.Errorf("querying sector for poll task: %w", err) + } + + if len(sectors) != 1 { + return false, xerrors.Errorf("expected 1 sector for poll task, got %d", len(sectors)) + } + sector := sectors[0] + + // Poll the provider in a loop until completion, failure, or ownership loss + for { + statusResp, err := p.client.GetStatus(ctx, sector.ProviderURL, sector.ProviderToken, &sealmarket.StatusRequest{ + SpID: sector.SpID, + SectorNumber: sector.SectorNumber, + }) + if err != nil { + // HTTP error - log and retry within the loop after a sleep + log.Warnw("remote seal poll: error polling provider, will retry", + "sp_id", sector.SpID, "sector", sector.SectorNumber, "error", err) + + time.Sleep(pollInterval) + + if !stillOwned() { + return false, xerrors.Errorf("yield") + } + continue + } + + switch statusResp.State { + case "complete": + // Provider is done with SDR+trees. Apply the completion (ticket comes from status response). + if err := sealmarket.ApplyRemoteCompletion(ctx, p.db, sector.SpID, sector.SectorNumber, sector.ProviderID, + statusResp.TreeDCid, statusResp.TreeRCid, statusResp.TicketEpoch, statusResp.TicketValue); err != nil { + return false, xerrors.Errorf("applying remote completion: %w", err) + } + + log.Infow("remote seal poll: sector completed", + "sp_id", sector.SpID, "sector", sector.SectorNumber, + "tree_d_cid", statusResp.TreeDCid, "tree_r_cid", statusResp.TreeRCid) + + return true, nil + + case "failed", "gone": + // Provider reports failure or sector is gone (never received / cleaned up). + reason := statusResp.FailReason + if statusResp.State == "gone" { + reason = "sector not found on provider" + } + + _, err := p.db.Exec(ctx, ` + UPDATE rseal_client_pipeline + SET failed = TRUE, failed_at = NOW(), failed_reason = 'provider', failed_reason_msg = $3, + task_id_poll = NULL + WHERE sp_id = $1 AND sector_number = $2`, + sector.SpID, sector.SectorNumber, reason) + if err != nil { + return false, xerrors.Errorf("marking sector failed: %w", err) + } + + log.Warnw("remote seal poll: sector failed on provider", + "sp_id", sector.SpID, "sector", sector.SectorNumber, + "state", statusResp.State, "reason", reason) + + return true, nil + + default: + // Still in progress (pending, sdr, trees) - sleep and poll again + log.Debugw("remote seal poll: sector still in progress", + "sp_id", sector.SpID, "sector", sector.SectorNumber, + "state", statusResp.State) + + time.Sleep(pollInterval) + + if !stillOwned() { + return false, xerrors.Errorf("yield") + } + } + } +} + +func (p *RSealClientPoll) CanAccept(ids []harmonytask.TaskID, _ *harmonytask.TaskEngine) ([]harmonytask.TaskID, error) { + return ids, nil +} + +func (p *RSealClientPoll) TypeDetails() harmonytask.TaskTypeDetails { + return harmonytask.TaskTypeDetails{ + Name: "RSealClientPoll", + CanYield: true, + Cost: resources.Resources{ + Cpu: 0, + Gpu: 0, + Ram: 16 << 20, // 16 MiB - just HTTP calls + }, + MaxFailures: 10, + RetryWait: taskhelp.RetryWaitLinear(30*time.Second, 0), + } +} + +func (p *RSealClientPoll) Adder(taskFunc harmonytask.AddTaskFunc) { + p.sp.pollers[pollerClientPoll].Set(taskFunc) +} + +func (p *RSealClientPoll) GetSpid(db *harmonydb.DB, taskID int64) string { + sid, err := p.GetSectorID(db, taskID) + if err != nil { + log.Errorf("getting sector id: %s", err) + return "" + } + return sid.Miner.String() +} + +func (p *RSealClientPoll) GetSectorID(db *harmonydb.DB, taskID int64) (*abi.SectorID, error) { + var spId, sectorNumber uint64 + err := db.QueryRow(context.Background(), + `SELECT sp_id, sector_number FROM rseal_client_pipeline WHERE task_id_poll = $1`, taskID).Scan(&spId, §orNumber) + if err != nil { + return nil, err + } + return &abi.SectorID{ + Miner: abi.ActorID(spId), + Number: abi.SectorNumber(sectorNumber), + }, nil +} + +var _ = harmonytask.Reg(&RSealClientPoll{}) +var _ harmonytask.TaskInterface = &RSealClientPoll{} diff --git a/tasks/remoteseal/task_provider_cleanup.go b/tasks/remoteseal/task_provider_cleanup.go new file mode 100644 index 000000000..408c2316c --- /dev/null +++ b/tasks/remoteseal/task_provider_cleanup.go @@ -0,0 +1,182 @@ +package remoteseal + +import ( + "context" + "time" + + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-state-types/abi" + + "github.com/filecoin-project/curio/harmony/harmonydb" + "github.com/filecoin-project/curio/harmony/harmonytask" + "github.com/filecoin-project/curio/harmony/resources" + "github.com/filecoin-project/curio/harmony/taskhelp" + "github.com/filecoin-project/curio/lib/paths" + "github.com/filecoin-project/curio/lib/slotmgr" + "github.com/filecoin-project/curio/lib/storiface" +) + +// RSealProviderCleanup removes all sector data (sealed, cache, unsealed) from storage +// after the client has finished with the sector. This is triggered either by an explicit +// cleanup request from the client or by the cleanup timeout expiring. +type RSealProviderCleanup struct { + db *harmonydb.DB + sp *RSealProviderPoller + storage *paths.Remote + + // Batch slot manager, may be nil if not using batch sealing + slots *slotmgr.SlotMgr + + max int +} + +func NewProviderCleanupTask(db *harmonydb.DB, sp *RSealProviderPoller, storage *paths.Remote, slots *slotmgr.SlotMgr, maxCleanup int) *RSealProviderCleanup { + return &RSealProviderCleanup{ + db: db, + sp: sp, + storage: storage, + slots: slots, + max: maxCleanup, + } +} + +func (c *RSealProviderCleanup) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done bool, err error) { + ctx := context.Background() + + var sectors []struct { + SpID int64 `db:"sp_id"` + SectorNumber int64 `db:"sector_number"` + RegSealProof int64 `db:"reg_seal_proof"` + } + + err = c.db.Select(ctx, §ors, `SELECT sp_id, sector_number, reg_seal_proof + FROM rseal_provider_pipeline + WHERE task_id_cleanup = $1`, taskID) + if err != nil { + return false, xerrors.Errorf("getting sector for cleanup: %w", err) + } + + if len(sectors) != 1 { + return false, xerrors.Errorf("expected 1 sector for cleanup, got %d", len(sectors)) + } + task := sectors[0] + + sectorID := abi.SectorID{ + Miner: abi.ActorID(task.SpID), + Number: abi.SectorNumber(task.SectorNumber), + } + + if !stillOwned() { + return false, xerrors.Errorf("task no longer owned") + } + + // Remove sealed data + if err := c.storage.Remove(ctx, sectorID, storiface.FTSealed, true, nil); err != nil { + log.Warnw("cleanup: failed to remove sealed data (may not exist)", "error", err, + "sp", task.SpID, "sector", task.SectorNumber) + } + + // Remove cache data + if err := c.storage.Remove(ctx, sectorID, storiface.FTCache, true, nil); err != nil { + log.Warnw("cleanup: failed to remove cache data (may not exist)", "error", err, + "sp", task.SpID, "sector", task.SectorNumber) + } + + // Remove unsealed data (if any) + if err := c.storage.Remove(ctx, sectorID, storiface.FTUnsealed, true, nil); err != nil { + log.Warnw("cleanup: failed to remove unsealed data (may not exist)", "error", err, + "sp", task.SpID, "sector", task.SectorNumber) + } + + // If this sector was part of a batch, release the batch slot + if c.slots != nil { + var batchRefs []struct { + PipelineSlot int64 `db:"pipeline_slot"` + HostAndPort string `db:"machine_host_and_port"` + } + + // Look up batch refs for this sector + err = c.db.Select(ctx, &batchRefs, `SELECT pipeline_slot, machine_host_and_port + FROM batch_sector_refs + WHERE sp_id = $1 AND sector_number = $2`, task.SpID, task.SectorNumber) + if err != nil { + log.Warnw("cleanup: failed to query batch refs", "error", err, + "sp", task.SpID, "sector", task.SectorNumber) + } + + for _, ref := range batchRefs { + if err := c.slots.SectorDone(ctx, uint64(ref.PipelineSlot), sectorID); err != nil { + log.Warnw("cleanup: failed to release batch slot", "error", err, + "sp", task.SpID, "sector", task.SectorNumber, "slot", ref.PipelineSlot) + } + } + } + + // Mark cleanup as done + n, err := c.db.Exec(ctx, `UPDATE rseal_provider_pipeline + SET after_cleanup = TRUE, task_id_cleanup = NULL + WHERE sp_id = $1 AND sector_number = $2 AND task_id_cleanup = $3`, + task.SpID, task.SectorNumber, taskID) + if err != nil { + return false, xerrors.Errorf("updating cleanup status: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("expected to update 1 row for cleanup, updated %d", n) + } + + log.Infow("cleaned up remote seal sector", + "sp", task.SpID, + "sector", task.SectorNumber) + + return true, nil +} + +func (c *RSealProviderCleanup) CanAccept(ids []harmonytask.TaskID, _ *harmonytask.TaskEngine) ([]harmonytask.TaskID, error) { + // Cleanup can run on any node with storage access. Accept all. + return ids, nil +} + +func (c *RSealProviderCleanup) TypeDetails() harmonytask.TaskTypeDetails { + return harmonytask.TaskTypeDetails{ + Max: taskhelp.Max(c.max), + Name: "RSealProvCleanup", + Cost: resources.Resources{ + Cpu: 1, + Gpu: 0, + Ram: 64 << 20, + }, + MaxFailures: 10, + RetryWait: taskhelp.RetryWaitLinear(60*time.Second, 30*time.Second), + } +} + +func (c *RSealProviderCleanup) Adder(taskFunc harmonytask.AddTaskFunc) { + c.sp.pollers[pollerProvCleanup].Set(taskFunc) +} + +func (c *RSealProviderCleanup) GetSpid(db *harmonydb.DB, taskID int64) string { + sid, err := c.GetSectorID(db, taskID) + if err != nil { + log.Errorf("getting sector id: %s", err) + return "" + } + return sid.Miner.String() +} + +func (c *RSealProviderCleanup) GetSectorID(db *harmonydb.DB, taskID int64) (*abi.SectorID, error) { + var spId, sectorNumber uint64 + err := db.QueryRow(context.Background(), `SELECT sp_id, sector_number + FROM rseal_provider_pipeline + WHERE task_id_cleanup = $1`, taskID).Scan(&spId, §orNumber) + if err != nil { + return nil, xerrors.Errorf("getting sector id for cleanup task: %w", err) + } + return &abi.SectorID{ + Miner: abi.ActorID(spId), + Number: abi.SectorNumber(sectorNumber), + }, nil +} + +var _ = harmonytask.Reg(&RSealProviderCleanup{}) +var _ harmonytask.TaskInterface = &RSealProviderCleanup{} diff --git a/tasks/remoteseal/task_provider_finalize.go b/tasks/remoteseal/task_provider_finalize.go new file mode 100644 index 000000000..e871a43f3 --- /dev/null +++ b/tasks/remoteseal/task_provider_finalize.go @@ -0,0 +1,257 @@ +package remoteseal + +import ( + "context" + "errors" + "time" + + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-state-types/abi" + + "github.com/filecoin-project/curio/harmony/harmonydb" + "github.com/filecoin-project/curio/harmony/harmonytask" + "github.com/filecoin-project/curio/harmony/resources" + "github.com/filecoin-project/curio/harmony/taskhelp" + "github.com/filecoin-project/curio/lib/ffi" + "github.com/filecoin-project/curio/lib/slotmgr" + "github.com/filecoin-project/curio/lib/storiface" +) + +// RSealProviderFinalize drops SDR layers (cache) after C1 has been supplied. +// This is similar to the regular FinalizeTask but operates on rseal_provider_pipeline +// and does not need to handle unsealed data or deal pieces (delegated sectors are CC). +type RSealProviderFinalize struct { + db *harmonydb.DB + sp *RSealProviderPoller + sc *ffi.SealCalls + + // Batch slot manager, may be nil if not using batch sealing + slots *slotmgr.SlotMgr + + max int +} + +func NewProviderFinalizeTask(db *harmonydb.DB, sp *RSealProviderPoller, sc *ffi.SealCalls, slots *slotmgr.SlotMgr, maxFinalize int) *RSealProviderFinalize { + return &RSealProviderFinalize{ + db: db, + sp: sp, + sc: sc, + slots: slots, + max: maxFinalize, + } +} + +func (f *RSealProviderFinalize) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done bool, err error) { + ctx := context.Background() + + var sectors []struct { + SpID int64 `db:"sp_id"` + SectorNumber int64 `db:"sector_number"` + RegSealProof int64 `db:"reg_seal_proof"` + } + + err = f.db.Select(ctx, §ors, `SELECT sp_id, sector_number, reg_seal_proof + FROM rseal_provider_pipeline + WHERE task_id_finalize = $1`, taskID) + if err != nil { + return false, xerrors.Errorf("getting sector for finalize: %w", err) + } + + if len(sectors) != 1 { + return false, xerrors.Errorf("expected 1 sector for finalize, got %d", len(sectors)) + } + task := sectors[0] + + sector := storiface.SectorRef{ + ID: abi.SectorID{ + Miner: abi.ActorID(task.SpID), + Number: abi.SectorNumber(task.SectorNumber), + }, + ProofType: abi.RegisteredSealProof(task.RegSealProof), + } + + var ownedBy []struct { + HostAndPort string `db:"host_and_port"` + } + var refs []struct { + PipelineSlot int64 `db:"pipeline_slot"` + } + + var refFound bool + if f.slots != nil { + // batch handling part 1: get machine id + err = f.db.Select(ctx, &ownedBy, `SELECT hm.host_and_port as host_and_port FROM harmony_task INNER JOIN harmony_machines hm on harmony_task.owner_id = hm.id WHERE harmony_task.id = $1`, taskID) + if err != nil { + return false, xerrors.Errorf("getting machine id: %w", err) + } + + if len(ownedBy) != 1 { + return false, xerrors.Errorf("expected one machine") + } + + err = f.db.Select(ctx, &refs, `SELECT pipeline_slot FROM batch_sector_refs WHERE sp_id = $1 AND sector_number = $2 AND machine_host_and_port = $3`, task.SpID, task.SectorNumber, ownedBy[0].HostAndPort) + if err != nil { + return false, xerrors.Errorf("getting batch refs: %w", err) + } + + if len(refs) > 1 { + return false, xerrors.Errorf("expected one batch ref, got %d", len(refs)) + } + + refFound = len(refs) == 1 + } + + if !stillOwned() { + return false, xerrors.Errorf("task no longer owned") + } + + // For remote seal finalize, we clear the cache (drop SDR layers). + // Delegated sectors are always CC (no unsealed data to preserve). + // If the sector files are already gone (cleanup ran first or client called cleanup early), + // treat it as a no-op and just mark the task as done. + err = f.sc.FinalizeSector(ctx, sector, false) + if err != nil { + if errors.Is(err, storiface.ErrSectorNotFound) { + log.Warnw("finalize: sector files already removed, treating as no-op", + "sp", task.SpID, "sector", task.SectorNumber) + } else { + return false, xerrors.Errorf("finalizing remote seal sector: %w", err) + } + } + + if refFound { + // batch handling part 2: release the batch slot + if err := f.slots.SectorDone(ctx, uint64(refs[0].PipelineSlot), sector.ID); err != nil { + return false, xerrors.Errorf("mark batch ref done: %w", err) + } + } + + // Mark finalize as done + n, err := f.db.Exec(ctx, `UPDATE rseal_provider_pipeline + SET after_finalize = TRUE, task_id_finalize = NULL + WHERE sp_id = $1 AND sector_number = $2 AND task_id_finalize = $3`, + task.SpID, task.SectorNumber, taskID) + if err != nil { + return false, xerrors.Errorf("updating finalize status: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("expected to update 1 row for finalize, updated %d", n) + } + + log.Infow("finalized remote seal sector", + "sp", task.SpID, + "sector", task.SectorNumber) + + return true, nil +} + +func (f *RSealProviderFinalize) CanAccept(ids []harmonytask.TaskID, _ *harmonytask.TaskEngine) ([]harmonytask.TaskID, error) { + // Check that the sector's cache is on local storage, similar to the regular finalize task. + // If the sector files have already been removed (e.g. cleanup ran first or client called + // cleanup early), accept the task on any node — Do() will treat it as a no-op. + var tasks []struct { + TaskID harmonytask.TaskID `db:"task_id_finalize"` + SpID int64 `db:"sp_id"` + SectorNumber int64 `db:"sector_number"` + StorageID *string `db:"storage_id"` + } + + if storiface.FTCache != 4 { + panic("storiface.FTCache != 4") + } + + ctx := context.Background() + + indIDs := make([]int64, len(ids)) + for i, id := range ids { + indIDs[i] = int64(id) + } + + // Use LEFT JOIN so that tasks whose sector_location rows have been removed + // still appear in the result set (with NULL storage_id). + err := f.db.Select(ctx, &tasks, ` + SELECT p.task_id_finalize, p.sp_id, p.sector_number, l.storage_id + FROM rseal_provider_pipeline p + LEFT JOIN sector_location l ON p.sp_id = l.miner_id AND p.sector_number = l.sector_num AND l.sector_filetype = 4 + WHERE task_id_finalize = ANY ($1)`, indIDs) + if err != nil { + return []harmonytask.TaskID{}, xerrors.Errorf("getting finalize tasks: %w", err) + } + + ls, err := f.sc.LocalStorage(ctx) + if err != nil { + return []harmonytask.TaskID{}, xerrors.Errorf("getting local storage: %w", err) + } + + acceptables := map[harmonytask.TaskID]bool{} + for _, id := range ids { + acceptables[id] = true + } + + var result []harmonytask.TaskID + for _, t := range tasks { + if _, ok := acceptables[t.TaskID]; !ok { + continue + } + + // If no sector_location entry exists, the files are already gone. + // Accept on any node — Do() will be a no-op beyond marking done. + if t.StorageID == nil { + result = append(result, t.TaskID) + continue + } + + for _, l := range ls { + if string(l.ID) == *t.StorageID { + result = append(result, t.TaskID) + } + } + } + + return result, nil +} + +func (f *RSealProviderFinalize) TypeDetails() harmonytask.TaskTypeDetails { + return harmonytask.TaskTypeDetails{ + Max: taskhelp.Max(f.max), + Name: "RSealProvFinal", + Cost: resources.Resources{ + Cpu: 1, + Gpu: 0, + Ram: 100 << 20, + }, + MaxFailures: 10, + RetryWait: taskhelp.RetryWaitLinear(30*time.Second, 15*time.Second), + } +} + +func (f *RSealProviderFinalize) Adder(taskFunc harmonytask.AddTaskFunc) { + f.sp.pollers[pollerProvFinalize].Set(taskFunc) +} + +func (f *RSealProviderFinalize) GetSpid(db *harmonydb.DB, taskID int64) string { + sid, err := f.GetSectorID(db, taskID) + if err != nil { + log.Errorf("getting sector id: %s", err) + return "" + } + return sid.Miner.String() +} + +func (f *RSealProviderFinalize) GetSectorID(db *harmonydb.DB, taskID int64) (*abi.SectorID, error) { + var spId, sectorNumber uint64 + err := db.QueryRow(context.Background(), `SELECT sp_id, sector_number + FROM rseal_provider_pipeline + WHERE task_id_finalize = $1`, taskID).Scan(&spId, §orNumber) + if err != nil { + return nil, xerrors.Errorf("getting sector id for finalize task: %w", err) + } + return &abi.SectorID{ + Miner: abi.ActorID(spId), + Number: abi.SectorNumber(sectorNumber), + }, nil +} + +var _ = harmonytask.Reg(&RSealProviderFinalize{}) +var _ harmonytask.TaskInterface = &RSealProviderFinalize{} diff --git a/tasks/remoteseal/task_provider_notify.go b/tasks/remoteseal/task_provider_notify.go new file mode 100644 index 000000000..08661efc2 --- /dev/null +++ b/tasks/remoteseal/task_provider_notify.go @@ -0,0 +1,216 @@ +package remoteseal + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "time" + + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-state-types/abi" + + "github.com/filecoin-project/curio/harmony/harmonydb" + "github.com/filecoin-project/curio/harmony/harmonytask" + "github.com/filecoin-project/curio/harmony/resources" + "github.com/filecoin-project/curio/harmony/taskhelp" + "github.com/filecoin-project/curio/market/sealmarket" +) + +const defaultCleanupTimeout = 72 * time.Hour + +type RSealProviderNotify struct { + db *harmonydb.DB + sp *RSealProviderPoller + + max int + cleanupTimeout time.Duration + + httpClient *http.Client +} + +func NewProviderNotifyTask(db *harmonydb.DB, sp *RSealProviderPoller, maxTasks int, cleanupTimeout time.Duration) *RSealProviderNotify { + if cleanupTimeout <= 0 { + cleanupTimeout = defaultCleanupTimeout + } + return &RSealProviderNotify{ + db: db, + sp: sp, + max: maxTasks, + cleanupTimeout: cleanupTimeout, + httpClient: &http.Client{ + Timeout: 60 * time.Second, + }, + } +} + +func (t *RSealProviderNotify) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done bool, err error) { + ctx := context.Background() + + // Find the sector assigned to this task + var sectors []struct { + SpID int64 `db:"sp_id"` + SectorNumber int64 `db:"sector_number"` + PartnerID int64 `db:"partner_id"` + TreeDCid string `db:"tree_d_cid"` + TreeRCid string `db:"tree_r_cid"` + TicketEpoch int64 `db:"ticket_epoch"` + TicketValue []byte `db:"ticket_value"` + } + + err = t.db.Select(ctx, §ors, `SELECT sp_id, sector_number, partner_id, tree_d_cid, tree_r_cid, ticket_epoch, ticket_value + FROM rseal_provider_pipeline + WHERE task_id_notify_client = $1`, taskID) + if err != nil { + return false, xerrors.Errorf("getting sector for notify: %w", err) + } + + if len(sectors) != 1 { + return false, xerrors.Errorf("expected 1 sector for notify, got %d", len(sectors)) + } + sector := sectors[0] + + // Look up the partner URL and token + var partners []struct { + PartnerURL string `db:"partner_url"` + PartnerToken string `db:"partner_token"` + } + + err = t.db.Select(ctx, &partners, `SELECT partner_url, partner_token + FROM rseal_delegated_partners + WHERE id = $1`, sector.PartnerID) + if err != nil { + return false, xerrors.Errorf("getting partner info: %w", err) + } + + if len(partners) != 1 { + return false, xerrors.Errorf("expected 1 partner, got %d", len(partners)) + } + partner := partners[0] + + if !stillOwned() { + return false, xerrors.Errorf("task no longer owned") + } + + // Notify the client that SDR+trees are complete (includes ticket for the client) + notification := sealmarket.CompleteNotification{ + PartnerToken: partner.PartnerToken, + SpID: sector.SpID, + SectorNumber: sector.SectorNumber, + TreeDCid: sector.TreeDCid, + TreeRCid: sector.TreeRCid, + TicketEpoch: sector.TicketEpoch, + TicketValue: sector.TicketValue, + } + + err = t.sendCompleteNotification(ctx, partner.PartnerURL, notification) + if err != nil { + return false, xerrors.Errorf("sending complete notification: %w", err) + } + + // Mark notification as done and set the cleanup timeout + cleanupTimeoutStr := fmt.Sprintf("%d seconds", int(t.cleanupTimeout.Seconds())) + n, err := t.db.Exec(ctx, `UPDATE rseal_provider_pipeline + SET after_notify_client = TRUE, task_id_notify_client = NULL, + cleanup_timeout = NOW() + $4::interval + WHERE sp_id = $1 AND sector_number = $2 AND task_id_notify_client = $3`, + sector.SpID, sector.SectorNumber, taskID, cleanupTimeoutStr) + if err != nil { + return false, xerrors.Errorf("updating notify status: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("expected to update 1 row for notify, updated %d", n) + } + + log.Infow("notified client of remote seal completion", + "sp", sector.SpID, + "sector", sector.SectorNumber, + "treeDCid", sector.TreeDCid, + "treeRCid", sector.TreeRCid) + + return true, nil +} + +func (t *RSealProviderNotify) sendCompleteNotification(ctx context.Context, partnerURL string, notification sealmarket.CompleteNotification) error { + body, err := json.Marshal(notification) + if err != nil { + return xerrors.Errorf("marshaling complete notification: %w", err) + } + + completeURL, err := url.JoinPath(partnerURL, sealmarket.DelegatedSealPath, "complete") + if err != nil { + return xerrors.Errorf("building complete notification URL: %w", err) + } + + httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, completeURL, bytes.NewReader(body)) + if err != nil { + return xerrors.Errorf("creating complete notification request: %w", err) + } + httpReq.Header.Set("Content-Type", "application/json") + + resp, err := t.httpClient.Do(httpReq) + if err != nil { + return xerrors.Errorf("sending complete notification: %w", err) + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + respBody, _ := io.ReadAll(resp.Body) + return xerrors.Errorf("complete notification failed with status %d: %s", resp.StatusCode, string(respBody)) + } + + return nil +} + +func (t *RSealProviderNotify) CanAccept(ids []harmonytask.TaskID, _ *harmonytask.TaskEngine) ([]harmonytask.TaskID, error) { + // Notification is a lightweight HTTP call; accept all offered tasks. + return ids, nil +} + +func (t *RSealProviderNotify) TypeDetails() harmonytask.TaskTypeDetails { + return harmonytask.TaskTypeDetails{ + Max: taskhelp.Max(t.max), + Name: "RSealProvNotify", + Cost: resources.Resources{ + Cpu: 1, + Gpu: 0, + Ram: 64 << 20, + }, + MaxFailures: 100, + RetryWait: taskhelp.RetryWaitLinear(60*time.Second, 30*time.Second), + } +} + +func (t *RSealProviderNotify) Adder(taskFunc harmonytask.AddTaskFunc) { + t.sp.pollers[pollerProvNotifyClient].Set(taskFunc) +} + +func (t *RSealProviderNotify) GetSpid(db *harmonydb.DB, taskID int64) string { + sid, err := t.GetSectorID(db, taskID) + if err != nil { + log.Errorf("getting sector id: %s", err) + return "" + } + return sid.Miner.String() +} + +func (t *RSealProviderNotify) GetSectorID(db *harmonydb.DB, taskID int64) (*abi.SectorID, error) { + var spId, sectorNumber uint64 + err := db.QueryRow(context.Background(), `SELECT sp_id, sector_number + FROM rseal_provider_pipeline + WHERE task_id_notify_client = $1`, taskID).Scan(&spId, §orNumber) + if err != nil { + return nil, xerrors.Errorf("getting sector id for notify task: %w", err) + } + return &abi.SectorID{ + Miner: abi.ActorID(spId), + Number: abi.SectorNumber(sectorNumber), + }, nil +} + +var _ = harmonytask.Reg(&RSealProviderNotify{}) +var _ harmonytask.TaskInterface = &RSealProviderNotify{} diff --git a/tasks/seal/cc_scheduler.go b/tasks/seal/cc_scheduler.go new file mode 100644 index 000000000..d8ce6612e --- /dev/null +++ b/tasks/seal/cc_scheduler.go @@ -0,0 +1,158 @@ +package seal + +import ( + "context" + "time" + + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-bitfield" + "github.com/filecoin-project/go-state-types/builtin" + miner12 "github.com/filecoin-project/go-state-types/builtin/v12/miner" + + "github.com/filecoin-project/curio/harmony/harmonydb" + "github.com/filecoin-project/curio/harmony/harmonytask" + "github.com/filecoin-project/curio/lib/passcall" + + lotusapi "github.com/filecoin-project/lotus/api" + apitypes "github.com/filecoin-project/lotus/api/types" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" + "github.com/filecoin-project/lotus/chain/types" +) + +// CCSchedulerAPI extends SDRAPI with methods needed for CC sector scheduling: +// allocating sector numbers and determining proof types from chain state. +type CCSchedulerAPI interface { + SDRAPI + StateMinerAllocated(context.Context, address.Address, types.TipSetKey) (*bitfield.BitField, error) + StateMinerInfo(context.Context, address.Address, types.TipSetKey) (lotusapi.MinerInfo, error) + StateNetworkVersion(context.Context, types.TipSetKey) (apitypes.NetworkVersion, error) +} + +type ccSchedule struct { + SpID int64 `db:"sp_id"` + ToSeal int64 `db:"to_seal"` + DurationDays int64 `db:"duration_days"` +} + +// scheduleCC is the IAmBored callback for the SDR task. It creates new CC sectors +// from the sectors_cc_scheduler table for SPs that do NOT have an enabled remote +// seal provider. This ensures CC sectors are only created where SDR can actually run +// locally (IAmBored is only invoked when the machine has SDR capacity). +// +// SPs with enabled remote providers are handled by RSealDelegate instead. +func (s *SDRTask) scheduleCC(taskFunc harmonytask.AddTaskFunc) error { + if s.ccAPI == nil { + return nil // CC scheduling not configured + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + // Find enabled CC schedules for SPs that do NOT have any enabled remote providers. + // If an SP has an enabled remote provider, RSealDelegate handles CC scheduling for it. + var schedules []ccSchedule + err := s.db.Select(ctx, &schedules, ` + SELECT cs.sp_id, cs.to_seal, cs.duration_days + FROM sectors_cc_scheduler cs + WHERE cs.enabled = TRUE + AND cs.to_seal > 0 + AND NOT EXISTS ( + SELECT 1 FROM rseal_client_providers rcp + WHERE rcp.sp_id = cs.sp_id AND rcp.enabled = TRUE + ) + ORDER BY cs.sp_id`) + if err != nil { + return xerrors.Errorf("querying cc_scheduler: %w", err) + } + + if len(schedules) == 0 { + return nil + } + + nv, err := s.ccAPI.StateNetworkVersion(ctx, types.EmptyTSK) + if err != nil { + return xerrors.Errorf("getting network version: %w", err) + } + + // Create one sector per IAmBored invocation (conservative — only when capacity confirmed). + // Pick the first SP with to_seal > 0. + schedule := schedules[0] + + maddr, err := address.NewIDAddress(uint64(schedule.SpID)) + if err != nil { + return xerrors.Errorf("creating miner address: %w", err) + } + + mi, err := s.ccAPI.StateMinerInfo(ctx, maddr, types.EmptyTSK) + if err != nil { + return xerrors.Errorf("getting miner info for %s: %w", maddr, err) + } + + spt, err := miner.PreferredSealProofTypeFromWindowPoStType(nv, mi.WindowPoStProofType, false) + if err != nil { + return xerrors.Errorf("getting seal proof type: %w", err) + } + + userDuration := schedule.DurationDays * builtin.EpochsInDay + if miner12.MaxSectorExpirationExtension < userDuration { + return xerrors.Errorf("duration exceeds max allowed: %d > %d", userDuration, miner12.MaxSectorExpirationExtension) + } + if miner12.MinSectorExpiration > userDuration { + return xerrors.Errorf("duration is too short: %d < %d", userDuration, miner12.MinSectorExpiration) + } + + taskFunc(func(id harmonytask.TaskID, tx *harmonydb.Tx) (bool, error) { + // Allocate one sector number inside the transaction + sectorNumbers, err := AllocateSectorNumbers(ctx, s.ccAPI, tx, maddr, 1) + if err != nil { + return false, xerrors.Errorf("allocating sector number: %w", err) + } + if len(sectorNumbers) != 1 { + return false, xerrors.Errorf("expected 1 sector number, got %d", len(sectorNumbers)) + } + + sectorNum := sectorNumbers[0] + + // Insert into sectors_sdr_pipeline + _, err = tx.Exec(`INSERT INTO sectors_sdr_pipeline (sp_id, sector_number, reg_seal_proof, user_sector_duration_epochs) + VALUES ($1, $2, $3, $4)`, + schedule.SpID, sectorNum, spt, userDuration) + if err != nil { + return false, xerrors.Errorf("inserting sector %d for SP %d: %w", sectorNum, schedule.SpID, err) + } + + // Assign SDR task to the new sector + n, err := tx.Exec(`UPDATE sectors_sdr_pipeline SET task_id_sdr = $1 + WHERE sp_id = $2 AND sector_number = $3 AND task_id_sdr IS NULL`, + id, schedule.SpID, sectorNum) + if err != nil { + return false, xerrors.Errorf("setting task_id_sdr: %w", err) + } + if n != 1 { + return false, xerrors.Errorf("expected to claim 1 sector, got %d", n) + } + + // Decrement to_seal + _, err = tx.Exec(`UPDATE sectors_cc_scheduler SET to_seal = to_seal - 1 WHERE sp_id = $1 AND to_seal > 0`, schedule.SpID) + if err != nil { + return false, xerrors.Errorf("decrementing to_seal: %w", err) + } + + log.Infow("CC scheduler: created local CC sector", + "sp_id", schedule.SpID, + "sector", sectorNum, + "proof", spt, + "duration_days", schedule.DurationDays) + + return true, nil + }) + + return nil +} + +// NewScheduleCCFunc returns a rate-limited scheduleCC suitable for IAmBored. +func (s *SDRTask) newScheduleCCFunc() func(harmonytask.AddTaskFunc) error { + return passcall.Every(15*time.Second, s.scheduleCC) +} diff --git a/tasks/seal/poller.go b/tasks/seal/poller.go index 69aa85e8c..ad06e6a2f 100644 --- a/tasks/seal/poller.go +++ b/tasks/seal/poller.go @@ -162,6 +162,9 @@ type pollTask struct { AfterMoveStorage bool `db:"after_move_storage"` // 1 byte AfterCommitMsg bool `db:"after_commit_msg"` // 1 byte AfterCommitMsgSuccess bool `db:"after_commit_msg_success"` // 1 byte + // Remote seal flags + IsRemote bool `db:"is_remote"` // true when sector has rseal_client_pipeline entry + AfterFetch bool `db:"after_fetch"` // true when sealed data has been fetched from provider (remote only) // Larger fields at end PoRepProof []byte `db:"porep_proof"` // 24 bytes - only used in specific stages FailedReason string `db:"failed_reason"` // 16 bytes - only used when Failed=true @@ -170,44 +173,47 @@ type pollTask struct { func (s *SealPoller) poll(ctx context.Context) error { var tasks []pollTask - err := s.db.Select(ctx, &tasks, `SELECT - p.sp_id, - p.sector_number, - p.reg_seal_proof, + err := s.db.Select(ctx, &tasks, `SELECT + p.sp_id, + p.sector_number, + p.reg_seal_proof, p.ticket_epoch, - p.task_id_sdr, + p.task_id_sdr, p.after_sdr, - p.task_id_tree_d, + p.task_id_tree_d, p.after_tree_d, - p.task_id_tree_c, + p.task_id_tree_c, p.after_tree_c, - p.task_id_tree_r, + p.task_id_tree_r, p.after_tree_r, - p.task_id_synth, + p.task_id_synth, p.after_synth, p.precommit_ready_at, - p.task_id_precommit_msg, + p.task_id_precommit_msg, p.after_precommit_msg, - p.after_precommit_msg_success, + p.after_precommit_msg_success, p.seed_epoch, - p.task_id_porep, - p.porep_proof, + p.task_id_porep, + p.porep_proof, p.after_porep, - p.task_id_finalize, + p.task_id_finalize, p.after_finalize, - p.task_id_move_storage, + p.task_id_move_storage, p.after_move_storage, p.commit_ready_at, - p.task_id_commit_msg, + p.task_id_commit_msg, p.after_commit_msg, p.after_commit_msg_success, - p.failed, + p.failed, p.failed_reason, - p.start_epoch - FROM + p.start_epoch, + (c.sp_id IS NOT NULL) AS is_remote, + COALESCE(c.after_fetch, TRUE) AS after_fetch + FROM sectors_sdr_pipeline p - WHERE - p.after_commit_msg_success != TRUE + LEFT JOIN rseal_client_pipeline c ON p.sp_id = c.sp_id AND p.sector_number = c.sector_number + WHERE + p.after_commit_msg_success != TRUE OR p.after_move_storage != TRUE;`) if err != nil { return err @@ -223,10 +229,16 @@ func (s *SealPoller) poll(ctx context.Context) error { for _, task := range tasks { - s.pollStartSDR(ctx, task) - s.pollStartSDRTreeD(ctx, task) - s.pollStartSDRTreeRC(ctx, task) - s.pollStartSynth(ctx, task) + if !task.IsRemote { + // Local sectors: run SDR, TreeD, TreeRC, Synth locally. + // Remote sectors skip these - they are handled by the provider + // and the client poller (RSealClientPoller) manages the remote pipeline. + s.pollStartSDR(ctx, task) + s.pollStartSDRTreeD(ctx, task) + s.pollStartSDRTreeRC(ctx, task) + s.pollStartSynth(ctx, task) + } + // PreCommit, PoRep, Finalize, MoveStorage, Commit run for both local and remote sectors s.mustPoll(s.pollPrecommitMsgLanded(ctx, task)) s.pollStartPoRep(ctx, task, ts) s.mustPoll(s.pollerAddStartEpoch(ctx, task)) @@ -334,6 +346,7 @@ func (t pollTask) afterPrecommitMsgSuccess() bool { func (s *SealPoller) pollStartPoRep(ctx context.Context, task pollTask, ts *types.TipSet) { if s.pollers[pollerPoRep].IsSet() && task.afterPrecommitMsgSuccess() && task.SeedEpoch.Valid && !task.TaskPoRep.Valid && !task.AfterPoRep && + task.AfterFetch && // Remote sectors: sealed data must be fetched before PoRep (local: always true) ts.Height() >= abi.ChainEpoch(task.SeedEpoch.Int64+seedEpochConfidence) { s.pollers[pollerPoRep].Val(ctx)(func(id harmonytask.TaskID, tx *harmonydb.Tx) (shouldCommit bool, seriousError error) { diff --git a/tasks/seal/task_porep.go b/tasks/seal/task_porep.go index fcff35802..f3e7cbae0 100644 --- a/tasks/seal/task_porep.go +++ b/tasks/seal/task_porep.go @@ -122,7 +122,8 @@ func (p *PoRepTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done } // COMPUTE THE PROOF! - + // GeneratePoRepVanillaProof (called by PoRepSnark) handles remote-sealed + // sectors transparently via the c1.url file in the cache directory. var proof []byte if p.cuzkClient != nil && p.cuzkClient.Enabled() { proof, err = p.sc.PoRepSnarkCuzk(ctx, p.cuzkClient, sr, sealed, unsealed, sectorParams.TicketValue, abi.InteractiveSealRandomness(rand)) diff --git a/tasks/seal/task_sdr.go b/tasks/seal/task_sdr.go index 9d87c5f7d..5ee1e493b 100644 --- a/tasks/seal/task_sdr.go +++ b/tasks/seal/task_sdr.go @@ -3,6 +3,12 @@ package seal import ( "bytes" "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "time" "go.opencensus.io/stats" "go.opencensus.io/tag" @@ -42,25 +48,44 @@ type SDRAPI interface { StateGetRandomnessFromTickets(context.Context, crypto.DomainSeparationTag, abi.ChainEpoch, []byte, types.TipSetKey) (abi.Randomness, error) } +// ProviderPollerSDR is an interface that allows registering the SDR task's +// AddTaskFunc with the remote seal provider poller. This enables the provider +// poller to schedule SDR tasks for rseal_provider_pipeline rows. +type ProviderPollerSDR interface { + SetPollerSDR(harmonytask.AddTaskFunc) +} + type SDRTask struct { - api SDRAPI - db *harmonydb.DB - sp *SealPoller + api SDRAPI + ccAPI CCSchedulerAPI // optional, nil disables CC scheduling + db *harmonydb.DB + sp *SealPoller sc *ffi2.SealCalls + provPoller ProviderPollerSDR // optional, nil when remote seal provider is not enabled + max taskhelp.Limiter min int } -func NewSDRTask(api SDRAPI, db *harmonydb.DB, sp *SealPoller, sc *ffi2.SealCalls, maxSDR taskhelp.Limiter, minSDR int) *SDRTask { +func NewSDRTask(api SDRAPI, db *harmonydb.DB, sp *SealPoller, sc *ffi2.SealCalls, maxSDR taskhelp.Limiter, minSDR int, provPoller ProviderPollerSDR) *SDRTask { + // If the API also satisfies CCSchedulerAPI, enable CC scheduling. + // This is the case when the full chain API is passed (normal operation). + var ccAPI CCSchedulerAPI + if ca, ok := api.(CCSchedulerAPI); ok { + ccAPI = ca + } + return &SDRTask{ - api: api, - db: db, - sp: sp, - sc: sc, - max: maxSDR, - min: minSDR, + api: api, + ccAPI: ccAPI, + db: db, + sp: sp, + sc: sc, + provPoller: provPoller, + max: maxSDR, + min: minSDR, } } @@ -71,11 +96,16 @@ func (s *SDRTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done bo SpID int64 `db:"sp_id"` SectorNumber int64 `db:"sector_number"` RegSealProof abi.RegisteredSealProof `db:"reg_seal_proof"` + Pipeline string `db:"pipeline"` } err = s.db.Select(ctx, §orParamsArr, ` - SELECT sp_id, sector_number, reg_seal_proof + SELECT sp_id, sector_number, reg_seal_proof, 'local' as pipeline FROM sectors_sdr_pipeline + WHERE task_id_sdr = $1 + UNION ALL + SELECT sp_id, sector_number, reg_seal_proof, 'remote' as pipeline + FROM rseal_provider_pipeline WHERE task_id_sdr = $1`, taskID) if err != nil { return false, xerrors.Errorf("getting sector params: %w", err) @@ -107,7 +137,15 @@ func (s *SDRTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done bo // FAIL: api may be down // FAIL-RESP: rely on harmony retry - ticket, ticketEpoch, err := GetTicket(ctx, s.api, maddr) + var ticket abi.SealRandomness + var ticketEpoch abi.ChainEpoch + + if sectorParams.Pipeline == "remote" { + // Remote sector: fetch ticket from the client's chain (may be a different network) + ticket, ticketEpoch, err = GetRemoteTicket(ctx, s.db, sectorParams.SpID, sectorParams.SectorNumber, maddr) + } else { + ticket, ticketEpoch, err = GetTicket(ctx, s.api, maddr) + } if err != nil { return false, xerrors.Errorf("getting ticket: %w", err) } @@ -128,10 +166,18 @@ func (s *SDRTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done bo } // store success! - n, err := s.db.Exec(ctx, `UPDATE sectors_sdr_pipeline - SET after_sdr = true, ticket_epoch = $3, ticket_value = $4, task_id_sdr = NULL - WHERE sp_id = $1 AND sector_number = $2`, - sectorParams.SpID, sectorParams.SectorNumber, ticketEpoch, []byte(ticket)) + var n int + if sectorParams.Pipeline == "remote" { + n, err = s.db.Exec(ctx, `UPDATE rseal_provider_pipeline + SET after_sdr = true, ticket_epoch = $3, ticket_value = $4, task_id_sdr = NULL + WHERE sp_id = $1 AND sector_number = $2`, + sectorParams.SpID, sectorParams.SectorNumber, ticketEpoch, []byte(ticket)) + } else { + n, err = s.db.Exec(ctx, `UPDATE sectors_sdr_pipeline + SET after_sdr = true, ticket_epoch = $3, ticket_value = $4, task_id_sdr = NULL + WHERE sp_id = $1 AND sector_number = $2`, + sectorParams.SpID, sectorParams.SectorNumber, ticketEpoch, []byte(ticket)) + } if err != nil { return false, xerrors.Errorf("store sdr success: updating pipeline: %w", err) } @@ -174,6 +220,66 @@ func GetTicket(ctx context.Context, api TicketNodeAPI, maddr address.Address) (a return abi.SealRandomness(rand), ticketEpoch, nil } +// GetRemoteTicket fetches seal randomness from a remote client's chain node +// via the client's /ticket HTTP endpoint. This is used when the provider is +// sealing a sector on behalf of a remote client that may be on a different +// network (e.g., provider on mainnet, client on calibnet). +func GetRemoteTicket(ctx context.Context, db *harmonydb.DB, spID, sectorNumber int64, maddr address.Address) (abi.SealRandomness, abi.ChainEpoch, error) { + // Look up partner_url for this remote sector + var partners []struct { + PartnerURL string `db:"partner_url"` + } + + err := db.Select(ctx, &partners, ` + SELECT dp.partner_url + FROM rseal_provider_pipeline rp + JOIN rseal_delegated_partners dp ON rp.partner_id = dp.id + WHERE rp.sp_id = $1 AND rp.sector_number = $2`, spID, sectorNumber) + if err != nil { + return nil, 0, xerrors.Errorf("looking up partner URL: %w", err) + } + if len(partners) == 0 { + return nil, 0, xerrors.Errorf("no partner found for remote sector %d/%d", spID, sectorNumber) + } + + partnerURL := partners[0].PartnerURL + + // Build ticket endpoint URL + ticketURL, err := url.JoinPath(partnerURL, "/remoteseal/delegated/v0/ticket") + if err != nil { + return nil, 0, xerrors.Errorf("building ticket URL: %w", err) + } + ticketURL = fmt.Sprintf("%s?maddr=%s", ticketURL, maddr.String()) + + // Make HTTP request to the client's ticket endpoint + httpClient := &http.Client{Timeout: 30 * time.Second} + req, err := http.NewRequestWithContext(ctx, http.MethodGet, ticketURL, nil) + if err != nil { + return nil, 0, xerrors.Errorf("creating ticket request: %w", err) + } + + resp, err := httpClient.Do(req) + if err != nil { + return nil, 0, xerrors.Errorf("fetching ticket from %s: %w", ticketURL, err) + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return nil, 0, xerrors.Errorf("ticket endpoint returned %d: %s", resp.StatusCode, string(body)) + } + + var ticketResp struct { + Ticket []byte `json:"ticket"` + Epoch int64 `json:"epoch"` + } + if err := json.NewDecoder(resp.Body).Decode(&ticketResp); err != nil { + return nil, 0, xerrors.Errorf("decoding ticket response: %w", err) + } + + return abi.SealRandomness(ticketResp.Ticket), abi.ChainEpoch(ticketResp.Epoch), nil +} + func (s *SDRTask) CanAccept(ids []harmonytask.TaskID, _ *harmonytask.TaskEngine) ([]harmonytask.TaskID, error) { if s.min > len(ids) { log.Debugw("did not accept task", "name", "SDR", "reason", "below min", "min", s.min, "count", len(ids)) @@ -200,6 +306,7 @@ func (s *SDRTask) TypeDetails() harmonytask.TaskTypeDetails { }, MaxFailures: 2, Follows: nil, + IAmBored: s.newScheduleCCFunc(), } if IsDevnet { @@ -212,6 +319,9 @@ func (s *SDRTask) TypeDetails() harmonytask.TaskTypeDetails { func (s *SDRTask) Adder(taskFunc harmonytask.AddTaskFunc) { s.sp.pollers[pollerSDR].Set(taskFunc) + if s.provPoller != nil { + s.provPoller.SetPollerSDR(taskFunc) + } } func (s *SDRTask) GetSpid(db *harmonydb.DB, taskID int64) string { @@ -225,7 +335,11 @@ func (s *SDRTask) GetSpid(db *harmonydb.DB, taskID int64) string { func (s *SDRTask) GetSectorID(db *harmonydb.DB, taskID int64) (*abi.SectorID, error) { var spId, sectorNumber uint64 - err := db.QueryRow(context.Background(), `SELECT sp_id,sector_number FROM sectors_sdr_pipeline WHERE task_id_sdr = $1`, taskID).Scan(&spId, §orNumber) + err := db.QueryRow(context.Background(), `SELECT sp_id, sector_number FROM ( + SELECT sp_id, sector_number FROM sectors_sdr_pipeline WHERE task_id_sdr = $1 + UNION ALL + SELECT sp_id, sector_number FROM rseal_provider_pipeline WHERE task_id_sdr = $1 + ) s`, taskID).Scan(&spId, §orNumber) if err != nil { return nil, err } @@ -240,7 +354,10 @@ var _ = harmonytask.Reg(&SDRTask{}) func (s *SDRTask) taskToSector(id harmonytask.TaskID) (ffi2.SectorRef, error) { var refs []ffi2.SectorRef - err := s.db.Select(context.Background(), &refs, `SELECT sp_id, sector_number, reg_seal_proof FROM sectors_sdr_pipeline WHERE task_id_sdr = $1`, id) + err := s.db.Select(context.Background(), &refs, ` + SELECT sp_id, sector_number, reg_seal_proof FROM sectors_sdr_pipeline WHERE task_id_sdr = $1 + UNION ALL + SELECT sp_id, sector_number, reg_seal_proof FROM rseal_provider_pipeline WHERE task_id_sdr = $1`, id) if err != nil { return ffi2.SectorRef{}, xerrors.Errorf("getting sector ref: %w", err) } diff --git a/tasks/seal/task_synth_proofs.go b/tasks/seal/task_synth_proofs.go index 1d568a822..a303c1b80 100644 --- a/tasks/seal/task_synth_proofs.go +++ b/tasks/seal/task_synth_proofs.go @@ -105,7 +105,7 @@ func (s *SyntheticProofTask) Do(taskID harmonytask.TaskID, stillOwned func() boo err = s.sc.SyntheticProofs(ctx, &taskID, sref, sealed, unsealed, sectorParams.TicketValue, dealData.PieceInfos, keepUnsealed) if err != nil { - serr := resetSectorSealingState(ctx, sectorParams.SpID, sectorParams.SectorNumber, err, s.db, s.TypeDetails().Name) + serr := resetSectorSealingState(ctx, sectorParams.SpID, sectorParams.SectorNumber, err, s.db, s.TypeDetails().Name, "local") if serr != nil { return false, xerrors.Errorf("generating synthetic proofs: %w", err) } @@ -128,13 +128,21 @@ func (s *SyntheticProofTask) Do(taskID harmonytask.TaskID, stillOwned func() boo return true, nil } -func resetSectorSealingState(ctx context.Context, spid, secNum int64, err error, db *harmonydb.DB, name string) error { +func resetSectorSealingState(ctx context.Context, spid, secNum int64, err error, db *harmonydb.DB, name string, pipeline string) error { if err != nil { if strings.Contains(err.Error(), "checking PreCommit") { - n, serr := db.Exec(ctx, `UPDATE sectors_sdr_pipeline + var n int + var serr error + if pipeline == "remote" { + n, serr = db.Exec(ctx, `UPDATE rseal_provider_pipeline + SET after_tree_d = false, tree_d_cid = NULL, after_tree_r = false, after_tree_c = false, task_id_tree_r = NULL, task_id_tree_c = NULL + WHERE sp_id = $1 AND sector_number = $2`, spid, secNum) + } else { + n, serr = db.Exec(ctx, `UPDATE sectors_sdr_pipeline SET after_tree_d = false, tree_d_cid = NULL, after_tree_r = false, after_tree_c = false, task_id_tree_r = NULL, task_id_tree_c = NULL, after_synth = false, task_id_synth = null WHERE sp_id = $1 AND sector_number = $2`, spid, secNum) + } if serr != nil { return xerrors.Errorf("store %s failure: updating pipeline: Original error %w: DB error %w", name, err, serr) } @@ -148,7 +156,9 @@ func resetSectorSealingState(ctx context.Context, spid, secNum int64, err error, } func (s *SyntheticProofTask) markFinished(ctx context.Context, spid, sector int64) error { - n, err := s.db.Exec(ctx, `UPDATE sectors_sdr_pipeline SET after_synth = true, task_id_synth = NULL + // Synth proofs only run on the local pipeline; the provider pipeline does not + // have task_id_synth / after_synth columns (provider does SDR+trees only). + n, err := s.db.Exec(ctx, `UPDATE sectors_sdr_pipeline SET after_synth = true, task_id_synth = NULL WHERE sp_id = $1 AND sector_number = $2`, spid, sector) if err != nil { @@ -222,7 +232,8 @@ func (s *SyntheticProofTask) TypeDetails() harmonytask.TaskTypeDetails { func (s *SyntheticProofTask) taskToSector(id harmonytask.TaskID) (ffi.SectorRef, error) { var refs []ffi.SectorRef - err := s.db.Select(context.Background(), &refs, `SELECT sp_id, sector_number, reg_seal_proof FROM sectors_sdr_pipeline WHERE task_id_synth = $1`, id) + err := s.db.Select(context.Background(), &refs, ` + SELECT sp_id, sector_number, reg_seal_proof FROM sectors_sdr_pipeline WHERE task_id_synth = $1`, id) if err != nil { return ffi.SectorRef{}, xerrors.Errorf("getting sector ref: %w", err) } @@ -251,7 +262,7 @@ func (s *SyntheticProofTask) GetSpid(db *harmonydb.DB, taskID int64) string { func (s *SyntheticProofTask) GetSectorID(db *harmonydb.DB, taskID int64) (*abi.SectorID, error) { var spId, sectorNumber uint64 - err := db.QueryRow(context.Background(), `SELECT sp_id,sector_number FROM sectors_sdr_pipeline WHERE task_id_synth = $1`, taskID).Scan(&spId, §orNumber) + err := db.QueryRow(context.Background(), `SELECT sp_id, sector_number FROM sectors_sdr_pipeline WHERE task_id_synth = $1`, taskID).Scan(&spId, §orNumber) if err != nil { return nil, err } diff --git a/tasks/seal/task_treed.go b/tasks/seal/task_treed.go index 99702e2d2..ee59db244 100644 --- a/tasks/seal/task_treed.go +++ b/tasks/seal/task_treed.go @@ -20,12 +20,20 @@ import ( "github.com/filecoin-project/curio/tasks/tasknames" ) +// ProviderPollerTreeD is an interface that allows registering the TreeD task's +// AddTaskFunc with the remote seal provider poller. +type ProviderPollerTreeD interface { + SetPollerTreeD(harmonytask.AddTaskFunc) +} + type TreeDTask struct { sp *SealPoller db *harmonydb.DB sc *ffi2.SealCalls bound bool + provPoller ProviderPollerTreeD // optional, nil when remote seal provider is not enabled + max int } @@ -59,7 +67,13 @@ func (t *TreeDTask) CanAccept(ids []harmonytask.TaskID, engine *harmonytask.Task SELECT p.task_id_tree_d FROM sectors_sdr_pipeline p INNER JOIN sector_location l ON p.sp_id = l.miner_id AND p.sector_number = l.sector_num AND l.sector_filetype = 4 INNER JOIN storage_path sp ON sp.storage_id = l.storage_id - WHERE p.task_id_tree_d = ANY($1::bigint[]) AND sp.urls IS NOT NULL AND sp.urls LIKE '%' || $2 || '%' LIMIT 100 + WHERE p.task_id_tree_d = ANY($1::bigint[]) AND sp.urls IS NOT NULL AND sp.urls LIKE '%' || $2 || '%' + UNION ALL + SELECT p.task_id_tree_d FROM rseal_provider_pipeline p + INNER JOIN sector_location l ON p.sp_id = l.miner_id AND p.sector_number = l.sector_num AND l.sector_filetype = 4 + INNER JOIN storage_path sp ON sp.storage_id = l.storage_id + WHERE p.task_id_tree_d = ANY($1::bigint[]) AND sp.urls IS NOT NULL AND sp.urls LIKE '%' || $2 || '%' + LIMIT 100 ) s`, indIDs, engine.Host()).Scan(&acceptedIDs) if err != nil { return nil, xerrors.Errorf("getting tasks from DB: %w", err) @@ -100,7 +114,11 @@ func (t *TreeDTask) GetSpid(db *harmonydb.DB, taskID int64) string { func (t *TreeDTask) GetSectorID(db *harmonydb.DB, taskID int64) (*abi.SectorID, error) { var spId, sectorNumber uint64 - err := db.QueryRow(context.Background(), `SELECT sp_id,sector_number FROM sectors_sdr_pipeline WHERE task_id_tree_d = $1`, taskID).Scan(&spId, §orNumber) + err := db.QueryRow(context.Background(), `SELECT sp_id, sector_number FROM ( + SELECT sp_id, sector_number FROM sectors_sdr_pipeline WHERE task_id_tree_d = $1 + UNION ALL + SELECT sp_id, sector_number FROM rseal_provider_pipeline WHERE task_id_tree_d = $1 + ) s`, taskID).Scan(&spId, §orNumber) if err != nil { return nil, err } @@ -115,7 +133,10 @@ var _ = harmonytask.Reg(&TreeDTask{}) func (t *TreeDTask) taskToSector(id harmonytask.TaskID) (ffi2.SectorRef, error) { var refs []ffi2.SectorRef - err := t.db.Select(context.Background(), &refs, `SELECT sp_id, sector_number, reg_seal_proof FROM sectors_sdr_pipeline WHERE task_id_tree_d = $1`, id) + err := t.db.Select(context.Background(), &refs, ` + SELECT sp_id, sector_number, reg_seal_proof FROM sectors_sdr_pipeline WHERE task_id_tree_d = $1 + UNION ALL + SELECT sp_id, sector_number, reg_seal_proof FROM rseal_provider_pipeline WHERE task_id_tree_d = $1`, id) if err != nil { return ffi2.SectorRef{}, xerrors.Errorf("getting sector ref: %w", err) } @@ -129,14 +150,19 @@ func (t *TreeDTask) taskToSector(id harmonytask.TaskID) (ffi2.SectorRef, error) func (t *TreeDTask) Adder(taskFunc harmonytask.AddTaskFunc) { t.sp.pollers[pollerTreeD].Set(taskFunc) + if t.provPoller != nil { + t.provPoller.SetPollerTreeD(taskFunc) + } } -func NewTreeDTask(sp *SealPoller, db *harmonydb.DB, sc *ffi2.SealCalls, maxTrees int, bound bool) *TreeDTask { +func NewTreeDTask(sp *SealPoller, db *harmonydb.DB, sc *ffi2.SealCalls, maxTrees int, bound bool, provPoller ProviderPollerTreeD) *TreeDTask { return &TreeDTask{ sp: sp, db: db, sc: sc, + provPoller: provPoller, + max: maxTrees, bound: bound, } @@ -149,11 +175,16 @@ func (t *TreeDTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done SpID int64 `db:"sp_id"` SectorNumber int64 `db:"sector_number"` RegSealProof abi.RegisteredSealProof `db:"reg_seal_proof"` + Pipeline string `db:"pipeline"` } err = t.db.Select(ctx, §orParamsArr, ` - SELECT sp_id, sector_number, reg_seal_proof + SELECT sp_id, sector_number, reg_seal_proof, 'local' as pipeline FROM sectors_sdr_pipeline + WHERE task_id_tree_d = $1 + UNION ALL + SELECT sp_id, sector_number, reg_seal_proof, 'remote' as pipeline + FROM rseal_provider_pipeline WHERE task_id_tree_d = $1`, taskID) if err != nil { return false, xerrors.Errorf("getting sector params: %w", err) @@ -196,9 +227,16 @@ func (t *TreeDTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done return false, xerrors.Errorf("failed to generate TreeD: %w", err) } - n, err := t.db.Exec(ctx, `UPDATE sectors_sdr_pipeline - SET after_tree_d = true, tree_d_cid = $3, task_id_tree_d = NULL WHERE sp_id = $1 AND sector_number = $2`, - sectorParams.SpID, sectorParams.SectorNumber, dealData.CommD) + var n int + if sectorParams.Pipeline == "remote" { + n, err = t.db.Exec(ctx, `UPDATE rseal_provider_pipeline + SET after_tree_d = true, tree_d_cid = $3, task_id_tree_d = NULL WHERE sp_id = $1 AND sector_number = $2`, + sectorParams.SpID, sectorParams.SectorNumber, dealData.CommD) + } else { + n, err = t.db.Exec(ctx, `UPDATE sectors_sdr_pipeline + SET after_tree_d = true, tree_d_cid = $3, task_id_tree_d = NULL WHERE sp_id = $1 AND sector_number = $2`, + sectorParams.SpID, sectorParams.SectorNumber, dealData.CommD) + } if err != nil { return false, xerrors.Errorf("store TreeD success: updating pipeline: %w", err) } diff --git a/tasks/seal/task_treerc.go b/tasks/seal/task_treerc.go index 0f6f08441..fadfbc077 100644 --- a/tasks/seal/task_treerc.go +++ b/tasks/seal/task_treerc.go @@ -22,20 +22,30 @@ import ( "github.com/filecoin-project/curio/tasks/tasknames" ) +// ProviderPollerTreeRC is an interface that allows registering the TreeRC task's +// AddTaskFunc with the remote seal provider poller. +type ProviderPollerTreeRC interface { + SetPollerTreeRC(harmonytask.AddTaskFunc) +} + type TreeRCTask struct { sp *SealPoller db *harmonydb.DB sc *ffi2.SealCalls + provPoller ProviderPollerTreeRC // optional, nil when remote seal provider is not enabled + max int } -func NewTreeRCTask(sp *SealPoller, db *harmonydb.DB, sc *ffi2.SealCalls, maxTrees int) *TreeRCTask { +func NewTreeRCTask(sp *SealPoller, db *harmonydb.DB, sc *ffi2.SealCalls, maxTrees int, provPoller ProviderPollerTreeRC) *TreeRCTask { return &TreeRCTask{ sp: sp, db: db, sc: sc, + provPoller: provPoller, + max: maxTrees, } } @@ -49,11 +59,16 @@ func (t *TreeRCTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done RegSealProof abi.RegisteredSealProof `db:"reg_seal_proof"` CommD string `db:"tree_d_cid"` TicketValue []byte `db:"ticket_value"` + Pipeline string `db:"pipeline"` } err = t.db.Select(ctx, §orParamsArr, ` - SELECT sp_id, sector_number, reg_seal_proof, tree_d_cid, ticket_value + SELECT sp_id, sector_number, reg_seal_proof, tree_d_cid, ticket_value, 'local' as pipeline FROM sectors_sdr_pipeline + WHERE task_id_tree_c = $1 AND task_id_tree_r = $1 + UNION ALL + SELECT sp_id, sector_number, reg_seal_proof, tree_d_cid, ticket_value, 'remote' as pipeline + FROM rseal_provider_pipeline WHERE task_id_tree_c = $1 AND task_id_tree_r = $1`, taskID) if err != nil { return false, xerrors.Errorf("getting sector params: %w", err) @@ -85,7 +100,7 @@ func (t *TreeRCTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done // R / C sealed, unsealed, err := t.sc.TreeRC(ctx, &taskID, sref, commd, sectorParams.TicketValue, dd.PieceInfos) if err != nil { - serr := resetSectorSealingState(ctx, sectorParams.SpID, sectorParams.SectorNumber, err, t.db, t.TypeDetails().Name) + serr := resetSectorSealingState(ctx, sectorParams.SpID, sectorParams.SectorNumber, err, t.db, t.TypeDetails().Name, sectorParams.Pipeline) if serr != nil { return false, xerrors.Errorf("computing tree r and c: %w", err) } @@ -95,10 +110,18 @@ func (t *TreeRCTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done return false, xerrors.Errorf("commd %s does match unsealed %s", commd.String(), unsealed.String()) } - n, err := t.db.Exec(ctx, `UPDATE sectors_sdr_pipeline - SET after_tree_r = true, after_tree_c = true, tree_r_cid = $3, task_id_tree_r = NULL, task_id_tree_c = NULL - WHERE sp_id = $1 AND sector_number = $2`, - sectorParams.SpID, sectorParams.SectorNumber, sealed) + var n int + if sectorParams.Pipeline == "remote" { + n, err = t.db.Exec(ctx, `UPDATE rseal_provider_pipeline + SET after_tree_r = true, after_tree_c = true, tree_r_cid = $3, task_id_tree_r = NULL, task_id_tree_c = NULL + WHERE sp_id = $1 AND sector_number = $2`, + sectorParams.SpID, sectorParams.SectorNumber, sealed) + } else { + n, err = t.db.Exec(ctx, `UPDATE sectors_sdr_pipeline + SET after_tree_r = true, after_tree_c = true, tree_r_cid = $3, task_id_tree_r = NULL, task_id_tree_c = NULL + WHERE sp_id = $1 AND sector_number = $2`, + sectorParams.SpID, sectorParams.SectorNumber, sealed) + } if err != nil { return false, xerrors.Errorf("store sdr-trees success: updating pipeline: %w", err) } @@ -138,7 +161,14 @@ func (t *TreeRCTask) CanAccept(ids []harmonytask.TaskID, engine *harmonytask.Tas INNER JOIN storage_path sp ON sp.storage_id = l.storage_id WHERE task_id_tree_c = ANY ($1) AND sp.urls IS NOT NULL - AND sp.urls LIKE '%' || $2 || '%' + AND sp.urls LIKE '%' || $2 || '%' + UNION ALL + SELECT p.task_id_tree_c FROM rseal_provider_pipeline p + INNER JOIN sector_location l ON p.sp_id = l.miner_id AND p.sector_number = l.sector_num AND l.sector_filetype = 4 + INNER JOIN storage_path sp ON sp.storage_id = l.storage_id + WHERE task_id_tree_c = ANY ($1) + AND sp.urls IS NOT NULL + AND sp.urls LIKE '%' || $2 || '%' LIMIT 100 ) s`, indIDs, engine.Host()).Scan(&acceptedIDs) if err != nil { @@ -185,7 +215,11 @@ func (t *TreeRCTask) GetSpid(db *harmonydb.DB, taskID int64) string { func (t *TreeRCTask) GetSectorID(db *harmonydb.DB, taskID int64) (*abi.SectorID, error) { var spId, sectorNumber uint64 - err := db.QueryRow(context.Background(), `SELECT sp_id,sector_number FROM sectors_sdr_pipeline WHERE task_id_tree_r = $1`, taskID).Scan(&spId, §orNumber) + err := db.QueryRow(context.Background(), `SELECT sp_id, sector_number FROM ( + SELECT sp_id, sector_number FROM sectors_sdr_pipeline WHERE task_id_tree_r = $1 + UNION ALL + SELECT sp_id, sector_number FROM rseal_provider_pipeline WHERE task_id_tree_r = $1 + ) s`, taskID).Scan(&spId, §orNumber) if err != nil { return nil, err } @@ -199,12 +233,18 @@ var _ = harmonytask.Reg(&TreeRCTask{}) func (t *TreeRCTask) Adder(taskFunc harmonytask.AddTaskFunc) { t.sp.pollers[pollerTreeRC].Set(taskFunc) + if t.provPoller != nil { + t.provPoller.SetPollerTreeRC(taskFunc) + } } func (t *TreeRCTask) taskToSector(id harmonytask.TaskID) (ffi2.SectorRef, error) { var refs []ffi2.SectorRef - err := t.db.Select(context.Background(), &refs, `SELECT sp_id, sector_number, reg_seal_proof FROM sectors_sdr_pipeline WHERE task_id_tree_r = $1`, id) + err := t.db.Select(context.Background(), &refs, ` + SELECT sp_id, sector_number, reg_seal_proof FROM sectors_sdr_pipeline WHERE task_id_tree_r = $1 + UNION ALL + SELECT sp_id, sector_number, reg_seal_proof FROM rseal_provider_pipeline WHERE task_id_tree_r = $1`, id) if err != nil { return ffi2.SectorRef{}, xerrors.Errorf("getting sector ref: %w", err) } diff --git a/tasks/sealsupra/task_supraseal.go b/tasks/sealsupra/task_supraseal.go index 5eb9c2afe..771abb4c0 100644 --- a/tasks/sealsupra/task_supraseal.go +++ b/tasks/sealsupra/task_supraseal.go @@ -265,13 +265,20 @@ func (s *SupraSeal) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done ctx := context.Background() var sectors []struct { - SpID int64 `db:"sp_id"` - SectorNumber int64 `db:"sector_number"` - - RegSealProof int64 `db:"reg_seal_proof"` - } - - err = s.db.Select(ctx, §ors, `SELECT sp_id, sector_number, reg_seal_proof FROM sectors_sdr_pipeline WHERE task_id_sdr = $1 AND task_id_tree_r = $1 AND task_id_tree_c = $1 AND task_id_tree_d = $1`, taskID) + SpID int64 `db:"sp_id"` + SectorNumber int64 `db:"sector_number"` + RegSealProof int64 `db:"reg_seal_proof"` + Pipeline string `db:"pipeline"` + TicketEpoch sql.NullInt64 `db:"ticket_epoch"` + TicketValue []byte `db:"ticket_value"` + } + + err = s.db.Select(ctx, §ors, ` + SELECT sp_id, sector_number, reg_seal_proof, 'local' as pipeline, NULL::bigint as ticket_epoch, NULL::bytea as ticket_value FROM sectors_sdr_pipeline + WHERE task_id_sdr = $1 AND task_id_tree_r = $1 AND task_id_tree_c = $1 AND task_id_tree_d = $1 + UNION ALL + SELECT sp_id, sector_number, reg_seal_proof, 'remote' as pipeline, ticket_epoch, ticket_value FROM rseal_provider_pipeline + WHERE task_id_sdr = $1 AND task_id_tree_r = $1 AND task_id_tree_c = $1 AND task_id_tree_d = $1`, taskID) if err != nil { return false, xerrors.Errorf("getting sector params: %w", err) } @@ -319,13 +326,19 @@ func (s *SupraSeal) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done return false, xerrors.Errorf("removing sector: %w", err) } - // get ticket maddr, err := address.NewIDAddress(uint64(t.SpID)) if err != nil { return false, xerrors.Errorf("getting miner address: %w", err) } - ticket, ticketEpoch, err := seal.GetTicket(ctx, s.api, maddr) + var ticket abi.SealRandomness + var ticketEpoch abi.ChainEpoch + if t.Pipeline == "remote" { + // Remote sector: fetch ticket from the client's chain (may be a different network) + ticket, ticketEpoch, err = seal.GetRemoteTicket(ctx, s.db, t.SpID, t.SectorNumber, maddr) + } else { + ticket, ticketEpoch, err = seal.GetTicket(ctx, s.api, maddr) + } if err != nil { return false, xerrors.Errorf("getting ticket: %w", err) } @@ -333,7 +346,7 @@ func (s *SupraSeal) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done tickets[i] = ticket spt := abi.RegisteredSealProof(t.RegSealProof) - replicaIDs[i], err = spt.ReplicaId(abi.ActorID(t.SpID), abi.SectorNumber(t.SectorNumber), ticket, commd) + replicaIDs[i], err = spt.ReplicaId(abi.ActorID(t.SpID), abi.SectorNumber(t.SectorNumber), tickets[i], commd) if err != nil { return false, xerrors.Errorf("getting replica id: %w", err) } @@ -485,16 +498,25 @@ func (s *SupraSeal) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done return false, xerrors.Errorf("getting sealed CID: %w", err) } - _, err = tx.Exec(`UPDATE sectors_sdr_pipeline SET after_sdr = TRUE, after_tree_c = TRUE, after_tree_r = TRUE, after_tree_d = TRUE, after_synth = TRUE, + pipelineSource := "local" + if sector.Pipeline == "remote" { + pipelineSource = "remote" + _, err = tx.Exec(`UPDATE rseal_provider_pipeline SET after_sdr = TRUE, after_tree_c = TRUE, after_tree_r = TRUE, after_tree_d = TRUE, + ticket_epoch = $3, ticket_value = $4, tree_d_cid = $5, tree_r_cid = $6, + task_id_sdr = NULL, task_id_tree_r = NULL, task_id_tree_c = NULL, task_id_tree_d = NULL + WHERE sp_id = $1 AND sector_number = $2`, sector.SpID, sector.SectorNumber, ticketEpochs[i], tickets[i], unsealedCID.String(), sealedCID) + } else { + _, err = tx.Exec(`UPDATE sectors_sdr_pipeline SET after_sdr = TRUE, after_tree_c = TRUE, after_tree_r = TRUE, after_tree_d = TRUE, after_synth = TRUE, ticket_epoch = $3, ticket_value = $4, tree_d_cid = $5, tree_r_cid = $6, task_id_sdr = NULL, task_id_tree_r = NULL, task_id_tree_c = NULL, task_id_tree_d = NULL WHERE sp_id = $1 AND sector_number = $2`, sector.SpID, sector.SectorNumber, ticketEpochs[i], tickets[i], unsealedCID.String(), sealedCID) + } if err != nil { return false, xerrors.Errorf("updating sector: %w", err) } // insert batch refs - _, err = tx.Exec(`INSERT INTO batch_sector_refs (sp_id, sector_number, machine_host_and_port, pipeline_slot) - VALUES ($1, $2, $3, $4) ON CONFLICT DO NOTHING`, sector.SpID, sector.SectorNumber, ownedBy[0].HostAndPort, slot) + _, err = tx.Exec(`INSERT INTO batch_sector_refs (sp_id, sector_number, machine_host_and_port, pipeline_slot, pipeline_source) + VALUES ($1, $2, $3, $4, $5) ON CONFLICT DO NOTHING`, sector.SpID, sector.SectorNumber, ownedBy[0].HostAndPort, slot, pipelineSource) if err != nil { return false, xerrors.Errorf("inserting batch refs: %w", err) } @@ -568,6 +590,7 @@ type sectorClaim struct { SpID int64 `db:"sp_id"` SectorNumber int64 `db:"sector_number"` TaskIDSDR sql.NullInt64 `db:"task_id_sdr"` + Pipeline string `db:"pipeline"` } func (s *SupraSeal) schedule(taskFunc harmonytask.AddTaskFunc) error { @@ -584,9 +607,17 @@ func (s *SupraSeal) schedule(taskFunc harmonytask.AddTaskFunc) error { // claim [sectors] pipeline entries var sectors []sectorClaim - err := tx.Select(§ors, `SELECT sp_id, sector_number, task_id_sdr FROM sectors_sdr_pipeline - LEFT JOIN harmony_task ht on sectors_sdr_pipeline.task_id_sdr = ht.id - WHERE after_sdr = FALSE AND (task_id_sdr IS NULL OR (ht.owner_id IS NULL AND ht.name = 'SDR')) LIMIT $1`, s.sectors) + err := tx.Select(§ors, ` + (SELECT sp_id, sector_number, task_id_sdr, 'local' as pipeline FROM sectors_sdr_pipeline + LEFT JOIN harmony_task ht on sectors_sdr_pipeline.task_id_sdr = ht.id + WHERE after_sdr = FALSE AND (task_id_sdr IS NULL OR (ht.owner_id IS NULL AND ht.name = 'SDR')) + AND NOT EXISTS (SELECT 1 FROM rseal_client_pipeline c WHERE c.sp_id = sectors_sdr_pipeline.sp_id AND c.sector_number = sectors_sdr_pipeline.sector_number) + LIMIT $1) + UNION ALL + (SELECT sp_id, sector_number, task_id_sdr, 'remote' as pipeline FROM rseal_provider_pipeline + LEFT JOIN harmony_task ht on rseal_provider_pipeline.task_id_sdr = ht.id + WHERE after_sdr = FALSE AND (task_id_sdr IS NULL OR (ht.owner_id IS NULL AND ht.name = 'SDR')) LIMIT $1) + LIMIT $1`, s.sectors) if err != nil { return false, xerrors.Errorf("getting tasks: %w", err) } @@ -610,7 +641,12 @@ func (s *SupraSeal) schedule(taskFunc harmonytask.AddTaskFunc) error { // assign to pipeline entries, set task_id_sdr, task_id_tree_r, task_id_tree_c for _, t := range sectors { - _, err := tx.Exec(`UPDATE sectors_sdr_pipeline SET task_id_sdr = $1, task_id_tree_r = $1, task_id_tree_c = $1, task_id_tree_d = $1 WHERE sp_id = $2 AND sector_number = $3`, id, t.SpID, t.SectorNumber) + var err error + if t.Pipeline == "remote" { + _, err = tx.Exec(`UPDATE rseal_provider_pipeline SET task_id_sdr = $1, task_id_tree_r = $1, task_id_tree_c = $1, task_id_tree_d = $1 WHERE sp_id = $2 AND sector_number = $3`, id, t.SpID, t.SectorNumber) + } else { + _, err = tx.Exec(`UPDATE sectors_sdr_pipeline SET task_id_sdr = $1, task_id_tree_r = $1, task_id_tree_c = $1, task_id_tree_d = $1 WHERE sp_id = $2 AND sector_number = $3`, id, t.SpID, t.SectorNumber) + } if err != nil { return false, xerrors.Errorf("updating task id: %w", err) } @@ -698,6 +734,7 @@ func (s *SupraSeal) claimsFromCCScheduler(tx *harmonydb.Tx, toSeal int64) ([]sec SpID: schedule.SpID, SectorNumber: int64(sectorNum), TaskIDSDR: sql.NullInt64{}, // New sector, no existing task + Pipeline: "local", }) userDuration := int64(schedule.DurationDays) * builtin.EpochsInDay @@ -738,7 +775,12 @@ func (s *SupraSeal) claimsFromCCScheduler(tx *harmonydb.Tx, toSeal int64) ([]sec func (s *SupraSeal) taskToSectors(id harmonytask.TaskID) ([]ffi.SectorRef, error) { var sectors []ffi.SectorRef - err := s.db.Select(context.Background(), §ors, `SELECT sp_id, sector_number, reg_seal_proof FROM sectors_sdr_pipeline WHERE task_id_sdr = $1 AND task_id_tree_r = $1 AND task_id_tree_c = $1 AND task_id_tree_d = $1`, id) + err := s.db.Select(context.Background(), §ors, ` + SELECT sp_id, sector_number, reg_seal_proof FROM sectors_sdr_pipeline + WHERE task_id_sdr = $1 AND task_id_tree_r = $1 AND task_id_tree_c = $1 AND task_id_tree_d = $1 + UNION ALL + SELECT sp_id, sector_number, reg_seal_proof FROM rseal_provider_pipeline + WHERE task_id_sdr = $1 AND task_id_tree_r = $1 AND task_id_tree_c = $1 AND task_id_tree_d = $1`, id) if err != nil { return nil, xerrors.Errorf("getting sector params: %w", err) } diff --git a/tasks/storage-market/task_fix_rawSize.go b/tasks/storage-market/task_fix_rawSize.go index 6c905c5ec..a21fd24e3 100644 --- a/tasks/storage-market/task_fix_rawSize.go +++ b/tasks/storage-market/task_fix_rawSize.go @@ -129,9 +129,9 @@ func (f *FixRawSize) CanAccept(ids []harmonytask.TaskID, engine *harmonytask.Tas var acceptedIDs []harmonytask.TaskID err := f.db.QueryRow(ctx, `SELECT COALESCE(array_agg(task_id), '{}')::bigint[] AS task_ids FROM ( - SELECT f.task_id FROM market_fix_raw_size f - INNER JOIN market_piece_deal mpd ON f.id = mpd.id - INNER JOIN sector_location l ON mpd.sp_id = l.miner_id AND mpd.sector_num = l.sector_num AND l.sector_filetype = 1 + SELECT f.task_id FROM market_fix_raw_size f + INNER JOIN market_piece_deal mpd ON f.id = mpd.id + INNER JOIN sector_location l ON mpd.sp_id = l.miner_id AND mpd.sector_num = l.sector_num AND l.sector_filetype = 1 INNER JOIN storage_path sp ON sp.storage_id = l.storage_id WHERE f.task_id = ANY($1::bigint[]) AND sp.urls IS NOT NULL AND sp.urls LIKE '%' || $2 || '%' LIMIT 100) s`, indIDs, engine.Host()).Scan(&acceptedIDs) if err != nil { diff --git a/web/api/webrpc/remoteseal.go b/web/api/webrpc/remoteseal.go new file mode 100644 index 000000000..db2396517 --- /dev/null +++ b/web/api/webrpc/remoteseal.go @@ -0,0 +1,602 @@ +package webrpc + +import ( + "bytes" + "context" + "crypto/rand" + "database/sql" + "encoding/base64" + "encoding/hex" + "encoding/json" + "fmt" + "net/http" + "net/url" + "time" + + "golang.org/x/xerrors" + + "github.com/filecoin-project/curio/market/sealmarket" +) + +// RSealPartner maps to rseal_delegated_partners (provider side). +type RSealPartner struct { + ID int64 `db:"id" json:"id"` + PartnerName string `db:"partner_name" json:"partner_name"` + PartnerURL string `db:"partner_url" json:"partner_url"` + PartnerToken string `db:"partner_token" json:"partner_token"` + AllowanceRemaining int64 `db:"allowance_remaining" json:"allowance_remaining"` + AllowanceTotal int64 `db:"allowance_total" json:"allowance_total"` + AllowedProofTypes []int64 `db:"allowed_proof_types" json:"allowed_proof_types"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` +} + +// RSealProvider maps to rseal_client_providers (client side). +type RSealProvider struct { + ID int64 `db:"id" json:"id"` + SpID int64 `db:"sp_id" json:"sp_id"` + ProviderURL string `db:"provider_url" json:"provider_url"` + ProviderToken string `db:"provider_token" json:"provider_token"` + ProviderName string `db:"provider_name" json:"provider_name"` + Enabled bool `db:"enabled" json:"enabled"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` +} + +// RSealProvPipelineRow is a summary view of a provider-side pipeline row. +type RSealProvPipelineRow struct { + SpID int64 `db:"sp_id" json:"sp_id"` + SectorNumber int64 `db:"sector_number" json:"sector_number"` + PartnerName string `db:"partner_name" json:"partner_name"` + + TaskIDSDR *int64 `db:"task_id_sdr" json:"task_id_sdr"` + AfterSDR bool `db:"after_sdr" json:"after_sdr"` + TaskIDTreeD *int64 `db:"task_id_tree_d" json:"task_id_tree_d"` + AfterTreeD bool `db:"after_tree_d" json:"after_tree_d"` + TaskIDTreeC *int64 `db:"task_id_tree_c" json:"task_id_tree_c"` + AfterTreeC bool `db:"after_tree_c" json:"after_tree_c"` + TaskIDTreeR *int64 `db:"task_id_tree_r" json:"task_id_tree_r"` + AfterTreeR bool `db:"after_tree_r" json:"after_tree_r"` + + TaskIDNotify *int64 `db:"task_id_notify_client" json:"task_id_notify_client"` + AfterNotify bool `db:"after_notify_client" json:"after_notify_client"` + TaskIDFinalize *int64 `db:"task_id_finalize" json:"task_id_finalize"` + AfterC1 bool `db:"after_c1_supplied" json:"after_c1_supplied"` + AfterFinalize bool `db:"after_finalize" json:"after_finalize"` + TaskIDCleanup *int64 `db:"task_id_cleanup" json:"task_id_cleanup"` + AfterCleanup bool `db:"after_cleanup" json:"after_cleanup"` + Failed bool `db:"failed" json:"failed"` + FailedReasonMsg string `db:"failed_reason_msg" json:"failed_reason_msg"` + + CreateTime time.Time `db:"create_time" json:"create_time"` +} + +// RSealClientPipelineRow is a summary view of a client-side pipeline row. +type RSealClientPipelineRow struct { + SpID int64 `db:"sp_id" json:"sp_id"` + SectorNumber int64 `db:"sector_number" json:"sector_number"` + ProviderName string `db:"provider_name" json:"provider_name"` + + TaskIDDelegate *int64 `db:"task_id_delegate" json:"task_id_delegate"` + AfterDelegate bool `db:"after_delegate" json:"after_delegate"` + TaskIDPoll *int64 `db:"task_id_poll" json:"task_id_poll"` + AfterSDR bool `db:"after_sdr" json:"after_sdr"` + + TaskIDFetch *int64 `db:"task_id_fetch" json:"task_id_fetch"` + AfterFetch bool `db:"after_fetch" json:"after_fetch"` + TaskIDCleanup *int64 `db:"task_id_cleanup" json:"task_id_cleanup"` + AfterCleanup bool `db:"after_cleanup" json:"after_cleanup"` + + Failed bool `db:"failed" json:"failed"` + FailedReasonMsg string `db:"failed_reason_msg" json:"failed_reason_msg"` + + CreateTime time.Time `db:"create_time" json:"create_time"` +} + +// RSealListPartners returns all remote seal partners (provider side). +func (a *WebRPC) RSealListPartners(ctx context.Context) ([]RSealPartner, error) { + var partners []RSealPartner + err := a.deps.DB.Select(ctx, &partners, `SELECT id, partner_name, partner_url, partner_token, allowance_remaining, allowance_total, allowed_proof_types, created_at, updated_at FROM rseal_delegated_partners ORDER BY id`) + if err != nil { + return nil, xerrors.Errorf("listing partners: %w", err) + } + if partners == nil { + partners = []RSealPartner{} + } + return partners, nil +} + +// RSealGetPartnerURL returns the current node's base URL for remote seal +// (derived from HTTP.DomainName config). Clients paste this when configuring +// their provider connection, and it is shown in the UI for reference. +func (a *WebRPC) RSealGetPartnerURL(ctx context.Context) (string, error) { + dn := a.deps.Cfg.HTTP.DomainName + if dn == "" { + return "", nil + } + return fmt.Sprintf("https://%s", dn), nil +} + +// RSealAddPartner creates a new remote seal partner with a generated token. +func (a *WebRPC) RSealAddPartner(ctx context.Context, name string, url string, allowance int64, allowedProofTypes []int64) (*RSealPartner, error) { + // Prevent adding ourselves as a partner + if a.deps.Cfg.HTTP.DomainName != "" { + selfURL := fmt.Sprintf("https://%s", a.deps.Cfg.HTTP.DomainName) + if url == selfURL || url == selfURL+"/" { + return nil, fmt.Errorf("cannot add yourself as a partner") + } + } + + if allowedProofTypes == nil { + allowedProofTypes = []int64{} + } + + tokenBytes := make([]byte, 32) + if _, err := rand.Read(tokenBytes); err != nil { + return nil, xerrors.Errorf("generating token: %w", err) + } + token := hex.EncodeToString(tokenBytes) + + var partner RSealPartner + err := a.deps.DB.QueryRow(ctx, `INSERT INTO rseal_delegated_partners (partner_name, partner_url, partner_token, allowance_remaining, allowance_total, allowed_proof_types) + VALUES ($1, $2, $3, $4, $4, $5) RETURNING id, partner_name, partner_url, partner_token, allowance_remaining, allowance_total, allowed_proof_types, created_at, updated_at`, + name, url, token, allowance, allowedProofTypes).Scan( + &partner.ID, &partner.PartnerName, &partner.PartnerURL, &partner.PartnerToken, + &partner.AllowanceRemaining, &partner.AllowanceTotal, &partner.AllowedProofTypes, &partner.CreatedAt, &partner.UpdatedAt) + if err != nil { + return nil, xerrors.Errorf("inserting partner: %w", err) + } + return &partner, nil +} + +// RSealRemovePartner deletes a remote seal partner. Rejects if active pipeline rows exist. +func (a *WebRPC) RSealRemovePartner(ctx context.Context, id int64) error { + var count int64 + err := a.deps.DB.QueryRow(ctx, `SELECT COUNT(*) FROM rseal_provider_pipeline WHERE partner_id = $1 AND after_cleanup = FALSE`, id).Scan(&count) + if err != nil { + return xerrors.Errorf("checking active pipelines: %w", err) + } + if count > 0 { + return fmt.Errorf("cannot remove partner: %d active pipeline rows exist", count) + } + + _, err = a.deps.DB.Exec(ctx, `DELETE FROM rseal_delegated_partners WHERE id = $1`, id) + if err != nil { + return xerrors.Errorf("deleting partner: %w", err) + } + return nil +} + +// RSealUpdatePartnerAllowance updates the allowance fields for a partner. +func (a *WebRPC) RSealUpdatePartnerAllowance(ctx context.Context, id int64, totalAllowance int64, remainingAllowance int64) error { + _, err := a.deps.DB.Exec(ctx, `UPDATE rseal_delegated_partners SET allowance_total = $2, allowance_remaining = $3, updated_at = CURRENT_TIMESTAMP WHERE id = $1`, + id, totalAllowance, remainingAllowance) + if err != nil { + return xerrors.Errorf("updating allowance: %w", err) + } + return nil +} + +// connectStringPayload is the JSON structure encoded in the connect string. +type connectStringPayload struct { + URL string `json:"url"` + Token string `json:"token"` +} + +// RSealGetConnectString returns a base64-encoded connect string for a partner. +func (a *WebRPC) RSealGetConnectString(ctx context.Context, id int64) (string, error) { + var token string + err := a.deps.DB.QueryRow(ctx, `SELECT partner_token FROM rseal_delegated_partners WHERE id = $1`, id).Scan(&token) + if err != nil { + return "", xerrors.Errorf("querying partner token: %w", err) + } + + // Build the base URL from HTTP config + baseURL := fmt.Sprintf("https://%s", a.deps.Cfg.HTTP.DomainName) + + payload := connectStringPayload{ + URL: baseURL, + Token: token, + } + + jsonBytes, err := json.Marshal(payload) + if err != nil { + return "", xerrors.Errorf("marshaling connect string: %w", err) + } + + return base64.StdEncoding.EncodeToString(jsonBytes), nil +} + +// RSealListProviders returns all remote seal providers (client side). +func (a *WebRPC) RSealListProviders(ctx context.Context) ([]RSealProvider, error) { + var providers []RSealProvider + err := a.deps.DB.Select(ctx, &providers, `SELECT id, sp_id, provider_url, provider_token, provider_name, enabled, created_at, updated_at FROM rseal_client_providers ORDER BY id`) + if err != nil { + return nil, xerrors.Errorf("listing providers: %w", err) + } + if providers == nil { + providers = []RSealProvider{} + } + return providers, nil +} + +// RSealAddProvider adds a remote seal provider from a connect string. +func (a *WebRPC) RSealAddProvider(ctx context.Context, spID int64, connectString string) (*RSealProvider, error) { + // Decode connect string + jsonBytes, err := base64.StdEncoding.DecodeString(connectString) + if err != nil { + return nil, xerrors.Errorf("decoding connect string: %w", err) + } + + var payload connectStringPayload + if err := json.Unmarshal(jsonBytes, &payload); err != nil { + return nil, xerrors.Errorf("parsing connect string: %w", err) + } + + if payload.URL == "" || payload.Token == "" { + return nil, fmt.Errorf("connect string missing url or token") + } + + // Prevent adding ourselves as a provider + if a.deps.Cfg.HTTP.DomainName != "" { + selfURL := fmt.Sprintf("https://%s", a.deps.Cfg.HTTP.DomainName) + if payload.URL == selfURL || payload.URL == selfURL+"/" { + return nil, fmt.Errorf("cannot add yourself as a provider") + } + } + + var provider RSealProvider + err = a.deps.DB.QueryRow(ctx, `INSERT INTO rseal_client_providers (sp_id, provider_url, provider_token, provider_name) + VALUES ($1, $2, $3, $4) RETURNING id, sp_id, provider_url, provider_token, provider_name, enabled, created_at, updated_at`, + spID, payload.URL, payload.Token, "").Scan( + &provider.ID, &provider.SpID, &provider.ProviderURL, &provider.ProviderToken, + &provider.ProviderName, &provider.Enabled, &provider.CreatedAt, &provider.UpdatedAt) + if err != nil { + return nil, xerrors.Errorf("inserting provider: %w", err) + } + return &provider, nil +} + +// RSealRemoveProvider deletes a remote seal provider. Rejects if active pipeline rows exist. +func (a *WebRPC) RSealRemoveProvider(ctx context.Context, id int64) error { + var count int64 + err := a.deps.DB.QueryRow(ctx, `SELECT COUNT(*) FROM rseal_client_pipeline WHERE provider_id = $1 AND after_cleanup = FALSE`, id).Scan(&count) + if err != nil { + // Table might not exist if no sectors have been delegated yet + if err != sql.ErrNoRows { + return xerrors.Errorf("checking active pipelines: %w", err) + } + } + if count > 0 { + return fmt.Errorf("cannot remove provider: %d active pipeline rows exist", count) + } + + _, err = a.deps.DB.Exec(ctx, `DELETE FROM rseal_client_providers WHERE id = $1`, id) + if err != nil { + return xerrors.Errorf("deleting provider: %w", err) + } + return nil +} + +// RSealToggleProvider enables or disables a remote seal provider. +func (a *WebRPC) RSealToggleProvider(ctx context.Context, id int64, enabled bool) error { + _, err := a.deps.DB.Exec(ctx, `UPDATE rseal_client_providers SET enabled = $2, updated_at = CURRENT_TIMESTAMP WHERE id = $1`, id, enabled) + if err != nil { + return xerrors.Errorf("toggling provider: %w", err) + } + return nil +} + +// RSealUpdateProviderName updates the display name for a client-side provider. +func (a *WebRPC) RSealUpdateProviderName(ctx context.Context, id int64, name string) error { + _, err := a.deps.DB.Exec(ctx, `UPDATE rseal_client_providers SET provider_name = $2, updated_at = CURRENT_TIMESTAMP WHERE id = $1`, id, name) + if err != nil { + return xerrors.Errorf("updating provider name: %w", err) + } + return nil +} + +// RSealProviderAvailability is the result of probing a provider's /available endpoint. +type RSealProviderAvailability struct { + ID int64 `json:"id"` + Available bool `json:"available"` + Error string `json:"error,omitempty"` // non-empty if the probe failed +} + +// RSealCheckProviderAvailability probes each configured provider's /available +// endpoint and returns whether it currently has slots. This is called from the +// client UI so operators can see at a glance which providers are ready. +func (a *WebRPC) RSealCheckProviderAvailability(ctx context.Context) ([]RSealProviderAvailability, error) { + var providers []struct { + ID int64 `db:"id"` + URL string `db:"provider_url"` + Token string `db:"provider_token"` + } + err := a.deps.DB.Select(ctx, &providers, `SELECT id, provider_url, provider_token FROM rseal_client_providers WHERE enabled = TRUE ORDER BY id`) + if err != nil { + return nil, xerrors.Errorf("listing providers for availability check: %w", err) + } + + httpClient := &http.Client{Timeout: 10 * time.Second} + + results := make([]RSealProviderAvailability, len(providers)) + for i, p := range providers { + results[i].ID = p.ID + + reqBody := sealmarket.AuthorizeRequest{PartnerToken: p.Token} + bodyBytes, err := json.Marshal(reqBody) + if err != nil { + results[i].Error = err.Error() + continue + } + + availURL, _ := url.JoinPath(p.URL, sealmarket.DelegatedSealPath, "available") + req, err := http.NewRequestWithContext(ctx, http.MethodPost, availURL, bytes.NewReader(bodyBytes)) + if err != nil { + results[i].Error = err.Error() + continue + } + req.Header.Set("Content-Type", "application/json") + + resp, err := httpClient.Do(req) + if err != nil { + results[i].Error = err.Error() + continue + } + + var availResp sealmarket.AvailableResponse + decErr := json.NewDecoder(resp.Body).Decode(&availResp) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK { + results[i].Error = fmt.Sprintf("HTTP %d", resp.StatusCode) + continue + } + if decErr != nil { + results[i].Error = decErr.Error() + continue + } + + results[i].Available = availResp.Available + } + + return results, nil +} + +// RSealProviderPipeline returns active provider-side pipeline rows. +func (a *WebRPC) RSealProviderPipeline(ctx context.Context) ([]RSealProvPipelineRow, error) { + var rows []RSealProvPipelineRow + err := a.deps.DB.Select(ctx, &rows, `SELECT p.sp_id, p.sector_number, d.partner_name, + p.task_id_sdr, p.after_sdr, p.task_id_tree_d, p.after_tree_d, + p.task_id_tree_c, p.after_tree_c, p.task_id_tree_r, p.after_tree_r, + p.task_id_notify_client, p.after_notify_client, p.after_c1_supplied, + p.task_id_finalize, p.after_finalize, p.task_id_cleanup, p.after_cleanup, + p.failed, p.failed_reason_msg, p.create_time + FROM rseal_provider_pipeline p + JOIN rseal_delegated_partners d ON p.partner_id = d.id + WHERE p.after_cleanup = FALSE + ORDER BY p.create_time DESC + LIMIT 100`) + if err != nil { + return nil, xerrors.Errorf("querying provider pipeline: %w", err) + } + if rows == nil { + rows = []RSealProvPipelineRow{} + } + return rows, nil +} + +// RSealClientPipeline returns active client-side pipeline rows. +func (a *WebRPC) RSealClientPipeline(ctx context.Context) ([]RSealClientPipelineRow, error) { + var rows []RSealClientPipelineRow + err := a.deps.DB.Select(ctx, &rows, `SELECT c.sp_id, c.sector_number, COALESCE(p.provider_name, p.provider_url) AS provider_name, + c.task_id_delegate, c.after_delegate, c.task_id_poll, c.after_sdr, + c.task_id_fetch, c.after_fetch, + c.task_id_cleanup, c.after_cleanup, + c.failed, c.failed_reason_msg, c.create_time + FROM rseal_client_pipeline c + JOIN rseal_client_providers p ON c.provider_id = p.id + WHERE c.after_cleanup = FALSE + ORDER BY c.create_time DESC + LIMIT 100`) + if err != nil { + return nil, xerrors.Errorf("querying client pipeline: %w", err) + } + if rows == nil { + rows = []RSealClientPipelineRow{} + } + return rows, nil +} + +// RSealProviderStats returns aggregate statistics for the remote seal provider pipeline. +func (a *WebRPC) RSealProviderStats(ctx context.Context) (*PipelineStats, error) { + var out PipelineStats + + const query = ` +WITH pipeline_data AS ( + SELECT p.*, + sdr.owner_id AS sdr_owner, + td.owner_id AS tree_d_owner, + tc.owner_id AS tree_c_owner, + tr.owner_id AS tree_r_owner, + notify.owner_id AS notify_owner, + fin.owner_id AS finalize_owner, + clean.owner_id AS cleanup_owner + FROM rseal_provider_pipeline p + LEFT JOIN harmony_task sdr ON sdr.id = p.task_id_sdr + LEFT JOIN harmony_task td ON td.id = p.task_id_tree_d + LEFT JOIN harmony_task tc ON tc.id = p.task_id_tree_c + LEFT JOIN harmony_task tr ON tr.id = p.task_id_tree_r + LEFT JOIN harmony_task notify ON notify.id = p.task_id_notify_client + LEFT JOIN harmony_task fin ON fin.id = p.task_id_finalize + LEFT JOIN harmony_task clean ON clean.id = p.task_id_cleanup + WHERE p.after_cleanup = FALSE AND p.failed = FALSE +) +SELECT + COUNT(*) AS total, + + -- SDR stage + COUNT(*) FILTER (WHERE after_sdr = false AND task_id_sdr IS NOT NULL AND sdr_owner IS NULL) AS sdr_pending, + COUNT(*) FILTER (WHERE after_sdr = false AND task_id_sdr IS NOT NULL AND sdr_owner IS NOT NULL) AS sdr_running, + + -- TreeD stage + COUNT(*) FILTER (WHERE after_sdr = true AND after_tree_d = false AND task_id_tree_d IS NOT NULL AND tree_d_owner IS NULL) AS treed_pending, + COUNT(*) FILTER (WHERE after_sdr = true AND after_tree_d = false AND task_id_tree_d IS NOT NULL AND tree_d_owner IS NOT NULL) AS treed_running, + + -- TreeC stage + COUNT(*) FILTER (WHERE after_tree_d = true AND after_tree_c = false AND task_id_tree_c IS NOT NULL AND tree_c_owner IS NULL) AS treec_pending, + COUNT(*) FILTER (WHERE after_tree_d = true AND after_tree_c = false AND task_id_tree_c IS NOT NULL AND tree_c_owner IS NOT NULL) AS treec_running, + + -- TreeR stage + COUNT(*) FILTER (WHERE after_tree_c = true AND after_tree_r = false AND task_id_tree_r IS NOT NULL AND tree_r_owner IS NULL) AS treer_pending, + COUNT(*) FILTER (WHERE after_tree_c = true AND after_tree_r = false AND task_id_tree_r IS NOT NULL AND tree_r_owner IS NOT NULL) AS treer_running, + + -- Notify stage + COUNT(*) FILTER (WHERE after_tree_r = true AND after_notify_client = false AND task_id_notify_client IS NOT NULL AND notify_owner IS NULL) AS notify_pending, + COUNT(*) FILTER (WHERE after_tree_r = true AND after_notify_client = false AND task_id_notify_client IS NOT NULL AND notify_owner IS NOT NULL) AS notify_running, + + -- Finalize stage + COUNT(*) FILTER (WHERE after_c1_supplied = true AND after_finalize = false AND task_id_finalize IS NOT NULL AND finalize_owner IS NULL) AS finalize_pending, + COUNT(*) FILTER (WHERE after_c1_supplied = true AND after_finalize = false AND task_id_finalize IS NOT NULL AND finalize_owner IS NOT NULL) AS finalize_running, + + -- Cleanup stage + COUNT(*) FILTER (WHERE after_finalize = true AND after_cleanup = false AND task_id_cleanup IS NOT NULL AND cleanup_owner IS NULL) AS cleanup_pending, + COUNT(*) FILTER (WHERE after_finalize = true AND after_cleanup = false AND task_id_cleanup IS NOT NULL AND cleanup_owner IS NOT NULL) AS cleanup_running +FROM pipeline_data +` + + var cts []struct { + Total int64 `db:"total"` + + SDRPending int64 `db:"sdr_pending"` + SDRRunning int64 `db:"sdr_running"` + TreeDPending int64 `db:"treed_pending"` + TreeDRunning int64 `db:"treed_running"` + TreeCPending int64 `db:"treec_pending"` + TreeCRunning int64 `db:"treec_running"` + TreeRPending int64 `db:"treer_pending"` + TreeRRunning int64 `db:"treer_running"` + NotifyPending int64 `db:"notify_pending"` + NotifyRunning int64 `db:"notify_running"` + FinalizePending int64 `db:"finalize_pending"` + FinalizeRunning int64 `db:"finalize_running"` + CleanupPending int64 `db:"cleanup_pending"` + CleanupRunning int64 `db:"cleanup_running"` + } + + err := a.deps.DB.Select(ctx, &cts, query) + if err != nil { + return nil, xerrors.Errorf("failed to run remote seal provider stats query: %w", err) + } + + if len(cts) == 0 { + return &PipelineStats{ + Total: 0, + Stages: []PipelineStage{ + {Name: "SDR", Pending: 0, Running: 0}, + {Name: "TreeD", Pending: 0, Running: 0}, + {Name: "TreeC", Pending: 0, Running: 0}, + {Name: "TreeR", Pending: 0, Running: 0}, + {Name: "Notify", Pending: 0, Running: 0}, + {Name: "Finalize", Pending: 0, Running: 0}, + {Name: "Cleanup", Pending: 0, Running: 0}, + }, + }, nil + } + + counts := cts[0] + + out.Total = counts.Total + out.Stages = []PipelineStage{ + {Name: "SDR", Pending: counts.SDRPending, Running: counts.SDRRunning}, + {Name: "TreeD", Pending: counts.TreeDPending, Running: counts.TreeDRunning}, + {Name: "TreeC", Pending: counts.TreeCPending, Running: counts.TreeCRunning}, + {Name: "TreeR", Pending: counts.TreeRPending, Running: counts.TreeRRunning}, + {Name: "Notify", Pending: counts.NotifyPending, Running: counts.NotifyRunning}, + {Name: "Finalize", Pending: counts.FinalizePending, Running: counts.FinalizeRunning}, + {Name: "Cleanup", Pending: counts.CleanupPending, Running: counts.CleanupRunning}, + } + + return &out, nil +} + +// RSealClientStats returns aggregate statistics for the remote seal client pipeline. +func (a *WebRPC) RSealClientStats(ctx context.Context) (*PipelineStats, error) { + var out PipelineStats + + const query = ` +WITH pipeline_data AS ( + SELECT c.*, + del.owner_id AS delegate_owner, + pol.owner_id AS poll_owner, + ftch.owner_id AS fetch_owner, + cln.owner_id AS cleanup_owner + FROM rseal_client_pipeline c + LEFT JOIN harmony_task del ON del.id = c.task_id_delegate + LEFT JOIN harmony_task pol ON pol.id = c.task_id_poll + LEFT JOIN harmony_task ftch ON ftch.id = c.task_id_fetch + LEFT JOIN harmony_task cln ON cln.id = c.task_id_cleanup + WHERE c.after_cleanup = FALSE AND c.failed = FALSE +) +SELECT + COUNT(*) AS total, + + -- Delegate stage + COUNT(*) FILTER (WHERE task_id_delegate IS NOT NULL AND delegate_owner IS NULL) AS delegate_pending, + COUNT(*) FILTER (WHERE task_id_delegate IS NOT NULL AND delegate_owner IS NOT NULL) AS delegate_running, + + -- Poll stage + COUNT(*) FILTER (WHERE after_sdr = false AND task_id_poll IS NOT NULL AND poll_owner IS NULL) AS poll_pending, + COUNT(*) FILTER (WHERE after_sdr = false AND task_id_poll IS NOT NULL AND poll_owner IS NOT NULL) AS poll_running, + + -- Fetch stage + COUNT(*) FILTER (WHERE after_sdr = true AND after_fetch = false AND task_id_fetch IS NOT NULL AND fetch_owner IS NULL) AS fetch_pending, + COUNT(*) FILTER (WHERE after_sdr = true AND after_fetch = false AND task_id_fetch IS NOT NULL AND fetch_owner IS NOT NULL) AS fetch_running, + + -- Cleanup stage + COUNT(*) FILTER (WHERE after_fetch = true AND after_cleanup = false AND task_id_cleanup IS NOT NULL AND cleanup_owner IS NULL) AS cleanup_pending, + COUNT(*) FILTER (WHERE after_fetch = true AND after_cleanup = false AND task_id_cleanup IS NOT NULL AND cleanup_owner IS NOT NULL) AS cleanup_running +FROM pipeline_data +` + + var cts []struct { + Total int64 `db:"total"` + + DelegatePending int64 `db:"delegate_pending"` + DelegateRunning int64 `db:"delegate_running"` + PollPending int64 `db:"poll_pending"` + PollRunning int64 `db:"poll_running"` + FetchPending int64 `db:"fetch_pending"` + FetchRunning int64 `db:"fetch_running"` + CleanupPending int64 `db:"cleanup_pending"` + CleanupRunning int64 `db:"cleanup_running"` + } + + err := a.deps.DB.Select(ctx, &cts, query) + if err != nil { + return nil, xerrors.Errorf("failed to run remote seal client stats query: %w", err) + } + + if len(cts) == 0 { + return &PipelineStats{ + Total: 0, + Stages: []PipelineStage{ + {Name: "Delegate", Pending: 0, Running: 0}, + {Name: "Poll", Pending: 0, Running: 0}, + {Name: "Fetch", Pending: 0, Running: 0}, + {Name: "Cleanup", Pending: 0, Running: 0}, + }, + }, nil + } + + counts := cts[0] + + out.Total = counts.Total + out.Stages = []PipelineStage{ + {Name: "Delegate", Pending: counts.DelegatePending, Running: counts.DelegateRunning}, + {Name: "Poll", Pending: counts.PollPending, Running: counts.PollRunning}, + {Name: "Fetch", Pending: counts.FetchPending, Running: counts.FetchRunning}, + {Name: "Cleanup", Pending: counts.CleanupPending, Running: counts.CleanupRunning}, + } + + return &out, nil +} diff --git a/web/api/webrpc/routes.go b/web/api/webrpc/routes.go index 9efd737f5..7acf3d385 100644 --- a/web/api/webrpc/routes.go +++ b/web/api/webrpc/routes.go @@ -34,6 +34,10 @@ func (a *WebRPC) BlockDelaySecs(context.Context) (uint64, error) { return build.BlockDelaySecs, nil } +func (a *WebRPC) InstanceColor(context.Context) (uint8, error) { + return a.deps.Cfg.Subsystems.WebInstanceColorHue, nil +} + func Routes(r *mux.Router, deps *deps.Deps, debug bool) { handler := &WebRPC{ deps: deps, diff --git a/web/static/pages/remote-seal/index.html b/web/static/pages/remote-seal/index.html new file mode 100644 index 000000000..62da7b434 --- /dev/null +++ b/web/static/pages/remote-seal/index.html @@ -0,0 +1,78 @@ + + +
+Configure remote seal providers that will handle SDR and tree computation for this node's sectors.
+ + ${this.ourURL ? html` +${this.ourURL} (share this with providers when they add you as a partner)| ID | +Miner | +Provider URL | +Name | +Enabled | +Available | +Created | +Actions | +
|---|---|---|---|---|---|---|---|
| ${p.id} | +f0${p.sp_id} | +${p.provider_url} | ++ this.renameProvider(p.id, p.provider_name)} style="cursor:pointer" title="Click to rename"> + ${p.provider_name || html`unnamed`} + + | ++ ${p.enabled ? 'Yes' : 'No'} + | +${this.renderAvailability(p.id)} | +${new Date(p.created_at).toLocaleDateString()} | ++ + + + | +
No providers configured yet.
`} +| SP | +Sector | +Partner | +SDR | +TreeD | +TreeC | +TreeR | +Notify | +C1 | +Finalize | +Cleanup | +Status | +
|---|---|---|---|---|---|---|---|---|---|---|---|
| f0${r.sp_id} | +${r.sector_number} | +${r.partner_name} | +${this.renderTaskStage(r.task_id_sdr, r.after_sdr)} | +${this.renderTaskStage(r.task_id_tree_d, r.after_tree_d)} | +${this.renderTaskStage(r.task_id_tree_c, r.after_tree_c)} | +${this.renderTaskStage(r.task_id_tree_r, r.after_tree_r)} | +${this.renderTaskStage(r.task_id_notify_client, r.after_notify_client)} | +${this.renderStage(r.after_c1_supplied)} | +${this.renderTaskStage(r.task_id_finalize, r.after_finalize)} | +${this.renderTaskStage(r.task_id_cleanup, r.after_cleanup)} | +${r.failed ? html`Failed` : html`Active`} | +
No active provider pipeline rows.
`} + + ${this.renderStats(this.clientStats, 'Client')} + +| SP | +Sector | +Provider | +Delegate | +Poll | +Fetch | +Cleanup | +Status | +
|---|---|---|---|---|---|---|---|
| f0${r.sp_id} | +${r.sector_number} | +${r.provider_name} | +${this.renderTaskStage(r.task_id_delegate, r.after_delegate)} | +${this.renderTaskStage(r.task_id_poll, r.after_sdr)} | +${this.renderTaskStage(r.task_id_fetch, r.after_fetch)} | +${this.renderTaskStage(r.task_id_cleanup, r.after_cleanup)} | +${r.failed ? html`Failed` : html`Active`} | +
No active client pipeline rows.
`} +Manage partners that are allowed to delegate sealing to this node.
+ + ${this.ourURL ? html` +${this.ourURL}| ID | +Name | +URL | +Sectors Remaining | +Sectors Total | +Proof Types | +Created | +Actions | +
|---|---|---|---|---|---|---|---|
| ${p.id} | +${p.partner_name} | +${p.partner_url} | +${p.allowance_remaining} | +${p.allowance_total} | +${this.proofTypeLabels(p.allowed_proof_types)} | +${new Date(p.created_at).toLocaleDateString()} | ++ + + + | +
No partners configured yet.
`} +