diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f2d47a009..608e28e4a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -176,6 +176,8 @@ jobs: target: "./itests/deal_e2e_test.go" - name: test-itest-dynamic-config target: "./itests/dyncfg_test.go" + - name: test-itest-grace-restart + target: "./itests/grace_restart_test.go" - name: test-itest-harmonyDB target: "./itests/harmonydb_test.go" - name: test-itest-park-piece diff --git a/api/api_curio.go b/api/api_curio.go index 0032470b7..676d9c207 100644 --- a/api/api_curio.go +++ b/api/api_curio.go @@ -25,6 +25,7 @@ type Curio interface { Version(context.Context) ([]int, error) //perm:admin Shutdown(context.Context) error //perm:admin + Restart(context.Context) error //perm:admin Cordon(context.Context) error //perm:admin Uncordon(context.Context) error //perm:admin Info(ctx context.Context) (*ltypes.NodeInfo, error) //perm:read diff --git a/api/proxy_gen.go b/api/proxy_gen.go index 98e86287f..fed06e06e 100644 --- a/api/proxy_gen.go +++ b/api/proxy_gen.go @@ -56,6 +56,8 @@ type CurioMethods struct { LogSetLevel func(p0 context.Context, p1 string, p2 string) error `perm:"admin"` + Restart func(p0 context.Context) error `perm:"admin"` + Shutdown func(p0 context.Context) error `perm:"admin"` StorageAddLocal func(p0 context.Context, p1 string) error `perm:"admin"` @@ -303,6 +305,17 @@ func (s *CurioStub) LogSetLevel(p0 context.Context, p1 string, p2 string) error return ErrNotSupported } +func (s *CurioStruct) Restart(p0 context.Context) error { + if s.Internal.Restart == nil { + return ErrNotSupported + } + return s.Internal.Restart(p0) +} + +func (s *CurioStub) Restart(p0 context.Context) error { + return ErrNotSupported +} + func (s *CurioStruct) Shutdown(p0 context.Context) error { if s.Internal.Shutdown == nil { return ErrNotSupported diff --git a/apt/curio.service b/apt/curio.service index ead59f24c..02ba167fb 100644 --- a/apt/curio.service +++ b/apt/curio.service @@ -4,6 +4,8 @@ After=network.target [Service] ExecStart=/usr/local/bin/curio run +ExecReload=/bin/kill -USR2 $MAINPID +KillSignal=SIGTERM Environment=GOLOG_FILE="/var/log/curio/curio.log" Environment=GOLOG_LOG_FMT="json" Environment=LD_LIBRARY_PATH="/usr/lib/x86_64-linux-gnu:/usr/lib64" diff --git a/build/openrpc/curio.json b/build/openrpc/curio.json index cacae06e6..d165c56ff 100644 --- a/build/openrpc/curio.json +++ b/build/openrpc/curio.json @@ -312,7 +312,7 @@ "deprecated": false, "externalDocs": { "description": "Github remote link", - "url": "https://github.com/filecoin-project/curio/blob/master/api/proxy_gen.go#L237" + "url": "https://github.com/filecoin-project/curio/blob/master/api/proxy_gen.go#L239" } }, { @@ -335,7 +335,7 @@ "deprecated": false, "externalDocs": { "description": "Github remote link", - "url": "https://github.com/filecoin-project/curio/blob/master/api/proxy_gen.go#L248" + "url": "https://github.com/filecoin-project/curio/blob/master/api/proxy_gen.go#L250" } }, { @@ -391,7 +391,7 @@ "deprecated": false, "externalDocs": { "description": "Github remote link", - "url": "https://github.com/filecoin-project/curio/blob/master/api/proxy_gen.go#L259" + "url": "https://github.com/filecoin-project/curio/blob/master/api/proxy_gen.go#L261" } }, { @@ -535,7 +535,7 @@ "deprecated": false, "externalDocs": { "description": "Github remote link", - "url": "https://github.com/filecoin-project/curio/blob/master/api/proxy_gen.go#L270" + "url": "https://github.com/filecoin-project/curio/blob/master/api/proxy_gen.go#L272" } }, { @@ -571,7 +571,7 @@ "deprecated": false, "externalDocs": { "description": "Github remote link", - "url": "https://github.com/filecoin-project/curio/blob/master/api/proxy_gen.go#L281" + "url": "https://github.com/filecoin-project/curio/blob/master/api/proxy_gen.go#L283" } }, { @@ -625,7 +625,30 @@ "deprecated": false, "externalDocs": { "description": "Github remote link", - "url": "https://github.com/filecoin-project/curio/blob/master/api/proxy_gen.go#L292" + "url": "https://github.com/filecoin-project/curio/blob/master/api/proxy_gen.go#L294" + } + }, + { + "name": "Filecoin.Restart", + "description": "```go\nfunc (s *CurioStruct) Restart(p0 context.Context) error {\n\tif s.Internal.Restart == nil {\n\t\treturn ErrNotSupported\n\t}\n\treturn s.Internal.Restart(p0)\n}\n```", + "summary": "", + "paramStructure": "by-position", + "params": [], + "result": { + "name": "Null", + "description": "Null", + "schema": { + "type": [ + "null" + ] + }, + "required": true, + "deprecated": false + }, + "deprecated": false, + "externalDocs": { + "description": "Github remote link", + "url": "https://github.com/filecoin-project/curio/blob/master/api/proxy_gen.go#L305" } }, { @@ -648,7 +671,7 @@ "deprecated": false, "externalDocs": { "description": "Github remote link", - "url": "https://github.com/filecoin-project/curio/blob/master/api/proxy_gen.go#L303" + "url": "https://github.com/filecoin-project/curio/blob/master/api/proxy_gen.go#L316" } }, { @@ -687,7 +710,7 @@ "deprecated": false, "externalDocs": { "description": "Github remote link", - "url": "https://github.com/filecoin-project/curio/blob/master/api/proxy_gen.go#L314" + "url": "https://github.com/filecoin-project/curio/blob/master/api/proxy_gen.go#L327" } }, { @@ -726,7 +749,7 @@ "deprecated": false, "externalDocs": { "description": "Github remote link", - "url": "https://github.com/filecoin-project/curio/blob/master/api/proxy_gen.go#L325" + "url": "https://github.com/filecoin-project/curio/blob/master/api/proxy_gen.go#L338" } }, { @@ -920,7 +943,7 @@ "deprecated": false, "externalDocs": { "description": "Github remote link", - "url": "https://github.com/filecoin-project/curio/blob/master/api/proxy_gen.go#L336" + "url": "https://github.com/filecoin-project/curio/blob/master/api/proxy_gen.go#L349" } }, { @@ -981,7 +1004,7 @@ "deprecated": false, "externalDocs": { "description": "Github remote link", - "url": "https://github.com/filecoin-project/curio/blob/master/api/proxy_gen.go#L347" + "url": "https://github.com/filecoin-project/curio/blob/master/api/proxy_gen.go#L360" } }, { @@ -1113,7 +1136,7 @@ "deprecated": false, "externalDocs": { "description": "Github remote link", - "url": "https://github.com/filecoin-project/curio/blob/master/api/proxy_gen.go#L358" + "url": "https://github.com/filecoin-project/curio/blob/master/api/proxy_gen.go#L371" } }, { @@ -1247,7 +1270,7 @@ "deprecated": false, "externalDocs": { "description": "Github remote link", - "url": "https://github.com/filecoin-project/curio/blob/master/api/proxy_gen.go#L369" + "url": "https://github.com/filecoin-project/curio/blob/master/api/proxy_gen.go#L382" } }, { @@ -1301,7 +1324,7 @@ "deprecated": false, "externalDocs": { "description": "Github remote link", - "url": "https://github.com/filecoin-project/curio/blob/master/api/proxy_gen.go#L380" + "url": "https://github.com/filecoin-project/curio/blob/master/api/proxy_gen.go#L393" } }, { @@ -1335,7 +1358,7 @@ "deprecated": false, "externalDocs": { "description": "Github remote link", - "url": "https://github.com/filecoin-project/curio/blob/master/api/proxy_gen.go#L391" + "url": "https://github.com/filecoin-project/curio/blob/master/api/proxy_gen.go#L404" } }, { @@ -1389,7 +1412,7 @@ "deprecated": false, "externalDocs": { "description": "Github remote link", - "url": "https://github.com/filecoin-project/curio/blob/master/api/proxy_gen.go#L402" + "url": "https://github.com/filecoin-project/curio/blob/master/api/proxy_gen.go#L415" } }, { @@ -1466,7 +1489,7 @@ "deprecated": false, "externalDocs": { "description": "Github remote link", - "url": "https://github.com/filecoin-project/curio/blob/master/api/proxy_gen.go#L413" + "url": "https://github.com/filecoin-project/curio/blob/master/api/proxy_gen.go#L426" } }, { @@ -1489,7 +1512,7 @@ "deprecated": false, "externalDocs": { "description": "Github remote link", - "url": "https://github.com/filecoin-project/curio/blob/master/api/proxy_gen.go#L424" + "url": "https://github.com/filecoin-project/curio/blob/master/api/proxy_gen.go#L437" } }, { @@ -1527,7 +1550,7 @@ "deprecated": false, "externalDocs": { "description": "Github remote link", - "url": "https://github.com/filecoin-project/curio/blob/master/api/proxy_gen.go#L435" + "url": "https://github.com/filecoin-project/curio/blob/master/api/proxy_gen.go#L448" } } ] diff --git a/cmd/curio/cli.go b/cmd/curio/cli.go index 3935f3801..0d12fcc0d 100644 --- a/cmd/curio/cli.go +++ b/cmd/curio/cli.go @@ -209,6 +209,7 @@ var cliCmd = &cli.Command{ logCmd, waitApiCmd, stopCmd, + restartCmd, cordonCmd, uncordonCmd, indexSampleCmd, diff --git a/cmd/curio/internal/translations/catalog.go b/cmd/curio/internal/translations/catalog.go index 07aff39e8..79536e55d 100644 --- a/cmd/curio/internal/translations/catalog.go +++ b/cmd/curio/internal/translations/catalog.go @@ -171,261 +171,263 @@ var messageKeyToIndex = map[string]int{ "Get a config layer by name. You may want to pipe the output to a file, or use 'less'": 128, "Get information about unsealed data": 129, "Go Back": 130, - "Host: %s": 131, - "How long to commit sectors for": 132, - "How would you like to proceed?": 133, - "I want to:": 134, - "If, however, the upgrade has a serious bug and you need to downgrade, first shutdown all nodes in your cluster and then run this command. Finally, only start downgraded nodes.": 135, - "Ignore sectors that cannot be migrated": 136, - "Import delegated wallet private key": 137, - "Increase reliability using redundancy: start multiple machines with at-least the post layer: 'curio run --layers=post'": 138, - "Initializing a new miner actor.": 139, - "Initiate a withdrawal request from the client's deposit": 140, - "Initiate a withdrawal request from the service pool": 141, - "Interpret stacked config layers by this version of curio, with system-generated comments.": 142, - "Invalid private key: %s": 143, - "Layer %s created. ": 144, - "Limit output to the last N events": 145, - "List config layers present in the DB.": 146, - "List data from the sectors_unseal_pipeline and sectors_meta tables": 147, - "List log systems": 148, - "List pipeline events": 149, - "Listen address for HTTP server": 150, - "Lotus-Miner to Curio Migration.": 151, - "Make sure to send FIL/tFIL to your 0x wallet address for PDP operations.": 152, - "Manage batch sealing operations": 153, - "Manage logging": 154, - "Manage node config by layers. The layer 'base' will always be applied at Curio start-up.": 155, - "Manage storage paths for this server.": 156, - "Manage the sealing pipeline": 157, - "Manage unsealed data": 158, - "Math Utils": 159, - "Migrate from existing Lotus-Miner": 160, - "Migrating lotus-miner config.toml to Curio in-database configuration.": 161, - "Migrating metadata for %d sectors.": 162, - "Miner %s created successfully": 163, - "Miner creation error occurred: %s ": 164, - "Minimum number of hugepages required": 165, - "Moves funds from the deal collateral wallet into escrow with the storage market actor": 166, - "Name of the Postgres database in Yugabyte cluster": 167, - "New Miner initialization complete.": 168, - "New service actor address": 169, - "Next steps:": 170, - "No address provided": 171, - "No domain provided, skipping HTTP configuration": 172, - "No host provided": 173, - "No path provided": 174, - "No path provided, abandoning migration ": 175, - "No private key provided": 176, - "No value provided": 177, - "No, abort": 178, - "No, keep it": 179, - "Non-SP cluster configuration complete": 180, - "Non-SP cluster configuration created successfully": 181, - "Non-SP cluster setup complete!": 182, - "Note: This command is intended to be used to verify PoSt compute performance.\nIt will not send any messages to the chain. Since it can compute any deadline, output may be incorrectly timed for the chain.": 183, - "Number of 1GB hugepages to configure": 184, - "Number of sectors to start": 185, - "One database can serve multiple miner IDs: Run a migration for each lotus-miner.": 186, - "Optional setup steps (you can skip these and configure later):": 187, - "Other": 188, - "Output file path (default: stdout)": 189, - "Owner Wallet: %s": 190, - "PDP": 191, - "PDP (Proof of Data Possession) Configuration": 192, - "PDP configuration layer created": 193, - "PDP layer already exists. What would you like to do?": 194, - "PDP setup complete!": 195, - "PDP wallet configured": 196, - "PDP wallet imported": 197, - "PDP wallet imported successfully!": 198, - "Password for connecting to the Postgres database in Yugabyte cluster": 199, - "Password: %s": 200, - "Path already exists": 201, - "Path to miner repo": 202, - "Please do not run guided-setup again as miner creation is not idempotent. You need to run 'curio config new-cluster %s' to finish the configuration": 203, - "Please do not run guided-setup again. You need to run 'curio config new-cluster' manually to finish the configuration": 204, - "Port for connecting to the Cassandra database in Yugabyte cluster": 205, - "Port for connecting to the Postgres database in Yugabyte cluster": 206, - "Port: %s": 207, - "Pre-initialization steps complete": 208, - "Print default node config": 209, - "Private key": 210, - "Private key cannot be empty": 211, - "Propose a new service actor": 212, - "Provider actor": 213, - "Provider actor address": 214, - "Provider sender address": 215, - "Provides a sample of CIDs from an indexed piece": 216, - "Query the service state": 217, - "Query the state of a client": 218, - "Query the state of a provider": 219, - "Read Miner Config": 220, - "Read piece CIDs one per line from stdin instead of from arguments": 221, - "Really delete %s?": 222, - "Reconfigure PDP": 223, - "Redeem a client voucher (service role)": 224, - "Redeem a provider voucher (provider role)": 225, - "Remove a named config layer.": 226, - "Remove unsealed copies of sector containing this deal": 227, - "Resolve each piece CID to sector(s) via market_piece_deal, then set target_unseal_state for those sectors.\n Accepts piece CID v1 or v2. Use --target-state to specify the desired state (true or false). Request all at once to minimize the sectors needed to be unsealed (if you have pieces stored in multiple sectors).\n Use --stdin to read piece CIDs one per line from stdin (for large lists).": 228, - "SP ID to compute WindowPoSt for": 229, - "Seal (fast storage for sealing operations)": 230, - "Sector Size: %s": 231, - "Sector selection failed: %s ": 232, - "Sectors can be stored across many filesystem paths. These\ncommands provide ways to manage the storage a Curio node will use to store sectors\nlong term for proving (references as 'store') as well as how sectors will be\nstored while moving through the sealing pipeline (references as 'seal').": 233, - "Select the Sector Size": 234, - "Select the location of your lotus-miner config directory?": 235, - "Sender Wallet: %s": 236, - "Sender address": 237, - "Service sender address": 238, - "Set a config layer or the base by providing a filename or stdin.": 239, - "Set log level": 240, - "Set the log level for logging systems:\n\n The system flag can be specified multiple times.\n\n eg) log set-level --system chain --system chainxchg debug\n\n Available Levels:\n debug\n info\n warn\n error\n\n Environment Variables:\n GOLOG_LOG_LEVEL - Default log level for all log systems\n GOLOG_LOG_FMT - Change output log format (json, nocolor)\n GOLOG_FILE - Write logs to file\n GOLOG_OUTPUT - Specify whether to output to file, stderr, stdout or a combination, i.e. file+stderr\n": 241, - "Set the target unseal state for a sector": 242, - "Set the target unseal state for a specific sector.\n : The storage provider ID\n : The sector number\n : The target state (true, false)\n\n The unseal target state indicates to curio how an unsealed copy of the sector should be maintained.\n\t If the target state is true, curio will ensure that the sector is unsealed.\n\t If the target state is false, curio will ensure that there is no unsealed copy of the sector.\n\t If the target state is none, curio will not change the current state of the sector.\n\n Currently when the curio will only start new unseal processes when the target state changes from another state to true.\n\n When the target state is false, and an unsealed sector file exists, the GC mark step will create a removal mark\n for the unsealed sector file. The file will only be removed after the removal mark is accepted.\n": 243, - "Set the target unseal state for sectors containing the given piece CIDs": 244, - "Setting up PDP wallet...": 245, - "Setup SPDK for batch sealing (configures hugepages and binds NVMe devices)": 246, - "Setup SPDK for batch sealing operations.\n\nThis command automatically:\n- Downloads SPDK if not already available\n- Configures 1GB hugepages (36 pages minimum)\n- Binds NVMe devices for use with SupraSeal\n\nRequires root/sudo access for SPDK setup operations.": 247, - "Setup non-Storage Provider cluster": 248, - "Skip PDP setup": 249, - "Skip optional steps": 250, - "Skip wallet setup for now": 251, - "Specify actor address for the deal": 252, - "Specify actor address to start sealing sectors for": 253, - "Specify wallet address to send the funds from": 254, - "Start Curio web interface": 255, - "Start a Curio process": 256, - "Start an instance of Curio web interface. \n\tThis creates the 'web' layer if it does not exist, then calls run with that layer.": 257, - "Start new sealing operations manually": 258, - "Start sealing new CC sectors": 259, - "Start sealing sectors for all actors now (not on schedule)": 260, - "Step Complete: %s\n": 261, - "Stop a running Curio process": 262, - "Storage": 263, - "Storage Configuration": 264, - "Storage can be attached to a Curio node using this command. The storage volume\nlist is stored local to the Curio node in storage.json set in curio run. We do not\nrecommend manually modifying this value without further understanding of the\nstorage system.\n\nEach storage volume contains a configuration file which describes the\ncapabilities of the volume. When the '--init' flag is provided, this file will\nbe created using the additional flags.\n\nWeight\nA high weight value means data will be more likely to be stored in this path\n\nSeal\nData for the sealing process will be stored here\n\nStore\nFinalized sectors that will be moved here for long term storage and be proven\nover time\n ": 265, - "Storage path %s added as %s. You'll need to initialize it with: curio cli storage attach --init --%s %s": 266, - "Storage path %s removed from configuration": 267, - "Storage path added": 268, - "Storage path deleted": 269, - "Storage paths for this server:": 270, - "Storage type for %s": 271, - "Store (long-term storage for sealed sectors)": 272, - "Supra consensus testing utilities": 273, - "Target state: true (ensure unsealed), false (ensure no unsealed copy)": 274, - "Test the windowpost scheduler by running it on the next available curio. If tasks fail all retries, you will need to ctrl+c to exit.": 275, - "The '%s' layer stores common configuration. All curio instances can include it in their %s argument.": 276, - "This interactive tool creates a new miner actor and creates the basic configuration layer for it.": 277, - "This interactive tool migrates lotus-miner to Curio in 5 minutes.": 278, - "This interactive tool sets up a non-Storage Provider cluster for protocols like PDP, Snark market, and others.": 279, - "This process is partially idempotent. Once a new miner actor has been created and subsequent steps fail, the user need to run 'curio config new-cluster < miner ID >' to finish the configuration.": 280, - "This setup does not create or migrate a Filecoin SP actor.": 281, - "This will configure PDP settings for your Curio cluster.": 282, - "To run Curio: With machine or cgroup isolation, use the command (with example layer selection):": 283, - "To start Curio with PDP enabled, run:": 284, - "To start the cluster, run: curio run --layers basic-cluster": 285, - "To start, ensure your sealing pipeline is drained and shut-down lotus-miner.": 286, - "To work with the config: ": 287, - "Tool Box for Curio": 288, - "Try the web interface with %s ": 289, - "Uncordon a machine, resume scheduling": 290, - "Unmigratable sectors found. Do you want to continue?": 291, - "Update data for messages in wait queue": 292, - "Updated DB with message data missing from chain node": 293, - "Use existing key, ending in %s": 294, - "Use synthetic PoRep": 295, - "Use the arrow keys to navigate: ↓ ↑ → ← ": 296, - "Username for connecting to the Postgres database in Yugabyte cluster": 297, - "Username: %s": 298, - "Using existing PDP wallet key: %s": 299, - "Utility functions for testing": 300, - "Validate a client voucher signature": 301, - "Validate a provider voucher signature": 302, - "Voucher nonce": 303, - "Voucher signature (hex)": 304, - "Wait for Curio api to come online": 305, - "Where should we save your database config file?": 306, - "Withdrawal amount (in FIL)": 307, - "Worker Wallet: %s": 308, - "YYYYMMDD when your cluster had the preferred schema. Ex: 20251128": 309, - "Yes, continue": 310, - "Yes, delete it": 311, - "You can add other layers for per-machine configuration changes.": 312, - "You can create a new delegated wallet using Lotus:": 313, - "You can now migrate your market node (%s), if applicable.": 314, - "You can now start using Curio for protocols like PDP, Snark markets, and others.": 315, - "You can set up the wallet later using the Curio GUI or CLI": 316, - "You need a delegated Filecoin wallet address to use with PDP.": 317, - "Your non-SP cluster has been configured successfully.": 318, - "Zen3 and later supports two sectors per thread, set to false for older CPUs": 319, - "[SP actor address...]": 320, - "[deadline index]": 321, - "[layer name]": 322, - "[level]": 323, - "[miner address] [sector number]": 324, - "[path]": 325, - "[sectorSize]": 326, - "`URL` to send the request to": 327, - "a layer's file name": 328, - "a list of layers to be interpreted as the final config": 329, - "allow overwrite of existing layer if source is a different layer": 330, - "attach local storage path": 331, - "check how many entries need to be fixed": 332, - "comma or space separated list of layers to be interpreted (base is always applied)": 333, - "could not get API info for FullNode: %w": 334, - "curio run --layers=gui,pdp": 335, - "custom node name": 336, - "deadline to compute WindowPoSt for ": 337, - "depends on output being a TTY": 338, - "detach local storage path": 339, - "do not interpret source layer": 340, - "don't check full-node sync status": 341, - "don't comment default values": 342, - "duration to wait till fail": 343, - "edit a config layer": 344, - "editor to use": 345, - "enables very verbose mode, useful for debugging the CLI": 346, - "find sector in the storage system": 347, - "generate ipni chunks from a file": 348, - "generate vanilla proof for a sector": 349, - "host address and port the gui will listen on": 350, - "host address and port the worker api will listen on": 351, - "indicates that deal index should not be announced to the IPNI": 352, - "initialize the path first": 353, - "layer name": 354, - "limit to log system": 355, - "list local storage paths": 356, - "list of layers to be interpreted (atop defaults). Default: base": 357, - "location of boost.db file": 358, - "lotus wallet export
| xxd -r -p | jq -r '.PrivateKey' | base64 -d | xxd -p -c 32": 359, - "lotus wallet new delegated": 360, - "machine host:port (curio run --listen address)": 361, - "manage open file limit": 362, - "manage sector storage": 363, - "maximum fee in FIL user is willing to pay for this message": 364, - "only list local storage paths": 365, - "only run init, then return": 366, - "output in json format": 367, - "partition to compute WindowPoSt for": 368, - "path group names": 369, - "path groups allowed to pull data from this path (allow all if not specified)": 370, - "piece-cid": 371, - "redeclare sectors in a local storage path": 372, - "save the whole config into the layer, not just the diff": 373, - "source config layer": 374, - "start epoch by when the deal should be proved by provider on-chain (default: 2 days from now)": 375, - "start sealing a deal sector early": 376, - "title of the config layer (req'd for stdin)": 377, - "true if --source is set": 378, - "use color in display output": 379, - "yugabyte hosts to connect to over cassandra interface eg '127.0.0.1'": 380, - "yugabyte password to connect to over cassandra interface eg 'cassandra'": 381, - "yugabyte username to connect to over cassandra interface eg 'cassandra'": 382, + "Gracefully restart a running Curio process": 131, + "Host: %s": 132, + "How long to commit sectors for": 133, + "How would you like to proceed?": 134, + "I want to:": 135, + "If, however, the upgrade has a serious bug and you need to downgrade, first shutdown all nodes in your cluster and then run this command. Finally, only start downgraded nodes.": 136, + "Ignore sectors that cannot be migrated": 137, + "Import delegated wallet private key": 138, + "Increase reliability using redundancy: start multiple machines with at-least the post layer: 'curio run --layers=post'": 139, + "Initializing a new miner actor.": 140, + "Initiate a withdrawal request from the client's deposit": 141, + "Initiate a withdrawal request from the service pool": 142, + "Interpret stacked config layers by this version of curio, with system-generated comments.": 143, + "Invalid private key: %s": 144, + "Layer %s created. ": 145, + "Limit output to the last N events": 146, + "List config layers present in the DB.": 147, + "List data from the sectors_unseal_pipeline and sectors_meta tables": 148, + "List log systems": 149, + "List pipeline events": 150, + "Listen address for HTTP server": 151, + "Lotus-Miner to Curio Migration.": 152, + "Make sure to send FIL/tFIL to your 0x wallet address for PDP operations.": 153, + "Manage batch sealing operations": 154, + "Manage logging": 155, + "Manage node config by layers. The layer 'base' will always be applied at Curio start-up.": 156, + "Manage storage paths for this server.": 157, + "Manage the sealing pipeline": 158, + "Manage unsealed data": 159, + "Math Utils": 160, + "Migrate from existing Lotus-Miner": 161, + "Migrating lotus-miner config.toml to Curio in-database configuration.": 162, + "Migrating metadata for %d sectors.": 163, + "Miner %s created successfully": 164, + "Miner creation error occurred: %s ": 165, + "Minimum number of hugepages required": 166, + "Moves funds from the deal collateral wallet into escrow with the storage market actor": 167, + "Name of the Postgres database in Yugabyte cluster": 168, + "New Miner initialization complete.": 169, + "New service actor address": 170, + "Next steps:": 171, + "No address provided": 172, + "No domain provided, skipping HTTP configuration": 173, + "No host provided": 174, + "No path provided": 175, + "No path provided, abandoning migration ": 176, + "No private key provided": 177, + "No value provided": 178, + "No, abort": 179, + "No, keep it": 180, + "Non-SP cluster configuration complete": 181, + "Non-SP cluster configuration created successfully": 182, + "Non-SP cluster setup complete!": 183, + "Note: This command is intended to be used to verify PoSt compute performance.\nIt will not send any messages to the chain. Since it can compute any deadline, output may be incorrectly timed for the chain.": 184, + "Number of 1GB hugepages to configure": 185, + "Number of sectors to start": 186, + "One database can serve multiple miner IDs: Run a migration for each lotus-miner.": 187, + "Optional setup steps (you can skip these and configure later):": 188, + "Other": 189, + "Output file path (default: stdout)": 190, + "Owner Wallet: %s": 191, + "PDP": 192, + "PDP (Proof of Data Possession) Configuration": 193, + "PDP configuration layer created": 194, + "PDP layer already exists. What would you like to do?": 195, + "PDP setup complete!": 196, + "PDP wallet configured": 197, + "PDP wallet imported": 198, + "PDP wallet imported successfully!": 199, + "Password for connecting to the Postgres database in Yugabyte cluster": 200, + "Password: %s": 201, + "Path already exists": 202, + "Path to miner repo": 203, + "Please do not run guided-setup again as miner creation is not idempotent. You need to run 'curio config new-cluster %s' to finish the configuration": 204, + "Please do not run guided-setup again. You need to run 'curio config new-cluster' manually to finish the configuration": 205, + "Port for connecting to the Cassandra database in Yugabyte cluster": 206, + "Port for connecting to the Postgres database in Yugabyte cluster": 207, + "Port: %s": 208, + "Pre-initialization steps complete": 209, + "Print default node config": 210, + "Private key": 211, + "Private key cannot be empty": 212, + "Propose a new service actor": 213, + "Provider actor": 214, + "Provider actor address": 215, + "Provider sender address": 216, + "Provides a sample of CIDs from an indexed piece": 217, + "Query the service state": 218, + "Query the state of a client": 219, + "Query the state of a provider": 220, + "Read Miner Config": 221, + "Read piece CIDs one per line from stdin instead of from arguments": 222, + "Really delete %s?": 223, + "Reconfigure PDP": 224, + "Redeem a client voucher (service role)": 225, + "Redeem a provider voucher (provider role)": 226, + "Remove a named config layer.": 227, + "Remove unsealed copies of sector containing this deal": 228, + "Resolve each piece CID to sector(s) via market_piece_deal, then set target_unseal_state for those sectors.\n Accepts piece CID v1 or v2. Use --target-state to specify the desired state (true or false). Request all at once to minimize the sectors needed to be unsealed (if you have pieces stored in multiple sectors).\n Use --stdin to read piece CIDs one per line from stdin (for large lists).": 229, + "SP ID to compute WindowPoSt for": 230, + "Seal (fast storage for sealing operations)": 231, + "Sector Size: %s": 232, + "Sector selection failed: %s ": 233, + "Sectors can be stored across many filesystem paths. These\ncommands provide ways to manage the storage a Curio node will use to store sectors\nlong term for proving (references as 'store') as well as how sectors will be\nstored while moving through the sealing pipeline (references as 'seal').": 234, + "Select the Sector Size": 235, + "Select the location of your lotus-miner config directory?": 236, + "Sender Wallet: %s": 237, + "Sender address": 238, + "Service sender address": 239, + "Set a config layer or the base by providing a filename or stdin.": 240, + "Set log level": 241, + "Set the log level for logging systems:\n\n The system flag can be specified multiple times.\n\n eg) log set-level --system chain --system chainxchg debug\n\n Available Levels:\n debug\n info\n warn\n error\n\n Environment Variables:\n GOLOG_LOG_LEVEL - Default log level for all log systems\n GOLOG_LOG_FMT - Change output log format (json, nocolor)\n GOLOG_FILE - Write logs to file\n GOLOG_OUTPUT - Specify whether to output to file, stderr, stdout or a combination, i.e. file+stderr\n": 242, + "Set the target unseal state for a sector": 243, + "Set the target unseal state for a specific sector.\n : The storage provider ID\n : The sector number\n : The target state (true, false)\n\n The unseal target state indicates to curio how an unsealed copy of the sector should be maintained.\n\t If the target state is true, curio will ensure that the sector is unsealed.\n\t If the target state is false, curio will ensure that there is no unsealed copy of the sector.\n\t If the target state is none, curio will not change the current state of the sector.\n\n Currently when the curio will only start new unseal processes when the target state changes from another state to true.\n\n When the target state is false, and an unsealed sector file exists, the GC mark step will create a removal mark\n for the unsealed sector file. The file will only be removed after the removal mark is accepted.\n": 244, + "Set the target unseal state for sectors containing the given piece CIDs": 245, + "Setting up PDP wallet...": 246, + "Setup SPDK for batch sealing (configures hugepages and binds NVMe devices)": 247, + "Setup SPDK for batch sealing operations.\n\nThis command automatically:\n- Downloads SPDK if not already available\n- Configures 1GB hugepages (36 pages minimum)\n- Binds NVMe devices for use with SupraSeal\n\nRequires root/sudo access for SPDK setup operations.": 248, + "Setup non-Storage Provider cluster": 249, + "Skip PDP setup": 250, + "Skip optional steps": 251, + "Skip wallet setup for now": 252, + "Specify actor address for the deal": 253, + "Specify actor address to start sealing sectors for": 254, + "Specify wallet address to send the funds from": 255, + "Start Curio web interface": 256, + "Start a Curio process": 257, + "Start an instance of Curio web interface. \n\tThis creates the 'web' layer if it does not exist, then calls run with that layer.": 258, + "Start new sealing operations manually": 259, + "Start sealing new CC sectors": 260, + "Start sealing sectors for all actors now (not on schedule)": 261, + "Step Complete: %s\n": 262, + "Stop a running Curio process": 263, + "Storage": 264, + "Storage Configuration": 265, + "Storage can be attached to a Curio node using this command. The storage volume\nlist is stored local to the Curio node in storage.json set in curio run. We do not\nrecommend manually modifying this value without further understanding of the\nstorage system.\n\nEach storage volume contains a configuration file which describes the\ncapabilities of the volume. When the '--init' flag is provided, this file will\nbe created using the additional flags.\n\nWeight\nA high weight value means data will be more likely to be stored in this path\n\nSeal\nData for the sealing process will be stored here\n\nStore\nFinalized sectors that will be moved here for long term storage and be proven\nover time\n ": 266, + "Storage path %s added as %s. You'll need to initialize it with: curio cli storage attach --init --%s %s": 267, + "Storage path %s removed from configuration": 268, + "Storage path added": 269, + "Storage path deleted": 270, + "Storage paths for this server:": 271, + "Storage type for %s": 272, + "Store (long-term storage for sealed sectors)": 273, + "Supra consensus testing utilities": 274, + "Target state: true (ensure unsealed), false (ensure no unsealed copy)": 275, + "Test the windowpost scheduler by running it on the next available curio. If tasks fail all retries, you will need to ctrl+c to exit.": 276, + "The '%s' layer stores common configuration. All curio instances can include it in their %s argument.": 277, + "This interactive tool creates a new miner actor and creates the basic configuration layer for it.": 278, + "This interactive tool migrates lotus-miner to Curio in 5 minutes.": 279, + "This interactive tool sets up a non-Storage Provider cluster for protocols like PDP, Snark market, and others.": 280, + "This process is partially idempotent. Once a new miner actor has been created and subsequent steps fail, the user need to run 'curio config new-cluster < miner ID >' to finish the configuration.": 281, + "This setup does not create or migrate a Filecoin SP actor.": 282, + "This will configure PDP settings for your Curio cluster.": 283, + "To run Curio: With machine or cgroup isolation, use the command (with example layer selection):": 284, + "To start Curio with PDP enabled, run:": 285, + "To start the cluster, run: curio run --layers basic-cluster": 286, + "To start, ensure your sealing pipeline is drained and shut-down lotus-miner.": 287, + "To work with the config: ": 288, + "Tool Box for Curio": 289, + "Try the web interface with %s ": 290, + "Uncordon a machine, resume scheduling": 291, + "Unmigratable sectors found. Do you want to continue?": 292, + "Update data for messages in wait queue": 293, + "Updated DB with message data missing from chain node": 294, + "Use existing key, ending in %s": 295, + "Use synthetic PoRep": 296, + "Use the arrow keys to navigate: ↓ ↑ → ← ": 297, + "Username for connecting to the Postgres database in Yugabyte cluster": 298, + "Username: %s": 299, + "Using existing PDP wallet key: %s": 300, + "Utility functions for testing": 301, + "Validate a client voucher signature": 302, + "Validate a provider voucher signature": 303, + "Voucher nonce": 304, + "Voucher signature (hex)": 305, + "Wait for Curio api to come online": 306, + "Where should we save your database config file?": 307, + "Withdrawal amount (in FIL)": 308, + "Worker Wallet: %s": 309, + "YYYYMMDD when your cluster had the preferred schema. Ex: 20251128": 310, + "Yes, continue": 311, + "Yes, delete it": 312, + "You can add other layers for per-machine configuration changes.": 313, + "You can create a new delegated wallet using Lotus:": 314, + "You can now migrate your market node (%s), if applicable.": 315, + "You can now start using Curio for protocols like PDP, Snark markets, and others.": 316, + "You can set up the wallet later using the Curio GUI or CLI": 317, + "You need a delegated Filecoin wallet address to use with PDP.": 318, + "Your non-SP cluster has been configured successfully.": 319, + "Zen3 and later supports two sectors per thread, set to false for older CPUs": 320, + "[SP actor address...]": 321, + "[deadline index]": 322, + "[layer name]": 323, + "[level]": 324, + "[miner address] [sector number]": 325, + "[path]": 326, + "[sectorSize]": 327, + "`URL` to send the request to": 328, + "a layer's file name": 329, + "a list of layers to be interpreted as the final config": 330, + "allow overwrite of existing layer if source is a different layer": 331, + "attach local storage path": 332, + "check how many entries need to be fixed": 333, + "comma or space separated list of layers to be interpreted (base is always applied)": 334, + "could not get API info for FullNode: %w": 335, + "curio run --layers=gui,pdp": 336, + "custom node name": 337, + "deadline to compute WindowPoSt for ": 338, + "depends on output being a TTY": 339, + "detach local storage path": 340, + "do not interpret source layer": 341, + "don't check full-node sync status": 342, + "don't comment default values": 343, + "duration to wait till fail": 344, + "edit a config layer": 345, + "editor to use": 346, + "enables very verbose mode, useful for debugging the CLI": 347, + "find sector in the storage system": 348, + "generate ipni chunks from a file": 349, + "generate vanilla proof for a sector": 350, + "host address and port the gui will listen on": 351, + "host address and port the worker api will listen on": 352, + "indicates that deal index should not be announced to the IPNI": 353, + "initialize the path first": 354, + "layer name": 355, + "limit to log system": 356, + "list local storage paths": 357, + "list of layers to be interpreted (atop defaults). Default: base": 358, + "location of boost.db file": 359, + "lotus wallet export
| xxd -r -p | jq -r '.PrivateKey' | base64 -d | xxd -p -c 32": 360, + "lotus wallet new delegated": 361, + "machine host:port (curio run --listen address)": 362, + "manage open file limit": 363, + "manage sector storage": 364, + "maximum fee in FIL user is willing to pay for this message": 365, + "only list local storage paths": 366, + "only run init, then return": 367, + "output in json format": 368, + "partition to compute WindowPoSt for": 369, + "path group names": 370, + "path groups allowed to pull data from this path (allow all if not specified)": 371, + "piece-cid": 372, + "redeclare sectors in a local storage path": 373, + "restart via local pid file instead of the RPC API": 374, + "save the whole config into the layer, not just the diff": 375, + "source config layer": 376, + "start epoch by when the deal should be proved by provider on-chain (default: 2 days from now)": 377, + "start sealing a deal sector early": 378, + "title of the config layer (req'd for stdin)": 379, + "true if --source is set": 380, + "use color in display output": 381, + "yugabyte hosts to connect to over cassandra interface eg '127.0.0.1'": 382, + "yugabyte password to connect to over cassandra interface eg 'cassandra'": 383, + "yugabyte username to connect to over cassandra interface eg 'cassandra'": 384, } -var enIndex = []uint32{ // 384 elements +var enIndex = []uint32{ // 386 elements // Entry 0 - 1F 0x00000000, 0x00000029, 0x0000005c, 0x000000a9, 0x000000c0, 0x000000ea, 0x0000010a, 0x00000168, @@ -464,79 +466,81 @@ var enIndex = []uint32{ // 384 elements 0x000016b7, 0x0000172e, 0x00001752, 0x00001862, // Entry 80 - 9F 0x00001876, 0x000018cb, 0x000018ef, 0x000018f7, - 0x00001903, 0x00001922, 0x00001941, 0x0000194c, - 0x000019fc, 0x00001a23, 0x00001a47, 0x00001abe, - 0x00001ade, 0x00001b16, 0x00001b4a, 0x00001ba4, - 0x00001bbf, 0x00001bd9, 0x00001bfb, 0x00001c21, - 0x00001c64, 0x00001c75, 0x00001c8a, 0x00001ca9, - 0x00001cc9, 0x00001d12, 0x00001d32, 0x00001d41, - 0x00001d9a, 0x00001dc0, 0x00001ddc, 0x00001df1, + 0x00001922, 0x0000192e, 0x0000194d, 0x0000196c, + 0x00001977, 0x00001a27, 0x00001a4e, 0x00001a72, + 0x00001ae9, 0x00001b09, 0x00001b41, 0x00001b75, + 0x00001bcf, 0x00001bea, 0x00001c04, 0x00001c26, + 0x00001c4c, 0x00001c8f, 0x00001ca0, 0x00001cb5, + 0x00001cd4, 0x00001cf4, 0x00001d3d, 0x00001d5d, + 0x00001d6c, 0x00001dc5, 0x00001deb, 0x00001e07, // Entry A0 - BF - 0x00001dfc, 0x00001e1e, 0x00001e64, 0x00001e8a, - 0x00001eab, 0x00001ed5, 0x00001efa, 0x00001f50, - 0x00001f82, 0x00001fa5, 0x00001fbf, 0x00001fcb, - 0x00001fdf, 0x0000200f, 0x00002020, 0x00002031, - 0x0000205d, 0x00002075, 0x00002087, 0x00002091, - 0x0000209d, 0x000020c3, 0x000020f5, 0x00002114, - 0x000021e0, 0x00002205, 0x00002220, 0x00002271, - 0x000022b0, 0x000022b6, 0x000022d9, 0x000022ed, + 0x00001e1c, 0x00001e27, 0x00001e49, 0x00001e8f, + 0x00001eb5, 0x00001ed6, 0x00001f00, 0x00001f25, + 0x00001f7b, 0x00001fad, 0x00001fd0, 0x00001fea, + 0x00001ff6, 0x0000200a, 0x0000203a, 0x0000204b, + 0x0000205c, 0x00002088, 0x000020a0, 0x000020b2, + 0x000020bc, 0x000020c8, 0x000020ee, 0x00002120, + 0x0000213f, 0x0000220b, 0x00002230, 0x0000224b, + 0x0000229c, 0x000022db, 0x000022e1, 0x00002304, // Entry C0 - DF - 0x000022f1, 0x0000231e, 0x0000233e, 0x00002373, - 0x00002387, 0x0000239d, 0x000023b1, 0x000023d3, - 0x00002418, 0x00002428, 0x0000243c, 0x0000244f, - 0x000024e6, 0x0000255c, 0x0000259e, 0x000025df, - 0x000025eb, 0x0000260d, 0x00002627, 0x00002633, - 0x0000264f, 0x0000266b, 0x0000267a, 0x00002691, - 0x000026a9, 0x000026d9, 0x000026f1, 0x0000270d, - 0x0000272b, 0x0000273d, 0x0000277f, 0x00002794, + 0x00002318, 0x0000231c, 0x00002349, 0x00002369, + 0x0000239e, 0x000023b2, 0x000023c8, 0x000023dc, + 0x000023fe, 0x00002443, 0x00002453, 0x00002467, + 0x0000247a, 0x00002511, 0x00002587, 0x000025c9, + 0x0000260a, 0x00002616, 0x00002638, 0x00002652, + 0x0000265e, 0x0000267a, 0x00002696, 0x000026a5, + 0x000026bc, 0x000026d4, 0x00002704, 0x0000271c, + 0x00002738, 0x00002756, 0x00002768, 0x000027aa, // Entry E0 - FF - 0x000027a4, 0x000027cb, 0x000027f5, 0x00002812, - 0x00002848, 0x000029d3, 0x000029f3, 0x00002a1e, - 0x00002a31, 0x00002a55, 0x00002b78, 0x00002b8f, - 0x00002bc9, 0x00002bde, 0x00002bed, 0x00002c04, - 0x00002c45, 0x00002c53, 0x00002e51, 0x00002e7a, - 0x000031f7, 0x0000323f, 0x00003258, 0x000032a3, - 0x000033a3, 0x000033c6, 0x000033d5, 0x000033e9, - 0x00003403, 0x00003426, 0x00003459, 0x00003487, + 0x000027bf, 0x000027cf, 0x000027f6, 0x00002820, + 0x0000283d, 0x00002873, 0x000029fe, 0x00002a1e, + 0x00002a49, 0x00002a5c, 0x00002a80, 0x00002ba3, + 0x00002bba, 0x00002bf4, 0x00002c09, 0x00002c18, + 0x00002c2f, 0x00002c70, 0x00002c7e, 0x00002e7c, + 0x00002ea5, 0x00003222, 0x0000326a, 0x00003283, + 0x000032ce, 0x000033ce, 0x000033f1, 0x00003400, + 0x00003414, 0x0000342e, 0x00003451, 0x00003484, // Entry 100 - 11F - 0x000034a1, 0x000034b7, 0x00003536, 0x0000355c, - 0x00003579, 0x000035b4, 0x000035ce, 0x000035eb, - 0x000035f3, 0x00003609, 0x000038b9, 0x0000392d, - 0x0000395b, 0x0000396e, 0x00003983, 0x000039a2, - 0x000039b9, 0x000039e6, 0x00003a08, 0x00003a4e, - 0x00003ad3, 0x00003b3e, 0x00003ba0, 0x00003be2, - 0x00003c51, 0x00003d14, 0x00003d4f, 0x00003d88, - 0x00003de8, 0x00003e0e, 0x00003e4a, 0x00003e97, + 0x000034b2, 0x000034cc, 0x000034e2, 0x00003561, + 0x00003587, 0x000035a4, 0x000035df, 0x000035f9, + 0x00003616, 0x0000361e, 0x00003634, 0x000038e4, + 0x00003958, 0x00003986, 0x00003999, 0x000039ae, + 0x000039cd, 0x000039e4, 0x00003a11, 0x00003a33, + 0x00003a79, 0x00003afe, 0x00003b69, 0x00003bcb, + 0x00003c0d, 0x00003c7c, 0x00003d3f, 0x00003d7a, + 0x00003db3, 0x00003e13, 0x00003e39, 0x00003e75, // Entry 120 - 13F - 0x00003eb5, 0x00003ec8, 0x00003eee, 0x00003f14, - 0x00003f49, 0x00003f70, 0x00003fa5, 0x00003fc7, - 0x00003fdb, 0x00004010, 0x00004055, 0x00004065, - 0x0000408a, 0x000040a8, 0x000040cc, 0x000040f2, - 0x00004100, 0x00004118, 0x0000413a, 0x0000416a, - 0x00004185, 0x0000419a, 0x000041dc, 0x000041ea, - 0x000041f9, 0x00004239, 0x0000426c, 0x000042a9, - 0x000042fa, 0x00004335, 0x00004373, 0x000043a9, + 0x00003ec2, 0x00003ee0, 0x00003ef3, 0x00003f19, + 0x00003f3f, 0x00003f74, 0x00003f9b, 0x00003fd0, + 0x00003ff2, 0x00004006, 0x0000403b, 0x00004080, + 0x00004090, 0x000040b5, 0x000040d3, 0x000040f7, + 0x0000411d, 0x0000412b, 0x00004143, 0x00004165, + 0x00004195, 0x000041b0, 0x000041c5, 0x00004207, + 0x00004215, 0x00004224, 0x00004264, 0x00004297, + 0x000042d4, 0x00004325, 0x00004360, 0x0000439e, // Entry 140 - 15F - 0x000043f5, 0x0000440b, 0x0000441c, 0x00004429, - 0x00004431, 0x00004451, 0x00004458, 0x00004465, - 0x00004482, 0x00004496, 0x000044cd, 0x0000450e, - 0x00004528, 0x00004550, 0x000045a3, 0x000045ce, - 0x000045e9, 0x000045fa, 0x00004622, 0x00004640, - 0x0000465a, 0x00004678, 0x0000469a, 0x000046b7, - 0x000046d2, 0x000046e6, 0x000046f4, 0x0000472c, - 0x0000474e, 0x0000476f, 0x00004793, 0x000047c0, + 0x000043d4, 0x00004420, 0x00004436, 0x00004447, + 0x00004454, 0x0000445c, 0x0000447c, 0x00004483, + 0x00004490, 0x000044ad, 0x000044c1, 0x000044f8, + 0x00004539, 0x00004553, 0x0000457b, 0x000045ce, + 0x000045f9, 0x00004614, 0x00004625, 0x0000464d, + 0x0000466b, 0x00004685, 0x000046a3, 0x000046c5, + 0x000046e2, 0x000046fd, 0x00004711, 0x0000471f, + 0x00004757, 0x00004779, 0x0000479a, 0x000047be, // Entry 160 - 17F - 0x000047f4, 0x00004832, 0x0000484c, 0x00004857, - 0x0000486b, 0x00004884, 0x000048c4, 0x000048de, - 0x00004939, 0x00004954, 0x00004983, 0x0000499a, - 0x000049b0, 0x000049eb, 0x00004a09, 0x00004a24, - 0x00004a3a, 0x00004a5e, 0x00004a6f, 0x00004abc, - 0x00004ac6, 0x00004af0, 0x00004b28, 0x00004b3c, - 0x00004b9a, 0x00004bbc, 0x00004be8, 0x00004c00, - 0x00004c1c, 0x00004c61, 0x00004ca9, 0x00004cf1, -} // Size: 1560 bytes + 0x000047eb, 0x0000481f, 0x0000485d, 0x00004877, + 0x00004882, 0x00004896, 0x000048af, 0x000048ef, + 0x00004909, 0x00004964, 0x0000497f, 0x000049ae, + 0x000049c5, 0x000049db, 0x00004a16, 0x00004a34, + 0x00004a4f, 0x00004a65, 0x00004a89, 0x00004a9a, + 0x00004ae7, 0x00004af1, 0x00004b1b, 0x00004b4d, + 0x00004b85, 0x00004b99, 0x00004bf7, 0x00004c19, + 0x00004c45, 0x00004c5d, 0x00004c79, 0x00004cbe, + // Entry 180 - 19F + 0x00004d06, 0x00004d4e, +} // Size: 1568 bytes -const enData string = "" + // Size: 19697 bytes +const enData string = "" + // Size: 19790 bytes "\x04\x03 \x00\x22\x02Then export its private key with:\x02(debug tool)" + " Copy LM sector metadata into Curio DB\x02(for init) limit storage space" + " for sectors (expensive for very large paths!)\x02(for init) path weight" + @@ -632,209 +636,211 @@ const enData string = "" + // Size: 19697 bytes "used directly with SupraSeal binaries to test it without involving Curio" + ".\x02Get Curio node info\x02Get a config layer by name. You may want to " + "pipe the output to a file, or use 'less'\x02Get information about unseal" + - "ed data\x02Go Back\x02Host: %[1]s\x02How long to commit sectors for\x02H" + - "ow would you like to proceed?\x02I want to:\x02If, however, the upgrade " + - "has a serious bug and you need to downgrade, first shutdown all nodes in" + - " your cluster and then run this command. Finally, only start downgraded " + - "nodes.\x02Ignore sectors that cannot be migrated\x02Import delegated wal" + - "let private key\x02Increase reliability using redundancy: start multiple" + - " machines with at-least the post layer: 'curio run --layers=post'\x02Ini" + - "tializing a new miner actor.\x02Initiate a withdrawal request from the c" + - "lient's deposit\x02Initiate a withdrawal request from the service pool" + - "\x02Interpret stacked config layers by this version of curio, with syste" + - "m-generated comments.\x02Invalid private key: %[1]s\x04\x00\x01 \x15\x02" + - "Layer %[1]s created.\x02Limit output to the last N events\x02List config" + - " layers present in the DB.\x02List data from the sectors_unseal_pipeline" + - " and sectors_meta tables\x02List log systems\x02List pipeline events\x02" + - "Listen address for HTTP server\x02Lotus-Miner to Curio Migration.\x02Mak" + - "e sure to send FIL/tFIL to your 0x wallet address for PDP operations." + - "\x02Manage batch sealing operations\x02Manage logging\x02Manage node con" + - "fig by layers. The layer 'base' will always be applied at Curio start-up" + - ".\x02Manage storage paths for this server.\x02Manage the sealing pipelin" + - "e\x02Manage unsealed data\x02Math Utils\x02Migrate from existing Lotus-M" + - "iner\x02Migrating lotus-miner config.toml to Curio in-database configura" + - "tion.\x02Migrating metadata for %[1]d sectors.\x02Miner %[1]s created su" + - "ccessfully\x04\x00\x01 %\x02Miner creation error occurred: %[1]s\x02Mini" + - "mum number of hugepages required\x02Moves funds from the deal collateral" + - " wallet into escrow with the storage market actor\x02Name of the Postgre" + - "s database in Yugabyte cluster\x02New Miner initialization complete.\x02" + - "New service actor address\x02Next steps:\x02No address provided\x02No do" + - "main provided, skipping HTTP configuration\x02No host provided\x02No pat" + - "h provided\x04\x00\x01 '\x02No path provided, abandoning migration\x02No" + - " private key provided\x02No value provided\x02No, abort\x02No, keep it" + - "\x02Non-SP cluster configuration complete\x02Non-SP cluster configuratio" + - "n created successfully\x02Non-SP cluster setup complete!\x02Note: This c" + - "ommand is intended to be used to verify PoSt compute performance.\x0aIt " + - "will not send any messages to the chain. Since it can compute any deadli" + - "ne, output may be incorrectly timed for the chain.\x02Number of 1GB huge" + - "pages to configure\x02Number of sectors to start\x02One database can ser" + - "ve multiple miner IDs: Run a migration for each lotus-miner.\x02Optional" + - " setup steps (you can skip these and configure later):\x02Other\x02Outpu" + - "t file path (default: stdout)\x02Owner Wallet: %[1]s\x02PDP\x02PDP (Proo" + - "f of Data Possession) Configuration\x02PDP configuration layer created" + - "\x02PDP layer already exists. What would you like to do?\x02PDP setup co" + - "mplete!\x02PDP wallet configured\x02PDP wallet imported\x02PDP wallet im" + - "ported successfully!\x02Password for connecting to the Postgres database" + - " in Yugabyte cluster\x02Password: %[1]s\x02Path already exists\x02Path t" + - "o miner repo\x02Please do not run guided-setup again as miner creation i" + - "s not idempotent. You need to run 'curio config new-cluster %[1]s' to fi" + - "nish the configuration\x02Please do not run guided-setup again. You need" + - " to run 'curio config new-cluster' manually to finish the configuration" + - "\x02Port for connecting to the Cassandra database in Yugabyte cluster" + - "\x02Port for connecting to the Postgres database in Yugabyte cluster\x02" + - "Port: %[1]s\x02Pre-initialization steps complete\x02Print default node c" + - "onfig\x02Private key\x02Private key cannot be empty\x02Propose a new ser" + - "vice actor\x02Provider actor\x02Provider actor address\x02Provider sende" + - "r address\x02Provides a sample of CIDs from an indexed piece\x02Query th" + - "e service state\x02Query the state of a client\x02Query the state of a p" + - "rovider\x02Read Miner Config\x02Read piece CIDs one per line from stdin " + - "instead of from arguments\x02Really delete %[1]s?\x02Reconfigure PDP\x02" + - "Redeem a client voucher (service role)\x02Redeem a provider voucher (pro" + - "vider role)\x02Remove a named config layer.\x02Remove unsealed copies of" + - " sector containing this deal\x02Resolve each piece CID to sector(s) via " + - "market_piece_deal, then set target_unseal_state for those sectors.\x0a " + - " Accepts piece CID v1 or v2. Use --target-state to specify the desired s" + - "tate (true or false). Request all at once to minimize the sectors needed" + - " to be unsealed (if you have pieces stored in multiple sectors).\x0a U" + - "se --stdin to read piece CIDs one per line from stdin (for large lists)." + - "\x02SP ID to compute WindowPoSt for\x02Seal (fast storage for sealing op" + - "erations)\x02Sector Size: %[1]s\x04\x00\x01 \x1f\x02Sector selection fai" + - "led: %[1]s\x02Sectors can be stored across many filesystem paths. These" + - "\x0acommands provide ways to manage the storage a Curio node will use to" + - " store sectors\x0along term for proving (references as 'store') as well " + - "as how sectors will be\x0astored while moving through the sealing pipeli" + - "ne (references as 'seal').\x02Select the Sector Size\x02Select the locat" + - "ion of your lotus-miner config directory?\x02Sender Wallet: %[1]s\x02Sen" + - "der address\x02Service sender address\x02Set a config layer or the base " + - "by providing a filename or stdin.\x02Set log level\x04\x00\x01\x0a\xf8" + - "\x03\x02Set the log level for logging systems:\x0a\x0a The system flag" + - " can be specified multiple times.\x0a\x0a eg) log set-level --system c" + - "hain --system chainxchg debug\x0a\x0a Available Levels:\x0a debug" + - "\x0a info\x0a warn\x0a error\x0a\x0a Environment Variables:\x0a " + - " GOLOG_LOG_LEVEL - Default log level for all log systems\x0a GOLOG_LO" + - "G_FMT - Change output log format (json, nocolor)\x0a GOLOG_FILE " + - " - Write logs to file\x0a GOLOG_OUTPUT - Specify whether to output " + - "to file, stderr, stdout or a combination, i.e. file+stderr\x02Set the ta" + - "rget unseal state for a sector\x04\x00\x01\x0a\xf7\x06\x02Set the target" + - " unseal state for a specific sector.\x0a : The storage provi" + - "der ID\x0a : The sector number\x0a : Th" + - "e target state (true, false)\x0a\x0a The unseal target state indicates" + - " to curio how an unsealed copy of the sector should be maintained.\x0a" + - "\x09 If the target state is true, curio will ensure that the sector is" + - " unsealed.\x0a\x09 If the target state is false, curio will ensure tha" + - "t there is no unsealed copy of the sector.\x0a\x09 If the target state" + - " is none, curio will not change the current state of the sector.\x0a\x0a" + - " Currently when the curio will only start new unseal processes when th" + - "e target state changes from another state to true.\x0a\x0a When the ta" + - "rget state is false, and an unsealed sector file exists, the GC mark ste" + - "p will create a removal mark\x0a for the unsealed sector file. The fil" + - "e will only be removed after the removal mark is accepted.\x02Set the ta" + - "rget unseal state for sectors containing the given piece CIDs\x02Setting" + - " up PDP wallet...\x02Setup SPDK for batch sealing (configures hugepages " + - "and binds NVMe devices)\x02Setup SPDK for batch sealing operations.\x0a" + - "\x0aThis command automatically:\x0a- Downloads SPDK if not already avail" + - "able\x0a- Configures 1GB hugepages (36 pages minimum)\x0a- Binds NVMe de" + - "vices for use with SupraSeal\x0a\x0aRequires root/sudo access for SPDK s" + - "etup operations.\x02Setup non-Storage Provider cluster\x02Skip PDP setup" + - "\x02Skip optional steps\x02Skip wallet setup for now\x02Specify actor ad" + - "dress for the deal\x02Specify actor address to start sealing sectors for" + - "\x02Specify wallet address to send the funds from\x02Start Curio web int" + - "erface\x02Start a Curio process\x02Start an instance of Curio web interf" + - "ace. \x0a\x09This creates the 'web' layer if it does not exist, then cal" + - "ls run with that layer.\x02Start new sealing operations manually\x02Star" + - "t sealing new CC sectors\x02Start sealing sectors for all actors now (no" + - "t on schedule)\x04\x00\x01\x0a\x15\x02Step Complete: %[1]s\x02Stop a run" + - "ning Curio process\x02Storage\x02Storage Configuration\x04\x00\x04\x0a " + - " \xa7\x05\x02Storage can be attached to a Curio node using this command." + - " The storage volume\x0alist is stored local to the Curio node in storage" + - ".json set in curio run. We do not\x0arecommend manually modifying this v" + - "alue without further understanding of the\x0astorage system.\x0a\x0aEach" + - " storage volume contains a configuration file which describes the\x0acap" + - "abilities of the volume. When the '--init' flag is provided, this file w" + - "ill\x0abe created using the additional flags.\x0a\x0aWeight\x0aA high we" + - "ight value means data will be more likely to be stored in this path\x0a" + - "\x0aSeal\x0aData for the sealing process will be stored here\x0a\x0aStor" + - "e\x0aFinalized sectors that will be moved here for long term storage and" + - " be proven\x0aover time\x02Storage path %[1]s added as %[2]s. You'll nee" + - "d to initialize it with: curio cli storage attach --init --%[3]s %[4]s" + - "\x02Storage path %[1]s removed from configuration\x02Storage path added" + - "\x02Storage path deleted\x02Storage paths for this server:\x02Storage ty" + - "pe for %[1]s\x02Store (long-term storage for sealed sectors)\x02Supra co" + - "nsensus testing utilities\x02Target state: true (ensure unsealed), false" + - " (ensure no unsealed copy)\x02Test the windowpost scheduler by running i" + - "t on the next available curio. If tasks fail all retries, you will need " + - "to ctrl+c to exit.\x02The '%[1]s' layer stores common configuration. All" + - " curio instances can include it in their %[2]s argument.\x02This interac" + - "tive tool creates a new miner actor and creates the basic configuration " + - "layer for it.\x02This interactive tool migrates lotus-miner to Curio in " + - "5 minutes.\x02This interactive tool sets up a non-Storage Provider clust" + - "er for protocols like PDP, Snark market, and others.\x02This process is " + - "partially idempotent. Once a new miner actor has been created and subseq" + - "uent steps fail, the user need to run 'curio config new-cluster < miner " + - "ID >' to finish the configuration.\x02This setup does not create or migr" + - "ate a Filecoin SP actor.\x02This will configure PDP settings for your Cu" + - "rio cluster.\x02To run Curio: With machine or cgroup isolation, use the " + - "command (with example layer selection):\x02To start Curio with PDP enabl" + - "ed, run:\x02To start the cluster, run: curio run --layers basic-cluster" + - "\x02To start, ensure your sealing pipeline is drained and shut-down lotu" + - "s-miner.\x04\x00\x01 \x19\x02To work with the config:\x02Tool Box for Cu" + - "rio\x04\x00\x01 !\x02Try the web interface with %[1]s\x02Uncordon a mach" + - "ine, resume scheduling\x02Unmigratable sectors found. Do you want to con" + - "tinue?\x02Update data for messages in wait queue\x02Updated DB with mess" + - "age data missing from chain node\x02Use existing key, ending in %[1]s" + - "\x02Use synthetic PoRep\x04\x00\x01 0\x02Use the arrow keys to navigate:" + - " ↓ ↑ → ←\x02Username for connecting to the Postgres database in Yugabyte" + - " cluster\x02Username: %[1]s\x02Using existing PDP wallet key: %[1]s\x02U" + - "tility functions for testing\x02Validate a client voucher signature\x02V" + - "alidate a provider voucher signature\x02Voucher nonce\x02Voucher signatu" + - "re (hex)\x02Wait for Curio api to come online\x02Where should we save yo" + - "ur database config file?\x02Withdrawal amount (in FIL)\x02Worker Wallet:" + - " %[1]s\x02YYYYMMDD when your cluster had the preferred schema. Ex: 20251" + - "128\x02Yes, continue\x02Yes, delete it\x02You can add other layers for p" + - "er-machine configuration changes.\x02You can create a new delegated wall" + - "et using Lotus:\x02You can now migrate your market node (%[1]s), if appl" + - "icable.\x02You can now start using Curio for protocols like PDP, Snark m" + - "arkets, and others.\x02You can set up the wallet later using the Curio G" + - "UI or CLI\x02You need a delegated Filecoin wallet address to use with PD" + - "P.\x02Your non-SP cluster has been configured successfully.\x02Zen3 and " + - "later supports two sectors per thread, set to false for older CPUs\x02[S" + - "P actor address...]\x02[deadline index]\x02[layer name]\x02[level]\x02[m" + - "iner address] [sector number]\x02[path]\x02[sectorSize]\x02`URL` to send" + - " the request to\x02a layer's file name\x02a list of layers to be interpr" + - "eted as the final config\x02allow overwrite of existing layer if source " + - "is a different layer\x02attach local storage path\x02check how many entr" + - "ies need to be fixed\x02comma or space separated list of layers to be in" + - "terpreted (base is always applied)\x02could not get API info for FullNod" + - "e: %[1]w\x02curio run --layers=gui,pdp\x02custom node name\x04\x00\x01 #" + - "\x02deadline to compute WindowPoSt for\x02depends on output being a TTY" + - "\x02detach local storage path\x02do not interpret source layer\x02don't " + - "check full-node sync status\x02don't comment default values\x02duration " + - "to wait till fail\x02edit a config layer\x02editor to use\x02enables ver" + - "y verbose mode, useful for debugging the CLI\x02find sector in the stora" + - "ge system\x02generate ipni chunks from a file\x02generate vanilla proof " + - "for a sector\x02host address and port the gui will listen on\x02host add" + - "ress and port the worker api will listen on\x02indicates that deal index" + - " should not be announced to the IPNI\x02initialize the path first\x02lay" + - "er name\x02limit to log system\x02list local storage paths\x02list of la" + - "yers to be interpreted (atop defaults). Default: base\x02location of boo" + - "st.db file\x02lotus wallet export
| xxd -r -p | jq -r '.Privat" + - "eKey' | base64 -d | xxd -p -c 32\x02lotus wallet new delegated\x02machin" + - "e host:port (curio run --listen address)\x02manage open file limit\x02ma" + - "nage sector storage\x02maximum fee in FIL user is willing to pay for thi" + - "s message\x02only list local storage paths\x02only run init, then return" + - "\x02output in json format\x02partition to compute WindowPoSt for\x02path" + - " group names\x02path groups allowed to pull data from this path (allow a" + - "ll if not specified)\x02piece-cid\x02redeclare sectors in a local storag" + - "e path\x02save the whole config into the layer, not just the diff\x02sou" + - "rce config layer\x02start epoch by when the deal should be proved by pro" + - "vider on-chain (default: 2 days from now)\x02start sealing a deal sector" + - " early\x02title of the config layer (req'd for stdin)\x02true if --sourc" + - "e is set\x02use color in display output\x02yugabyte hosts to connect to " + - "over cassandra interface eg '127.0.0.1'\x02yugabyte password to connect " + - "to over cassandra interface eg 'cassandra'\x02yugabyte username to conne" + - "ct to over cassandra interface eg 'cassandra'" + "ed data\x02Go Back\x02Gracefully restart a running Curio process\x02Host" + + ": %[1]s\x02How long to commit sectors for\x02How would you like to proce" + + "ed?\x02I want to:\x02If, however, the upgrade has a serious bug and you " + + "need to downgrade, first shutdown all nodes in your cluster and then run" + + " this command. Finally, only start downgraded nodes.\x02Ignore sectors t" + + "hat cannot be migrated\x02Import delegated wallet private key\x02Increas" + + "e reliability using redundancy: start multiple machines with at-least th" + + "e post layer: 'curio run --layers=post'\x02Initializing a new miner acto" + + "r.\x02Initiate a withdrawal request from the client's deposit\x02Initiat" + + "e a withdrawal request from the service pool\x02Interpret stacked config" + + " layers by this version of curio, with system-generated comments.\x02Inv" + + "alid private key: %[1]s\x04\x00\x01 \x15\x02Layer %[1]s created.\x02Limi" + + "t output to the last N events\x02List config layers present in the DB." + + "\x02List data from the sectors_unseal_pipeline and sectors_meta tables" + + "\x02List log systems\x02List pipeline events\x02Listen address for HTTP " + + "server\x02Lotus-Miner to Curio Migration.\x02Make sure to send FIL/tFIL " + + "to your 0x wallet address for PDP operations.\x02Manage batch sealing op" + + "erations\x02Manage logging\x02Manage node config by layers. The layer 'b" + + "ase' will always be applied at Curio start-up.\x02Manage storage paths f" + + "or this server.\x02Manage the sealing pipeline\x02Manage unsealed data" + + "\x02Math Utils\x02Migrate from existing Lotus-Miner\x02Migrating lotus-m" + + "iner config.toml to Curio in-database configuration.\x02Migrating metada" + + "ta for %[1]d sectors.\x02Miner %[1]s created successfully\x04\x00\x01 %" + + "\x02Miner creation error occurred: %[1]s\x02Minimum number of hugepages " + + "required\x02Moves funds from the deal collateral wallet into escrow with" + + " the storage market actor\x02Name of the Postgres database in Yugabyte c" + + "luster\x02New Miner initialization complete.\x02New service actor addres" + + "s\x02Next steps:\x02No address provided\x02No domain provided, skipping " + + "HTTP configuration\x02No host provided\x02No path provided\x04\x00\x01 '" + + "\x02No path provided, abandoning migration\x02No private key provided" + + "\x02No value provided\x02No, abort\x02No, keep it\x02Non-SP cluster conf" + + "iguration complete\x02Non-SP cluster configuration created successfully" + + "\x02Non-SP cluster setup complete!\x02Note: This command is intended to " + + "be used to verify PoSt compute performance.\x0aIt will not send any mess" + + "ages to the chain. Since it can compute any deadline, output may be inco" + + "rrectly timed for the chain.\x02Number of 1GB hugepages to configure\x02" + + "Number of sectors to start\x02One database can serve multiple miner IDs:" + + " Run a migration for each lotus-miner.\x02Optional setup steps (you can " + + "skip these and configure later):\x02Other\x02Output file path (default: " + + "stdout)\x02Owner Wallet: %[1]s\x02PDP\x02PDP (Proof of Data Possession) " + + "Configuration\x02PDP configuration layer created\x02PDP layer already ex" + + "ists. What would you like to do?\x02PDP setup complete!\x02PDP wallet co" + + "nfigured\x02PDP wallet imported\x02PDP wallet imported successfully!\x02" + + "Password for connecting to the Postgres database in Yugabyte cluster\x02" + + "Password: %[1]s\x02Path already exists\x02Path to miner repo\x02Please d" + + "o not run guided-setup again as miner creation is not idempotent. You ne" + + "ed to run 'curio config new-cluster %[1]s' to finish the configuration" + + "\x02Please do not run guided-setup again. You need to run 'curio config " + + "new-cluster' manually to finish the configuration\x02Port for connecting" + + " to the Cassandra database in Yugabyte cluster\x02Port for connecting to" + + " the Postgres database in Yugabyte cluster\x02Port: %[1]s\x02Pre-initial" + + "ization steps complete\x02Print default node config\x02Private key\x02Pr" + + "ivate key cannot be empty\x02Propose a new service actor\x02Provider act" + + "or\x02Provider actor address\x02Provider sender address\x02Provides a sa" + + "mple of CIDs from an indexed piece\x02Query the service state\x02Query t" + + "he state of a client\x02Query the state of a provider\x02Read Miner Conf" + + "ig\x02Read piece CIDs one per line from stdin instead of from arguments" + + "\x02Really delete %[1]s?\x02Reconfigure PDP\x02Redeem a client voucher (" + + "service role)\x02Redeem a provider voucher (provider role)\x02Remove a n" + + "amed config layer.\x02Remove unsealed copies of sector containing this d" + + "eal\x02Resolve each piece CID to sector(s) via market_piece_deal, then s" + + "et target_unseal_state for those sectors.\x0a Accepts piece CID v1 or " + + "v2. Use --target-state to specify the desired state (true or false). Req" + + "uest all at once to minimize the sectors needed to be unsealed (if you h" + + "ave pieces stored in multiple sectors).\x0a Use --stdin to read piece " + + "CIDs one per line from stdin (for large lists).\x02SP ID to compute Wind" + + "owPoSt for\x02Seal (fast storage for sealing operations)\x02Sector Size:" + + " %[1]s\x04\x00\x01 \x1f\x02Sector selection failed: %[1]s\x02Sectors can" + + " be stored across many filesystem paths. These\x0acommands provide ways " + + "to manage the storage a Curio node will use to store sectors\x0along ter" + + "m for proving (references as 'store') as well as how sectors will be\x0a" + + "stored while moving through the sealing pipeline (references as 'seal')." + + "\x02Select the Sector Size\x02Select the location of your lotus-miner co" + + "nfig directory?\x02Sender Wallet: %[1]s\x02Sender address\x02Service sen" + + "der address\x02Set a config layer or the base by providing a filename or" + + " stdin.\x02Set log level\x04\x00\x01\x0a\xf8\x03\x02Set the log level fo" + + "r logging systems:\x0a\x0a The system flag can be specified multiple t" + + "imes.\x0a\x0a eg) log set-level --system chain --system chainxchg debu" + + "g\x0a\x0a Available Levels:\x0a debug\x0a info\x0a warn\x0a er" + + "ror\x0a\x0a Environment Variables:\x0a GOLOG_LOG_LEVEL - Default log" + + " level for all log systems\x0a GOLOG_LOG_FMT - Change output log for" + + "mat (json, nocolor)\x0a GOLOG_FILE - Write logs to file\x0a GOL" + + "OG_OUTPUT - Specify whether to output to file, stderr, stdout or a co" + + "mbination, i.e. file+stderr\x02Set the target unseal state for a sector" + + "\x04\x00\x01\x0a\xf7\x06\x02Set the target unseal state for a specific s" + + "ector.\x0a : The storage provider ID\x0a : " + + "The sector number\x0a : The target state (true, false)" + + "\x0a\x0a The unseal target state indicates to curio how an unsealed co" + + "py of the sector should be maintained.\x0a\x09 If the target state is " + + "true, curio will ensure that the sector is unsealed.\x0a\x09 If the ta" + + "rget state is false, curio will ensure that there is no unsealed copy of" + + " the sector.\x0a\x09 If the target state is none, curio will not chang" + + "e the current state of the sector.\x0a\x0a Currently when the curio wi" + + "ll only start new unseal processes when the target state changes from an" + + "other state to true.\x0a\x0a When the target state is false, and an un" + + "sealed sector file exists, the GC mark step will create a removal mark" + + "\x0a for the unsealed sector file. The file will only be removed after" + + " the removal mark is accepted.\x02Set the target unseal state for sector" + + "s containing the given piece CIDs\x02Setting up PDP wallet...\x02Setup S" + + "PDK for batch sealing (configures hugepages and binds NVMe devices)\x02S" + + "etup SPDK for batch sealing operations.\x0a\x0aThis command automaticall" + + "y:\x0a- Downloads SPDK if not already available\x0a- Configures 1GB huge" + + "pages (36 pages minimum)\x0a- Binds NVMe devices for use with SupraSeal" + + "\x0a\x0aRequires root/sudo access for SPDK setup operations.\x02Setup no" + + "n-Storage Provider cluster\x02Skip PDP setup\x02Skip optional steps\x02S" + + "kip wallet setup for now\x02Specify actor address for the deal\x02Specif" + + "y actor address to start sealing sectors for\x02Specify wallet address t" + + "o send the funds from\x02Start Curio web interface\x02Start a Curio proc" + + "ess\x02Start an instance of Curio web interface. \x0a\x09This creates th" + + "e 'web' layer if it does not exist, then calls run with that layer.\x02S" + + "tart new sealing operations manually\x02Start sealing new CC sectors\x02" + + "Start sealing sectors for all actors now (not on schedule)\x04\x00\x01" + + "\x0a\x15\x02Step Complete: %[1]s\x02Stop a running Curio process\x02Stor" + + "age\x02Storage Configuration\x04\x00\x04\x0a \xa7\x05\x02Storage can b" + + "e attached to a Curio node using this command. The storage volume\x0alis" + + "t is stored local to the Curio node in storage.json set in curio run. We" + + " do not\x0arecommend manually modifying this value without further under" + + "standing of the\x0astorage system.\x0a\x0aEach storage volume contains a" + + " configuration file which describes the\x0acapabilities of the volume. W" + + "hen the '--init' flag is provided, this file will\x0abe created using th" + + "e additional flags.\x0a\x0aWeight\x0aA high weight value means data will" + + " be more likely to be stored in this path\x0a\x0aSeal\x0aData for the se" + + "aling process will be stored here\x0a\x0aStore\x0aFinalized sectors that" + + " will be moved here for long term storage and be proven\x0aover time\x02" + + "Storage path %[1]s added as %[2]s. You'll need to initialize it with: cu" + + "rio cli storage attach --init --%[3]s %[4]s\x02Storage path %[1]s remove" + + "d from configuration\x02Storage path added\x02Storage path deleted\x02St" + + "orage paths for this server:\x02Storage type for %[1]s\x02Store (long-te" + + "rm storage for sealed sectors)\x02Supra consensus testing utilities\x02T" + + "arget state: true (ensure unsealed), false (ensure no unsealed copy)\x02" + + "Test the windowpost scheduler by running it on the next available curio." + + " If tasks fail all retries, you will need to ctrl+c to exit.\x02The '%[1" + + "]s' layer stores common configuration. All curio instances can include i" + + "t in their %[2]s argument.\x02This interactive tool creates a new miner " + + "actor and creates the basic configuration layer for it.\x02This interact" + + "ive tool migrates lotus-miner to Curio in 5 minutes.\x02This interactive" + + " tool sets up a non-Storage Provider cluster for protocols like PDP, Sna" + + "rk market, and others.\x02This process is partially idempotent. Once a n" + + "ew miner actor has been created and subsequent steps fail, the user need" + + " to run 'curio config new-cluster < miner ID >' to finish the configurat" + + "ion.\x02This setup does not create or migrate a Filecoin SP actor.\x02Th" + + "is will configure PDP settings for your Curio cluster.\x02To run Curio: " + + "With machine or cgroup isolation, use the command (with example layer se" + + "lection):\x02To start Curio with PDP enabled, run:\x02To start the clust" + + "er, run: curio run --layers basic-cluster\x02To start, ensure your seali" + + "ng pipeline is drained and shut-down lotus-miner.\x04\x00\x01 \x19\x02To" + + " work with the config:\x02Tool Box for Curio\x04\x00\x01 !\x02Try the we" + + "b interface with %[1]s\x02Uncordon a machine, resume scheduling\x02Unmig" + + "ratable sectors found. Do you want to continue?\x02Update data for messa" + + "ges in wait queue\x02Updated DB with message data missing from chain nod" + + "e\x02Use existing key, ending in %[1]s\x02Use synthetic PoRep\x04\x00" + + "\x01 0\x02Use the arrow keys to navigate: ↓ ↑ → ←\x02Username for connec" + + "ting to the Postgres database in Yugabyte cluster\x02Username: %[1]s\x02" + + "Using existing PDP wallet key: %[1]s\x02Utility functions for testing" + + "\x02Validate a client voucher signature\x02Validate a provider voucher s" + + "ignature\x02Voucher nonce\x02Voucher signature (hex)\x02Wait for Curio a" + + "pi to come online\x02Where should we save your database config file?\x02" + + "Withdrawal amount (in FIL)\x02Worker Wallet: %[1]s\x02YYYYMMDD when your" + + " cluster had the preferred schema. Ex: 20251128\x02Yes, continue\x02Yes," + + " delete it\x02You can add other layers for per-machine configuration cha" + + "nges.\x02You can create a new delegated wallet using Lotus:\x02You can n" + + "ow migrate your market node (%[1]s), if applicable.\x02You can now start" + + " using Curio for protocols like PDP, Snark markets, and others.\x02You c" + + "an set up the wallet later using the Curio GUI or CLI\x02You need a dele" + + "gated Filecoin wallet address to use with PDP.\x02Your non-SP cluster ha" + + "s been configured successfully.\x02Zen3 and later supports two sectors p" + + "er thread, set to false for older CPUs\x02[SP actor address...]\x02[dead" + + "line index]\x02[layer name]\x02[level]\x02[miner address] [sector number" + + "]\x02[path]\x02[sectorSize]\x02`URL` to send the request to\x02a layer's" + + " file name\x02a list of layers to be interpreted as the final config\x02" + + "allow overwrite of existing layer if source is a different layer\x02atta" + + "ch local storage path\x02check how many entries need to be fixed\x02comm" + + "a or space separated list of layers to be interpreted (base is always ap" + + "plied)\x02could not get API info for FullNode: %[1]w\x02curio run --laye" + + "rs=gui,pdp\x02custom node name\x04\x00\x01 #\x02deadline to compute Wind" + + "owPoSt for\x02depends on output being a TTY\x02detach local storage path" + + "\x02do not interpret source layer\x02don't check full-node sync status" + + "\x02don't comment default values\x02duration to wait till fail\x02edit a" + + " config layer\x02editor to use\x02enables very verbose mode, useful for " + + "debugging the CLI\x02find sector in the storage system\x02generate ipni " + + "chunks from a file\x02generate vanilla proof for a sector\x02host addres" + + "s and port the gui will listen on\x02host address and port the worker ap" + + "i will listen on\x02indicates that deal index should not be announced to" + + " the IPNI\x02initialize the path first\x02layer name\x02limit to log sys" + + "tem\x02list local storage paths\x02list of layers to be interpreted (ato" + + "p defaults). Default: base\x02location of boost.db file\x02lotus wallet " + + "export
| xxd -r -p | jq -r '.PrivateKey' | base64 -d | xxd -p " + + "-c 32\x02lotus wallet new delegated\x02machine host:port (curio run --li" + + "sten address)\x02manage open file limit\x02manage sector storage\x02maxi" + + "mum fee in FIL user is willing to pay for this message\x02only list loca" + + "l storage paths\x02only run init, then return\x02output in json format" + + "\x02partition to compute WindowPoSt for\x02path group names\x02path grou" + + "ps allowed to pull data from this path (allow all if not specified)\x02p" + + "iece-cid\x02redeclare sectors in a local storage path\x02restart via loc" + + "al pid file instead of the RPC API\x02save the whole config into the lay" + + "er, not just the diff\x02source config layer\x02start epoch by when the " + + "deal should be proved by provider on-chain (default: 2 days from now)" + + "\x02start sealing a deal sector early\x02title of the config layer (req'" + + "d for stdin)\x02true if --source is set\x02use color in display output" + + "\x02yugabyte hosts to connect to over cassandra interface eg '127.0.0.1'" + + "\x02yugabyte password to connect to over cassandra interface eg 'cassand" + + "ra'\x02yugabyte username to connect to over cassandra interface eg 'cass" + + "andra'" -var koIndex = []uint32{ // 384 elements +var koIndex = []uint32{ // 386 elements // Entry 0 - 1F 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, @@ -943,11 +949,13 @@ var koIndex = []uint32{ // 384 elements 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, -} // Size: 1560 bytes + // Entry 180 - 19F + 0x00000000, 0x00000000, +} // Size: 1568 bytes const koData string = "" -var zhIndex = []uint32{ // 384 elements +var zhIndex = []uint32{ // 386 elements // Entry 0 - 1F 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, @@ -1056,8 +1064,10 @@ var zhIndex = []uint32{ // 384 elements 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, -} // Size: 1560 bytes + // Entry 180 - 19F + 0x00000000, 0x00000000, +} // Size: 1568 bytes const zhData string = "" -// Total table size 24377 bytes (23KiB); checksum: 5591ED58 +// Total table size 24494 bytes (23KiB); checksum: 17B7D3 diff --git a/cmd/curio/internal/translations/locales/en/out.gotext.json b/cmd/curio/internal/translations/locales/en/out.gotext.json index 741c1e16d..11e70ea77 100644 --- a/cmd/curio/internal/translations/locales/en/out.gotext.json +++ b/cmd/curio/internal/translations/locales/en/out.gotext.json @@ -1372,6 +1372,13 @@ "translatorComment": "Copied from source.", "fuzzy": true }, + { + "id": "Gracefully restart a running Curio process", + "message": "Gracefully restart a running Curio process", + "translation": "Gracefully restart a running Curio process", + "translatorComment": "Copied from source.", + "fuzzy": true + }, { "id": "Host: {Join}", "message": "Host: {Join}", @@ -3358,6 +3365,13 @@ "translatorComment": "Copied from source.", "fuzzy": true }, + { + "id": "restart via local pid file instead of the RPC API", + "message": "restart via local pid file instead of the RPC API", + "translation": "restart via local pid file instead of the RPC API", + "translatorComment": "Copied from source.", + "fuzzy": true + }, { "id": "save the whole config into the layer, not just the diff", "message": "save the whole config into the layer, not just the diff", diff --git a/cmd/curio/internal/translations/locales/ko/messages.gotext.json b/cmd/curio/internal/translations/locales/ko/messages.gotext.json index bec455234..c25d6088f 100644 --- a/cmd/curio/internal/translations/locales/ko/messages.gotext.json +++ b/cmd/curio/internal/translations/locales/ko/messages.gotext.json @@ -3159,6 +3159,18 @@ "translation": "목표 상태: true (봉인 해제 보장), false (봉인 해제 사본 없음 보장)", "message": "Target state: true (ensure unsealed), false (ensure no unsealed copy)", "placeholder": null + }, + { + "id": "Gracefully restart a running Curio process", + "translation": "실행 중인 Curio 프로세스 정상 재시작", + "message": "Gracefully restart a running Curio process", + "placeholder": null + }, + { + "id": "restart via local pid file instead of the RPC API", + "translation": "RPC API 대신 로컬 pid 파일로 재시작", + "message": "restart via local pid file instead of the RPC API", + "placeholder": null } ] } \ No newline at end of file diff --git a/cmd/curio/internal/translations/locales/zh/messages.gotext.json b/cmd/curio/internal/translations/locales/zh/messages.gotext.json index 41b9cff89..aff3c038e 100644 --- a/cmd/curio/internal/translations/locales/zh/messages.gotext.json +++ b/cmd/curio/internal/translations/locales/zh/messages.gotext.json @@ -3129,6 +3129,18 @@ "translation": "目标状态：true（确保已解封）、false（确保无解封副本）", "message": "Target state: true (ensure unsealed), false (ensure no unsealed copy)", "placeholder": null + }, + { + "id": "Gracefully restart a running Curio process", + "translation": "优雅重启正在运行的 Curio 进程", + "message": "Gracefully restart a running Curio process", + "placeholder": null + }, + { + "id": "restart via local pid file instead of the RPC API", + "translation": "通过本地 pid 文件而非 RPC API 重启", + "message": "restart via local pid file instead of the RPC API", + "placeholder": null } ] } \ No newline at end of file diff --git a/cmd/curio/main.go b/cmd/curio/main.go index c428dc02d..9f35670d6 100644 --- a/cmd/curio/main.go +++ b/cmd/curio/main.go @@ -46,7 +46,8 @@ func SetupLogLevels() { func setupCloseHandler() { c := make(chan os.Signal, 1) // Register our handler first, before runApp's handler - signal.Notify(c, os.Interrupt, syscall.SIGTERM, syscall.SIGABRT) + // SIGTERM/SIGINT are handled by gracehttp during `curio run`. + signal.Notify(c, os.Interrupt, syscall.SIGABRT) go func() { <-c fmt.Println("\r- Ctrl+C pressed in Terminal") diff --git a/cmd/curio/restart.go b/cmd/curio/restart.go new file mode 100644 index 000000000..394c9d43f --- /dev/null +++ b/cmd/curio/restart.go @@ -0,0 +1,43 @@ +package main + +import ( + _ "net/http/pprof" + "path/filepath" + + "github.com/mitchellh/go-homedir" + "github.com/urfave/cli/v2" + + "github.com/filecoin-project/curio/cmd/curio/internal/translations" + "github.com/filecoin-project/curio/cmd/curio/rpc" + "github.com/filecoin-project/curio/deps" + "github.com/filecoin-project/curio/lib/gracehttpsvc" + "github.com/filecoin-project/curio/lib/reqcontext" +) + +var restartCmd = &cli.Command{ + Name: "restart", + Usage: translations.T("Gracefully restart a running Curio process"), + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "local", + Usage: translations.T("restart via local pid file instead of the RPC API"), + }, + }, + Action: func(cctx *cli.Context) error { + if cctx.Bool("local") { + repoPath, err := homedir.Expand(cctx.String(deps.FlagRepoPath)) + if err != nil { + return err + } + return gracehttpsvc.RestartFromPIDFile(filepath.Join(repoPath, "curio.pid")) + } + + api, closer, err := rpc.GetCurioAPI(cctx) + if err != nil { + return err + } + defer closer() + + return api.Restart(reqcontext.ReqContext(cctx)) + }, +} diff --git a/cmd/curio/rpc/rpc.go b/cmd/curio/rpc/rpc.go index 7370a81da..9bcda7499 100644 --- a/cmd/curio/rpc/rpc.go +++ b/cmd/curio/rpc/rpc.go @@ -24,7 +24,6 @@ import ( "github.com/multiformats/go-multihash" "github.com/urfave/cli/v2" "go.opencensus.io/tag" - "golang.org/x/sync/errgroup" "golang.org/x/xerrors" ffi "github.com/filecoin-project/filecoin-ffi" @@ -39,6 +38,7 @@ import ( ltypes "github.com/filecoin-project/curio/api/types" "github.com/filecoin-project/curio/build" "github.com/filecoin-project/curio/deps" + "github.com/filecoin-project/curio/lib/gracehttpsvc" "github.com/filecoin-project/curio/lib/metrics" "github.com/filecoin-project/curio/lib/paths" "github.com/filecoin-project/curio/lib/repo" @@ -97,10 +97,17 @@ func CurioHandler( return ah } +// TaskEngineRestarter begins an in-memory graceful restart: it cordons the +// node (without persisting to the DB), drains running tasks, then restarts. +type TaskEngineRestarter interface { + RequestRestart() +} + type CurioAPI struct { *deps.Deps paths.SectorIndex ShutdownChan chan struct{} + restarter TaskEngineRestarter } func (p *CurioAPI) Version(context.Context) ([]int, error) { @@ -217,8 +224,23 @@ func (p *CurioAPI) AllocatePieceToSector(ctx context.Context, maddr address.Addr // Trigger shutdown func (p *CurioAPI) Shutdown(context.Context) error { - close(p.ShutdownChan) - return nil + select { + case <-p.ShutdownChan: + default: + close(p.ShutdownChan) + } + return gracehttpsvc.TriggerShutdown() +} + +// Restart initiates a graceful restart. When a task engine is wired in, it +// cordons the node in-memory and drains running tasks before triggering the +// zero-downtime restart; otherwise it triggers the restart immediately. +func (p *CurioAPI) Restart(context.Context) error { + if p.restarter != nil { + p.restarter.RequestRestart() + return nil + } + return gracehttpsvc.TriggerRestart() } func (p *CurioAPI) StorageInit(ctx context.Context, path string, opts storiface.LocalStorageMeta) error { @@ -447,7 +469,7 @@ func (p *CurioAPI) IndexSamples(ctx context.Context, pcid cid.Cid) ([]multihash. return p.IndexStore.GetPieceHashRange(ctx, pcid, firstHash, chunk.NumberOfBlocks, true) } -func ListenAndServe(ctx context.Context, dependencies *deps.Deps, shutdownChan chan struct{}) error { +func ListenAndServe(ctx context.Context, dependencies *deps.Deps, shutdownChan chan struct{}, restarter TaskEngineRestarter, extraServers ...*http.Server) error { fh := &paths.FetchHandler{Local: dependencies.LocalStore, PfHandler: &paths.DefaultPartialFileHandler{}} remoteHandler := func(w http.ResponseWriter, r *http.Request) { if !auth.HasPerm(r.Context(), nil, lapi.PermAdmin) { @@ -479,7 +501,7 @@ func ListenAndServe(ctx context.Context, dependencies *deps.Deps, shutdownChan c Handler: CurioHandler( authVerify, remoteHandler, - &CurioAPI{dependencies, dependencies.Si, shutdownChan}, + &CurioAPI{dependencies, dependencies.Si, shutdownChan, restarter}, prometheusServiceDiscovery(ctx, dependencies), permissioned), ReadHeaderTimeout: time.Minute * 3, @@ -491,27 +513,14 @@ func ListenAndServe(ctx context.Context, dependencies *deps.Deps, shutdownChan c } log.Infof("Setting up RPC server at %s", dependencies.ListenAddr) - eg := errgroup.Group{} - eg.Go(srv.ListenAndServe) + servers := []*http.Server{srv} if dependencies.Cfg.Subsystems.EnableWebGui { - web, err := web.GetSrv(ctx, dependencies, false) + webSrv, err := web.GetSrv(ctx, dependencies, false) if err != nil { return err } - go func() { - <-ctx.Done() - log.Warn("Shutting down...") - if err := srv.Shutdown(context.TODO()); err != nil { - log.Errorf("shutting down RPC server failed: %s", err) - } - if err := web.Shutdown(context.Background()); err != nil { - log.Errorf("shutting down web server failed: %s", err) - } - log.Warn("Graceful shutdown successful") - }() - uiAddress := dependencies.Cfg.Subsystems.GuiAddress if uiAddress == "" || uiAddress[0] == ':' || uiAddress == "0.0.0.0:4701" { split := strings.Split(uiAddress, ":") @@ -521,9 +530,15 @@ func ListenAndServe(ctx context.Context, dependencies *deps.Deps, shutdownChan c } log.Infof("GUI: http://%s", uiAddress) - eg.Go(web.ListenAndServe) + servers = append(servers, webSrv) } - return eg.Wait() + for _, s := range extraServers { + if s != nil { + servers = append(servers, s) + } + } + + return gracehttpsvc.Serve(servers...) } func GetCurioAPI(ctx *cli.Context) (api.Curio, jsonrpc.ClientCloser, error) { diff --git a/cmd/curio/run.go b/cmd/curio/run.go index 9d09fa6bf..f6926482e 100644 --- a/cmd/curio/run.go +++ b/cmd/curio/run.go @@ -3,10 +3,12 @@ package main import ( "context" "fmt" + "net/http" "os" "path/filepath" "strings" + "github.com/mitchellh/go-homedir" "github.com/pkg/errors" "github.com/urfave/cli/v2" "go.opencensus.io/stats" @@ -16,8 +18,8 @@ import ( "github.com/filecoin-project/curio/cmd/curio/rpc" "github.com/filecoin-project/curio/cmd/curio/tasks" "github.com/filecoin-project/curio/deps" + "github.com/filecoin-project/curio/lib/gracehttpsvc" curiometrics "github.com/filecoin-project/curio/lib/metrics" - "github.com/filecoin-project/curio/lib/shutdown" "github.com/filecoin-project/lotus/lib/ulimit" "github.com/filecoin-project/lotus/metrics" @@ -90,6 +92,28 @@ var runCmd = &cli.Command{ log.Errorf("ensuring tempdir exists: %s", err) } + repoPath, err := homedir.Expand(cctx.String(deps.FlagRepoPath)) + if err != nil { + return xerrors.Errorf("expanding repo path: %w", err) + } + pidPath := filepath.Join(repoPath, "curio.pid") + + if shouldCheckAlreadyRunning(cctx) { + restarted, err := gracehttpsvc.RestartIfAlreadyRunning(pidPath) + if err != nil { + return xerrors.Errorf("checking for running curio: %w", err) + } + if restarted { + log.Info("Curio is already running; graceful restart triggered") + return nil + } + } + + if err := gracehttpsvc.WritePIDFile(pidPath); err != nil { + return xerrors.Errorf("writing pid file: %w", err) + } + defer gracehttpsvc.RemovePIDFile(pidPath) + if os.Getenv("GOLOG_FILE") != "" { err := os.MkdirAll(filepath.Dir(os.Getenv("GOLOG_FILE")), 0755) if err != nil { @@ -128,26 +152,44 @@ var runCmd = &cli.Command{ go ffiSelfTest() // Panics on failure - taskEngine, err := tasks.StartTasks(ctx, dependencies, shutdownChan) - + taskEngine, marketServer, err := tasks.StartTasks(ctx, dependencies, shutdownChan) if err != nil { return xerrors.Errorf("starting tasks: %w", err) } defer taskEngine.GracefullyTerminate() - err = rpc.ListenAndServe(ctx, dependencies, shutdownChan) // Monitor for shutdown. + gracehttpsvc.SetPreRestartHook(func() error { + taskEngine.GracefullyTerminate() + return nil + }) + + var extraServers []*http.Server + if marketServer != nil { + extraServers = append(extraServers, marketServer) + } + + err = rpc.ListenAndServe(ctx, dependencies, shutdownChan, taskEngine, extraServers...) if err != nil { return err } - finishCh := shutdown.MonitorShutdown(shutdownChan) //node.ShutdownHandler{Component: "rpc server", StopFunc: rpcStopper}, - //node.ShutdownHandler{Component: "curio", StopFunc: stop}, - - <-finishCh return nil }, } +func shouldCheckAlreadyRunning(cctx *cli.Context) bool { + if gracehttpsvc.IsGraceHandoff() { + return false + } + if cctx.Bool("halt-after-init") { + return false + } + if os.Getenv("CURIO_SKIP_ALREADY_RUNNING") != "" { + return false + } + return true +} + var layersFlag = &cli.StringSliceFlag{ Name: "layers", Usage: translations.T("list of layers to be interpreted (atop defaults). Default: base"), diff --git a/cmd/curio/tasks/tasks.go b/cmd/curio/tasks/tasks.go index 5cb75a9f8..f2e3a19e6 100644 --- a/cmd/curio/tasks/tasks.go +++ b/cmd/curio/tasks/tasks.go @@ -3,6 +3,7 @@ package tasks import ( "context" + "net/http" "os" "slices" "sort" @@ -94,7 +95,8 @@ func WindowPostScheduler(ctx context.Context, fc config.CurioFees, pc config.Cur return computeTask, submitTask, recoverTask, nil } -func StartTasks(ctx context.Context, dependencies *deps.Deps, shutdownChan chan struct{}) (*harmonytask.TaskEngine, error) { +func StartTasks(ctx context.Context, dependencies *deps.Deps, shutdownChan chan struct{}) (*harmonytask.TaskEngine, *http.Server, error) { + var marketServer *http.Server cfg := dependencies.Cfg db := dependencies.DB full := dependencies.Chain @@ -194,7 +196,7 @@ func StartTasks(ctx context.Context, dependencies *deps.Deps, shutdownChan chan as, maddrs, db, stor, si, cfg.Subsystems.WindowPostMaxTasks, cuzkClient) if err != nil { - return nil, err + return nil, nil, err } activeTasks = append(activeTasks, wdPostTask, wdPoStSubmitTask, derlareRecoverTask) } @@ -240,7 +242,7 @@ func StartTasks(ctx context.Context, dependencies *deps.Deps, shutdownChan chan if hasAnySealingTask { sealingTasks, p2a, err := addSealingTasks(ctx, hasAnySealingTask, db, full, sender, as, cfg, slrLazy, asyncParams, si, stor, bstore, machine, prover, cuzkClient) if err != nil { - return nil, err + return nil, nil, err } activeTasks = append(activeTasks, sealingTasks...) p2Active = p2a @@ -251,7 +253,7 @@ func StartTasks(ctx context.Context, dependencies *deps.Deps, shutdownChan chan if cfg.Subsystems.EnableParkPiece { parkPieceTask, err := piece2.NewParkPieceTask(db, must.One(slrLazy.Val()), stor, cfg.Subsystems.ParkPieceMaxTasks, cfg.Subsystems.ParkPieceMaxInPark, p2Active, cfg.Subsystems.ParkPieceMinFreeStoragePercent) if err != nil { - return nil, err + return nil, nil, err } cleanupPieceTask := piece2.NewCleanupPieceTask(db, must.One(slrLazy.Val()), 0) aggregateChunksTask := piece2.NewAggregateChunksTask(db, stor, must.One(slrLazy.Val())) @@ -279,7 +281,7 @@ func StartTasks(ctx context.Context, dependencies *deps.Deps, shutdownChan chan dm = storage_market.NewCurioStorageDealMarket(miners, db, cfg, must.One(dependencies.EthClient.Val()), si, full, as, must.One(slrLazy.Val())) err := dm.StartMarket(ctx) if err != nil { - return nil, err + return nil, nil, err } sdeps.DealMarket = dm @@ -306,7 +308,7 @@ func StartTasks(ctx context.Context, dependencies *deps.Deps, shutdownChan chan } sc, err := slrLazy.Val() if err != nil { - return nil, err + return nil, nil, err } // Enable PDP v1 subsystem @@ -385,9 +387,9 @@ func StartTasks(ctx context.Context, dependencies *deps.Deps, shutdownChan chan //if !cfg.Subsystems.EnableDealMarket { // return nil, xerrors.New("deal market must be enabled on HTTP server") //} - err = cuhttp.StartHTTPServer(ctx, dependencies, &sdeps) + marketServer, err = cuhttp.NewHTTPServer(ctx, dependencies, &sdeps) if err != nil { - return nil, xerrors.Errorf("failed to start the HTTP server: %w", err) + return nil, nil, xerrors.Errorf("failed to configure the HTTP server: %w", err) } } } @@ -403,7 +405,7 @@ func StartTasks(ctx context.Context, dependencies *deps.Deps, shutdownChan chan ht, err := harmonytask.New(db, activeTasks, dependencies.ListenAddr) if err != nil { - return nil, err + return nil, nil, err } go machineDetails(dependencies, activeTasks, ht.ResourcesAvailable().MachineID, dependencies.Name) @@ -412,7 +414,7 @@ func StartTasks(ctx context.Context, dependencies *deps.Deps, shutdownChan chan if hasAnySealingTask { watcher, err := message.NewMessageWatcher(db, ht, chainSched, full) if err != nil { - return nil, err + return nil, nil, err } _ = watcher } @@ -420,7 +422,7 @@ func StartTasks(ctx context.Context, dependencies *deps.Deps, shutdownChan chan if senderEth != nil { watcherEth, err := message.NewMessageWatcherEth(db, ht, chainSched, must.One(dependencies.EthClient.Val())) if err != nil { - return nil, err + return nil, nil, err } _ = watcherEth @@ -430,7 +432,7 @@ func StartTasks(ctx context.Context, dependencies *deps.Deps, shutdownChan chan go chainSched.Run(ctx) } - return ht, nil + return ht, marketServer, nil } func addSealingTasks( diff --git a/cuhttp/server.go b/cuhttp/server.go index 2421732e8..d0a367074 100644 --- a/cuhttp/server.go +++ b/cuhttp/server.go @@ -146,8 +146,9 @@ type ServiceDeps struct { AlertTask *alertmanager.AlertTask } -// This starts the public-facing server for market calls. -func StartHTTPServer(ctx context.Context, d *deps.Deps, sd *ServiceDeps) error { +// NewHTTPServer builds the public-facing server for market calls without +// starting it. The caller should pass the returned server to gracehttpsvc.Serve. +func NewHTTPServer(ctx context.Context, d *deps.Deps, sd *ServiceDeps) (*http.Server, error) { cfg := d.Cfg.HTTP // Setup the Chi router for more complex routing (if needed in the future) @@ -201,7 +202,7 @@ func StartHTTPServer(ctx context.Context, d *deps.Deps, sd *ServiceDeps) error { chiRouter, err = attachRouters(ctx, chiRouter, d, sd) if err != nil { - return xerrors.Errorf("failed to attach routers: %w", err) + return nil, xerrors.Errorf("failed to attach routers: %w", err) } // Set up the HTTP server with proper timeouts @@ -225,33 +226,13 @@ func StartHTTPServer(ctx context.Context, d *deps.Deps, sd *ServiceDeps) error { server.TLSConfig = certManager.TLSConfig() } - // We don't need to run an HTTP server. Any HTTP request should simply be handled as HTTPS. - - // Start the server with TLS - go func() { - log.Infof("Starting HTTPS server for https://%s on %s", cfg.DomainName, cfg.ListenAddress) - var serr error - if !cfg.DelegateTLS { - serr = server.ListenAndServeTLS("", "") - } else { - serr = server.ListenAndServe() - } - if serr != nil && !errors.Is(serr, http.ErrServerClosed) { - log.Errorf("Failed to start HTTPS server: %s", serr) - panic(serr) - } - }() - - go func() { - <-ctx.Done() - log.Warn("Shutting down HTTP Server...") - if err := server.Shutdown(context.Background()); err != nil { - log.Errorf("shutting down web server failed: %s", err) - } - log.Warn("HTTP Server graceful shutdown successful") - }() + if !cfg.DelegateTLS { + log.Infof("Configured HTTPS server for https://%s on %s", cfg.DomainName, cfg.ListenAddress) + } else { + log.Infof("Configured HTTP server (delegated TLS) on %s", cfg.ListenAddress) + } - return nil + return server, nil } type cache struct { diff --git a/documentation/en/curio-cli/curio.md b/documentation/en/curio-cli/curio.md index 52dfa0d63..7522d76fe 100644 --- a/documentation/en/curio-cli/curio.md +++ b/documentation/en/curio-cli/curio.md @@ -55,6 +55,7 @@ COMMANDS: log Manage logging wait-api Wait for Curio api to come online stop Stop a running Curio process + restart Gracefully restart a running Curio process cordon Cordon a machine, set it to maintenance mode uncordon Uncordon a machine, resume scheduling index-sample Provides a sample of CIDs from an indexed piece @@ -302,6 +303,19 @@ OPTIONS: --help, -h show help ``` +### curio cli restart +``` +NAME: + curio cli restart - Gracefully restart a running Curio process + +USAGE: + curio cli restart [command options] + +OPTIONS: + --local restart via local pid file instead of the RPC API (default: false) + --help, -h show help +``` + ### curio cli cordon ``` NAME: diff --git a/documentation/en/developer/api.md b/documentation/en/developer/api.md index fdcf9a6bf..ad00d678e 100644 --- a/documentation/en/developer/api.md +++ b/documentation/en/developer/api.md @@ -11,6 +11,7 @@ description: Curio API references * [DefaultGroup](api.md#DefaultGroup) * [Cordon](api.md#Cordon) * [Info](api.md#Info) + * [Restart](api.md#Restart) * [Shutdown](api.md#Shutdown) * [Uncordon](api.md#Uncordon) * [Version](api.md#Version) @@ -166,6 +167,15 @@ Response: } ``` +#### Restart + + +Perms: admin + +Inputs: `null` + +Response: `{}` + #### Shutdown diff --git a/go.mod b/go.mod index f8db62bec..6682cb1f2 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( github.com/dustin/go-humanize v1.0.1 github.com/elastic/go-sysinfo v1.15.4 github.com/ethereum/go-ethereum v1.17.2 + github.com/facebookgo/grace v0.0.0-20180706040059-75cf19382434 github.com/fatih/color v1.19.0 github.com/filecoin-project/filecoin-ffi v1.36.0 github.com/filecoin-project/go-address v1.2.0 @@ -181,6 +182,9 @@ require ( github.com/emicklei/dot v1.6.2 // indirect github.com/etclabscore/go-jsonschema-walk v0.0.6 // indirect github.com/ethereum/c-kzg-4844/v2 v2.1.7 // indirect + github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect + github.com/facebookgo/httpdown v0.0.0-20180706035922-5979d39b15c2 // indirect + github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/ferranbt/fastssz v0.1.4 // indirect github.com/filecoin-project/go-amt-ipld/v2 v2.1.0 // indirect diff --git a/go.sum b/go.sum index 1f0dd3171..1655c62f9 100644 --- a/go.sum +++ b/go.sum @@ -322,6 +322,14 @@ github.com/ethereum/go-bigmodexpfix v0.0.0-20250911101455-f9e208c548ab h1:rvv6MJ github.com/ethereum/go-bigmodexpfix v0.0.0-20250911101455-f9e208c548ab/go.mod h1:IuLm4IsPipXKF7CW5Lzf68PIbZ5yl7FFd74l/E0o9A8= github.com/ethereum/go-ethereum v1.17.2 h1:ag6geu0kn8Hv5FLKTpH+Hm2DHD+iuFtuqKxEuwUsDOI= github.com/ethereum/go-ethereum v1.17.2/go.mod h1:KHcRXfGOUfUmKg51IhQ0IowiqZ6PqZf08CMtk0g5K1o= +github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a h1:yDWHCSQ40h88yih2JAcL6Ls/kVkSE8GFACTGVnMPruw= +github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a/go.mod h1:7Ga40egUymuWXxAe151lTNnCv97MddSOVsjpPPkityA= +github.com/facebookgo/grace v0.0.0-20180706040059-75cf19382434 h1:mOp33BLbcbJ8fvTAmZacbBiOASfxN+MLcLxymZCIrGE= +github.com/facebookgo/grace v0.0.0-20180706040059-75cf19382434/go.mod h1:KigFdumBXUPSwzLDbeuzyt0elrL7+CP7TKuhrhT4bcU= +github.com/facebookgo/httpdown v0.0.0-20180706035922-5979d39b15c2 h1:nXeeRHmgNgjLxi+7dY9l9aDvSS1uwVlNLqUWIY4Ath0= +github.com/facebookgo/httpdown v0.0.0-20180706035922-5979d39b15c2/go.mod h1:TUV/fX3XrTtBQb5+ttSUJzcFgLNpILONFTKmBuk5RSw= +github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4 h1:0YtRCqIZs2+Tz49QuH6cJVw/IFqzo39gEqZ0iYLxD2M= +github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4/go.mod h1:vsJz7uE339KUCpBXx3JAJzSRH7Uk4iGGyJzR529qDIA= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.19.0 h1:Zp3PiM21/9Ld6FzSKyL5c/BULoe/ONr9KlbYVOfG8+w= github.com/fatih/color v1.19.0/go.mod h1:zNk67I0ZUT1bEGsSGyCZYZNrHuTkJJB+r6Q9VuMi0LE= diff --git a/harmony/harmonytask/harmonytask.go b/harmony/harmonytask/harmonytask.go index eee4b7946..f2c1ee71c 100644 --- a/harmony/harmonytask/harmonytask.go +++ b/harmony/harmonytask/harmonytask.go @@ -16,6 +16,7 @@ import ( "github.com/filecoin-project/curio/harmony/harmonydb" "github.com/filecoin-project/curio/harmony/resources" "github.com/filecoin-project/curio/harmony/taskhelp" + "github.com/filecoin-project/curio/lib/gracehttpsvc" ) // Consts (except for unit test) @@ -145,6 +146,11 @@ type TaskEngine struct { // runtime flags yieldBackground atomic.Bool + // restartRequested is an in-memory cordon set when a graceful restart is + // requested via RPC. It is intentionally not persisted to the DB so that + // the node returns to its previous cordon state after restarting. + restartRequested atomic.Bool + // synchronous to the single-threaded poller lastFollowTime time.Time lastCleanup atomic.Value @@ -288,6 +294,11 @@ func (e *TaskEngine) poller() { continue } + // An in-memory restart request cordons the node without touching the DB. + if e.restartRequested.Load() { + schedulable = false + } + e.yieldBackground.Store(!schedulable) accepted := e.pollerTryAllWork(schedulable) @@ -518,6 +529,55 @@ func (e *TaskEngine) checkNodeFlags() (bool, error) { return !unschedulable, nil } +func (e *TaskEngine) activeTaskCount() int { + count := 0 + for _, h := range e.handlers { + count += h.Max.ActiveThis() + } + return count +} + +// RequestRestart begins an in-memory graceful restart. The node immediately +// stops accepting new tasks (the same effect as cordoning), waits for the +// currently-running tasks to drain, then triggers a zero-downtime restart. +// +// The cordon is kept in-memory only and is never written to harmony_machines, +// so after the restart the node returns to whatever cordon state was persisted +// in the database before the restart was requested. +func (e *TaskEngine) RequestRestart() { + if !e.restartRequested.CompareAndSwap(false, true) { + // a restart is already in progress + return + } + log.Infow("restart requested; cordoning in-memory and draining tasks", "ownerID", e.ownerID) + go e.drainAndRestart() +} + +func (e *TaskEngine) drainAndRestart() { + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + + for { + select { + case <-e.ctx.Done(): + return + case <-ticker.C: + } + + if active := e.activeTaskCount(); active > 0 { + log.Infow("restart waiting for tasks to drain", "ownerID", e.ownerID, "activeTasks ", active) + continue + } + + log.Infow("no tasks running, triggering graceful restart", "ownerID", e.ownerID) + if err := gracehttpsvc.TriggerRestart(); err != nil { + log.Errorw("graceful restart failed, falling back to exit", "error", err) + os.Exit(ExitStatusRestartRequest) + } + return + } +} + func (e *TaskEngine) restartIfNoTasksPending(pendingSince time.Time) { var tasksPending int err := e.db.QueryRow(e.ctx, `SELECT COUNT(*) FROM harmony_task WHERE owner_id=$1`, e.ownerID).Scan(&tasksPending) @@ -525,17 +585,29 @@ func (e *TaskEngine) restartIfNoTasksPending(pendingSince time.Time) { log.Error("Unable to check for tasks pending: ", err) return } - if tasksPending == 0 { - log.Infow("no tasks pending, restarting", "ownerID", e.ownerID, "pendingSince", pendingSince, "took", time.Since(pendingSince)) - // unset the flags first - _, err = e.db.Exec(e.ctx, `UPDATE harmony_machines SET restart_request=NULL, unschedulable=FALSE WHERE host_and_port=$1`, e.hostAndPort) - if err != nil { - log.Error("Unable to unset restart request: ", err) - return - } + activeTasks := e.activeTaskCount() + if tasksPending > 0 || activeTasks > 0 { + log.Infow("restart waiting for tasks to finish", + "ownerID", e.ownerID, + "pendingSince", pendingSince, + "tasksPending", tasksPending, + "activeTasks", activeTasks) + return + } + + log.Infow("no tasks pending, restarting", "ownerID", e.ownerID, "pendingSince", pendingSince, "took", time.Since(pendingSince)) + + // Clear restart_request only; stay cordoned until the operator uncordons after restart. + _, err = e.db.Exec(e.ctx, `UPDATE harmony_machines SET restart_request=NULL WHERE host_and_port=$1`, e.hostAndPort) + if err != nil { + log.Error("Unable to unset restart request: ", err) + return + } - // then exit + // zero-downtime restart via gracehttp; fall back to exit 100 for systemd + if err := gracehttpsvc.TriggerRestart(); err != nil { + log.Errorw("graceful restart failed, falling back to exit", "error", err) os.Exit(ExitStatusRestartRequest) } } diff --git a/itests/grace_restart_test.go b/itests/grace_restart_test.go new file mode 100644 index 000000000..724b1d152 --- /dev/null +++ b/itests/grace_restart_test.go @@ -0,0 +1,95 @@ +package itests + +import ( + "context" + "os" + "path/filepath" + "runtime" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/filecoin-project/curio/itests/helpers" + "github.com/filecoin-project/curio/lib/gracehttpsvc" +) + +func TestGraceRestartDuringPDPPing(t *testing.T) { + if testing.Short() { + t.Skip("skipping grace restart integration test in short mode") + } + if runtime.GOOS != "linux" { + t.Skip("gracehttp restart integration test requires linux") + } + + if os.Getenv(helpers.GraceRestartWorkerEnv) == "1" { + helpers.RunGraceRestartWorker(t) + return + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + repoPath := filepath.Join(t.TempDir(), "grace-restart-repo") + _ = helpers.StartGraceRestartWorker(t, repoPath) + + baseURL, err := helpers.GraceRestartBaseURL(repoPath) + require.NoError(t, err) + + helpers.WaitForPDPPing(t, baseURL, 2*time.Minute) + + pidPath := filepath.Join(repoPath, helpers.GraceRestartPIDFile) + initialPID, err := gracehttpsvc.ReadPID(pidPath) + require.NoError(t, err) + require.Greater(t, initialPID, 0) + + var pingFailures atomic.Int64 + var pingSuccesses atomic.Int64 + stopPinger := make(chan struct{}) + pingerDone := make(chan struct{}) + + go func() { + defer close(pingerDone) + ticker := time.NewTicker(25 * time.Millisecond) + defer ticker.Stop() + + for { + select { + case <-stopPinger: + return + case <-ctx.Done(): + return + case <-ticker.C: + if err := helpers.PDPPing(ctx, baseURL); err != nil { + pingFailures.Add(1) + continue + } + pingSuccesses.Add(1) + } + } + }() + + require.Eventually(t, func() bool { + return pingSuccesses.Load() >= 5 + }, 30*time.Second, 25*time.Millisecond, "expected initial successful PDP pings before restart") + + require.NoError(t, gracehttpsvc.RestartFromPIDFile(pidPath)) + + require.Eventually(t, func() bool { + currentPID, err := gracehttpsvc.ReadPID(pidPath) + if err != nil { + return false + } + return currentPID != initialPID && currentPID > 0 + }, 2*time.Minute, 100*time.Millisecond, "expected curio.pid to reflect a new process after restart") + + require.Eventually(t, func() bool { + return pingSuccesses.Load() >= 10 + }, 2*time.Minute, 25*time.Millisecond, "expected continued successful PDP pings through restart") + + close(stopPinger) + <-pingerDone + + require.Zero(t, pingFailures.Load(), "expected zero failed PDP pings during graceful restart") +} diff --git a/itests/helpers/grace_restart.go b/itests/helpers/grace_restart.go new file mode 100644 index 000000000..46009766b --- /dev/null +++ b/itests/helpers/grace_restart.go @@ -0,0 +1,149 @@ +package helpers + +import ( + "context" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "testing" + "time" + + "github.com/go-chi/chi/v5" + "github.com/stretchr/testify/require" + + "github.com/filecoin-project/curio/harmony/harmonydb" + "github.com/filecoin-project/curio/lib/gracehttpsvc" + "github.com/filecoin-project/curio/pdp" +) + +const ( + GraceRestartWorkerEnv = "CURIO_GRACE_RESTART_WORKER" + GraceRestartListenFile = "listen.addr" + GraceRestartPIDFile = "curio.pid" + graceRestartWorkerTimeout = 3 * time.Minute +) + +// PDPPingURL returns the PDP ping endpoint URL for a market HTTP base URL. +func PDPPingURL(baseURL string) string { + return baseURL + pdp.PDPRoutePath + "/ping" +} + +// PDPPing checks GET /pdp/ping and returns an error when the response is not 200 OK. +func PDPPing(ctx context.Context, baseURL string) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, PDPPingURL(baseURL), nil) + if err != nil { + return err + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer func() { _ = resp.Body.Close() }() + _, _ = io.Copy(io.Discard, resp.Body) + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("pdp ping returned status %d", resp.StatusCode) + } + return nil +} + +// WaitForPDPPing blocks until GET /pdp/ping succeeds. +func WaitForPDPPing(t *testing.T, baseURL string, timeout time.Duration) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + require.NoError(t, WaitForCondition(ctx, timeout, 100*time.Millisecond, func() (bool, error) { + err := PDPPing(ctx, baseURL) + return err == nil, nil + })) +} + +// GraceRestartBaseURL reads the worker listen address file and returns an HTTP base URL. +func GraceRestartBaseURL(repoPath string) (string, error) { + data, err := os.ReadFile(filepath.Join(repoPath, GraceRestartListenFile)) + if err != nil { + return "", err + } + addr := string(data) + if len(addr) > 0 && addr[len(addr)-1] == '\n' { + addr = addr[:len(addr)-1] + } + if addr == "" { + return "", fmt.Errorf("empty listen address in %s", GraceRestartListenFile) + } + return "http://" + loopbackDialAddr(addr), nil +} + +// RunGraceRestartWorker starts a subprocess-friendly gracehttp server exposing the +// real PDP /pdp/ping handler. It is invoked when CURIO_GRACE_RESTART_WORKER=1. +func RunGraceRestartWorker(t *testing.T) { + t.Helper() + + repoPath := os.Getenv("CURIO_REPO_PATH") + require.NotEmpty(t, repoPath) + + listenAddr := FreeListenAddr(t) + require.NoError(t, os.WriteFile(filepath.Join(repoPath, GraceRestartListenFile), []byte(listenAddr+"\n"), 0644)) + + db, err := harmonydb.NewFromConfigWithITestID(t) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + pdsvc := pdp.NewPDPService(ctx, db, nil, nil, nil, nil, nil, nil) + + router := chi.NewRouter() + pdp.Routes(router, pdsvc) + + server := &http.Server{ + Addr: listenAddr, + Handler: router, + } + + pidPath := filepath.Join(repoPath, GraceRestartPIDFile) + require.NoError(t, gracehttpsvc.WritePIDFile(pidPath)) + defer gracehttpsvc.RemovePIDFile(pidPath) + + require.NoError(t, gracehttpsvc.Serve(server)) +} + +// StartGraceRestartWorker launches a child test process that serves /pdp/ping via gracehttp. +func StartGraceRestartWorker(t *testing.T, repoPath string) *os.Process { + t.Helper() + + require.NoError(t, os.MkdirAll(repoPath, 0755)) + + cmd := os.Args[0] + args := []string{ + "-test.run=^TestGraceRestartDuringPDPPing$", + "-test.count=1", + } + + procAttr := &os.ProcAttr{ + Dir: repoPath, + Env: append(os.Environ(), + GraceRestartWorkerEnv+"=1", + "CURIO_REPO_PATH="+repoPath, + "CURIO_SKIP_ALREADY_RUNNING=1", + ), + Files: []*os.File{os.Stdin, os.Stdout, os.Stderr}, + Sys: nil, + } + + process, err := os.StartProcess(cmd, append([]string{cmd}, args...), procAttr) + require.NoError(t, err) + + t.Cleanup(func() { + if process != nil { + _ = gracehttpsvc.ShutdownPID(process.Pid) + _, _ = process.Wait() + } + }) + + return process +} diff --git a/itests/helpers/harness.go b/itests/helpers/harness.go index 2c5c17f1a..f9bf78889 100644 --- a/itests/helpers/harness.go +++ b/itests/helpers/harness.go @@ -5,6 +5,7 @@ import ( "encoding/base64" "fmt" "net" + "net/http" "os" "testing" "time" @@ -91,11 +92,15 @@ func StartCurioHarness( require.NoError(t, os.Setenv("CURIO_REPO_PATH", dir)) require.NoError(t, dependencies.PopulateRemainingDeps(ctx, cctx, false)) - taskEngine, err := tasks.StartTasks(ctx, dependencies, shutdownChan) + taskEngine, marketServer, err := tasks.StartTasks(ctx, dependencies, shutdownChan) require.NoError(t, err) go func() { - err := rpc.ListenAndServe(ctx, dependencies, shutdownChan) + var extra []*http.Server + if marketServer != nil { + extra = append(extra, marketServer) + } + err := rpc.ListenAndServe(ctx, dependencies, shutdownChan, taskEngine, extra...) if err != nil { t.Errorf("failed to start the Curio RPC server: %v", err) } diff --git a/lib/gracehttpsvc/gracehttpsvc.go b/lib/gracehttpsvc/gracehttpsvc.go new file mode 100644 index 000000000..2bfa1f762 --- /dev/null +++ b/lib/gracehttpsvc/gracehttpsvc.go @@ -0,0 +1,158 @@ +// Package gracehttpsvc integrates facebookgo/gracehttp for zero-downtime +// HTTP restarts via SIGUSR2 and graceful shutdown via SIGTERM. +package gracehttpsvc + +import ( + "errors" + "fmt" + "net/http" + "os" + "path/filepath" + "strconv" + "syscall" + + "github.com/facebookgo/grace/gracehttp" +) + +const listenFdsKey = "LISTEN_FDS" + +var preRestartHook func() error + +// SetPreRestartHook registers a callback invoked immediately before a graceful +// restart forks the successor process. Use it to drain in-flight work. +func SetPreRestartHook(hook func() error) { + preRestartHook = hook +} + +// TriggerRestart requests a zero-downtime restart by sending SIGUSR2 to the +// current process. gracehttp will fork a successor and hand off listeners. +func TriggerRestart() error { + return syscall.Kill(os.Getpid(), syscall.SIGUSR2) +} + +// TriggerShutdown requests a graceful shutdown by sending SIGTERM to the +// current process. gracehttp will drain HTTP connections before exiting. +func TriggerShutdown() error { + return ShutdownPID(os.Getpid()) +} + +// ShutdownPID sends SIGTERM to the process identified by pid. +func ShutdownPID(pid int) error { + return syscall.Kill(pid, syscall.SIGTERM) +} + +// RestartPID sends SIGUSR2 to the process identified by pid. +func RestartPID(pid int) error { + return syscall.Kill(pid, syscall.SIGUSR2) +} + +// WritePIDFile records the current process id at path. +func WritePIDFile(path string) error { + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + return fmt.Errorf("creating pid file directory: %w", err) + } + return os.WriteFile(path, []byte(strconv.Itoa(os.Getpid())+"\n"), 0644) +} + +// RemovePIDFile deletes the pid file at path. +func RemovePIDFile(path string) { + _ = os.Remove(path) +} + +// IsGraceHandoff reports whether this process inherited listener FDs from a +// graceful restart predecessor. +func IsGraceHandoff() bool { + return os.Getenv(listenFdsKey) != "" +} + +// IsProcessAlive reports whether pid refers to a running process. +func IsProcessAlive(pid int) bool { + if pid <= 0 || pid == os.Getpid() { + return false + } + proc, err := os.FindProcess(pid) + if err != nil { + return false + } + return proc.Signal(syscall.Signal(0)) == nil +} + +func parsePID(data []byte) (int, error) { + text := string(data) + if len(text) > 0 && text[len(text)-1] == '\n' { + text = text[:len(text)-1] + } + return strconv.Atoi(text) +} + +// ReadPID reads a process id from path. +func ReadPID(path string) (int, error) { + data, err := os.ReadFile(path) + if err != nil { + return 0, fmt.Errorf("reading pid file %s: %w", path, err) + } + pid, err := parsePID(data) + if err != nil { + return 0, fmt.Errorf("parsing pid from %s: %w", path, err) + } + return pid, nil +} + +// RestartFromPIDFile reads a pid from path and sends SIGUSR2 to that process. +func RestartFromPIDFile(path string) error { + pid, err := ReadPID(path) + if err != nil { + return err + } + if err := RestartPID(pid); err != nil { + return fmt.Errorf("sending SIGUSR2 to pid %d: %w", pid, err) + } + return nil +} + +// RestartIfAlreadyRunning checks pidPath for a live process. When one is found, +// SIGUSR2 is sent to trigger a graceful restart and true is returned so the +// caller can exit without starting a second instance. Stale pid files are removed. +func RestartIfAlreadyRunning(pidPath string) (bool, error) { + data, err := os.ReadFile(pidPath) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, fmt.Errorf("reading pid file %s: %w", pidPath, err) + } + + pid, err := parsePID(data) + if err != nil { + RemovePIDFile(pidPath) + return false, nil + } + + if !IsProcessAlive(pid) { + RemovePIDFile(pidPath) + return false, nil + } + + if err := RestartPID(pid); err != nil { + if errors.Is(err, syscall.ESRCH) { + RemovePIDFile(pidPath) + return false, nil + } + return false, err + } + + return true, nil +} + +// Serve starts one or more HTTP servers with gracehttp, enabling graceful +// shutdown on SIGTERM/SIGINT and zero-downtime restart on SIGUSR2. +func Serve(servers ...*http.Server) error { + if len(servers) == 0 { + return fmt.Errorf("no http servers to serve") + } + + if preRestartHook != nil { + return gracehttp.ServeWithOptions(servers, gracehttp.PreStartProcess(preRestartHook)) + } + return gracehttp.Serve(servers...) +} diff --git a/lib/gracehttpsvc/gracehttpsvc_test.go b/lib/gracehttpsvc/gracehttpsvc_test.go new file mode 100644 index 000000000..aa91dc581 --- /dev/null +++ b/lib/gracehttpsvc/gracehttpsvc_test.go @@ -0,0 +1,109 @@ +package gracehttpsvc + +import ( + "os" + "path/filepath" + "testing" +) + +func TestParsePID(t *testing.T) { + t.Parallel() + + pid, err := parsePID([]byte("12345\n")) + if err != nil || pid != 12345 { + t.Fatalf("expected pid 12345, got %d (%v)", pid, err) + } + + pid, err = parsePID([]byte("99")) + if err != nil || pid != 99 { + t.Fatalf("expected pid 99, got %d (%v)", pid, err) + } + + if _, err := parsePID([]byte("not-a-pid")); err == nil { + t.Fatal("expected parse error") + } +} + +func TestRestartIfAlreadyRunningMissingPIDFile(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + pidPath := filepath.Join(dir, "curio.pid") + + restarted, err := RestartIfAlreadyRunning(pidPath) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if restarted { + t.Fatal("expected no restart when pid file is missing") + } +} + +func TestRestartIfAlreadyRunningStalePIDFile(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + pidPath := filepath.Join(dir, "curio.pid") + if err := os.WriteFile(pidPath, []byte("999999999\n"), 0644); err != nil { + t.Fatal(err) + } + + restarted, err := RestartIfAlreadyRunning(pidPath) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if restarted { + t.Fatal("expected no restart for stale pid") + } + if _, err := os.Stat(pidPath); !os.IsNotExist(err) { + t.Fatal("expected stale pid file to be removed") + } +} + +func TestRestartIfAlreadyRunningCurrentProcess(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + pidPath := filepath.Join(dir, "curio.pid") + if err := WritePIDFile(pidPath); err != nil { + t.Fatal(err) + } + + restarted, err := RestartIfAlreadyRunning(pidPath) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if restarted { + t.Fatal("expected no restart when pid file refers to current process") + } +} + +func TestReadPID(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + pidPath := filepath.Join(dir, "curio.pid") + if err := WritePIDFile(pidPath); err != nil { + t.Fatal(err) + } + + pid, err := ReadPID(pidPath) + if err != nil { + t.Fatal(err) + } + if pid != os.Getpid() { + t.Fatalf("expected pid %d, got %d", os.Getpid(), pid) + } +} + +func TestIsGraceHandoff(t *testing.T) { + t.Setenv(listenFdsKey, "") + if IsGraceHandoff() { + t.Fatal("expected no handoff without LISTEN_FDS") + } + + t.Setenv(listenFdsKey, "2") + if !IsGraceHandoff() { + t.Fatal("expected handoff when LISTEN_FDS is set") + } +}