From aa7360fc69ade5c8aef7b881903f28b6f926461a Mon Sep 17 00:00:00 2001 From: frrist Date: Wed, 28 Jan 2026 08:06:43 -0800 Subject: [PATCH 1/2] docs: update docs on configuration and CLI --- docs/content/cli/client/admin/config/get.md | 27 ++++++ docs/content/cli/client/admin/config/index.md | 27 ++++++ docs/content/cli/client/admin/config/list.md | 24 +++++ .../content/cli/client/admin/config/reload.md | 25 +++++ docs/content/cli/client/admin/config/set.md | 34 +++++++ docs/content/cli/client/admin/index.md | 19 ++++ docs/content/cli/client/admin/log/index.md | 23 +++++ docs/content/cli/client/admin/log/list.md | 25 +++++ .../content/cli/client/admin/log/set-regex.md | 30 ++++++ docs/content/cli/client/admin/log/set.md | 28 ++++++ docs/content/cli/client/index.md | 31 ++++++ docs/content/cli/client/pdp/index.md | 15 +++ docs/content/cli/client/pdp/proofset/index.md | 15 +++ docs/content/cli/client/pdp/proofset/state.md | 50 ++++++++++ docs/content/cli/identity/generate.md | 25 +++++ docs/content/cli/identity/index.md | 23 +++++ docs/content/cli/identity/parse.md | 25 +++++ docs/content/cli/index.md | 49 ++++++++++ docs/content/cli/init.md | 97 +++++++++++++++++++ docs/content/cli/serve/index.md | 76 +++++++++++++++ docs/content/cli/status/index.md | 47 +++++++++ docs/content/cli/status/upgrade-check.md | 29 ++++++ docs/content/cli/update.md | 32 ++++++ docs/content/cli/wallet/import.md | 34 +++++++ docs/content/cli/wallet/index.md | 21 ++++ docs/content/cli/wallet/list.md | 19 ++++ docs/content/configuration/identity.md | 20 ++++ docs/content/configuration/index.md | 93 ++++++++++++++++++ docs/content/configuration/network.md | 24 +++++ .../pdp/aggregation/aggregator.md | 49 ++++++++++ .../configuration/pdp/aggregation/commp.md | 43 ++++++++ .../configuration/pdp/aggregation/index.md | 22 +++++ .../configuration/pdp/aggregation/manager.md | 96 ++++++++++++++++++ docs/content/configuration/pdp/index.md | 81 ++++++++++++++++ .../configuration/repo/blob-storage.md | 14 +++ docs/content/configuration/repo/index.md | 31 ++++++ docs/content/configuration/server.md | 32 ++++++ docs/content/configuration/telemetry.md | 40 ++++++++ docs/content/configuration/ucan.md | 88 +++++++++++++++++ docs/mkdocs.yml | 54 +++++++++++ 40 files changed, 1537 insertions(+) create mode 100644 docs/content/cli/client/admin/config/get.md create mode 100644 docs/content/cli/client/admin/config/index.md create mode 100644 docs/content/cli/client/admin/config/list.md create mode 100644 docs/content/cli/client/admin/config/reload.md create mode 100644 docs/content/cli/client/admin/config/set.md create mode 100644 docs/content/cli/client/admin/index.md create mode 100644 docs/content/cli/client/admin/log/index.md create mode 100644 docs/content/cli/client/admin/log/list.md create mode 100644 docs/content/cli/client/admin/log/set-regex.md create mode 100644 docs/content/cli/client/admin/log/set.md create mode 100644 docs/content/cli/client/index.md create mode 100644 docs/content/cli/client/pdp/index.md create mode 100644 docs/content/cli/client/pdp/proofset/index.md create mode 100644 docs/content/cli/client/pdp/proofset/state.md create mode 100644 docs/content/cli/identity/generate.md create mode 100644 docs/content/cli/identity/index.md create mode 100644 docs/content/cli/identity/parse.md create mode 100644 docs/content/cli/index.md create mode 100644 docs/content/cli/init.md create mode 100644 docs/content/cli/serve/index.md create mode 100644 docs/content/cli/status/index.md create mode 100644 docs/content/cli/status/upgrade-check.md create mode 100644 docs/content/cli/update.md create mode 100644 docs/content/cli/wallet/import.md create mode 100644 docs/content/cli/wallet/index.md create mode 100644 docs/content/cli/wallet/list.md create mode 100644 docs/content/configuration/identity.md create mode 100644 docs/content/configuration/index.md create mode 100644 docs/content/configuration/network.md create mode 100644 docs/content/configuration/pdp/aggregation/aggregator.md create mode 100644 docs/content/configuration/pdp/aggregation/commp.md create mode 100644 docs/content/configuration/pdp/aggregation/index.md create mode 100644 docs/content/configuration/pdp/aggregation/manager.md create mode 100644 docs/content/configuration/pdp/index.md create mode 100644 docs/content/configuration/repo/blob-storage.md create mode 100644 docs/content/configuration/repo/index.md create mode 100644 docs/content/configuration/server.md create mode 100644 docs/content/configuration/telemetry.md create mode 100644 docs/content/configuration/ucan.md diff --git a/docs/content/cli/client/admin/config/get.md b/docs/content/cli/client/admin/config/get.md new file mode 100644 index 00000000..2a478250 --- /dev/null +++ b/docs/content/cli/client/admin/config/get.md @@ -0,0 +1,27 @@ +# get + +Get a specific dynamic configuration value. + +## Usage + +``` +piri client admin config get +``` + +## Arguments + +| Argument | Description | +|----------|-------------| +| `` | The configuration key to retrieve | + +## Example + +```bash +piri client admin config get pdp.aggregation.manager.poll_interval +``` + +``` +30s +``` + +See [Dynamic Configuration](../../../../configuration/index.md#dynamic-configuration) for the list of available keys. diff --git a/docs/content/cli/client/admin/config/index.md b/docs/content/cli/client/admin/config/index.md new file mode 100644 index 00000000..caea485c --- /dev/null +++ b/docs/content/cli/client/admin/config/index.md @@ -0,0 +1,27 @@ +# config + +Manage dynamic configuration values at runtime. + +## Usage + +``` +piri client admin config [command] +``` + +## Subcommands + +### [list](list.md) + +List all dynamic configuration values. + +### [get](get.md) + +Get a specific configuration value. + +### [set](set.md) + +Set a configuration value. + +### [reload](reload.md) + +Reload configuration from file. diff --git a/docs/content/cli/client/admin/config/list.md b/docs/content/cli/client/admin/config/list.md new file mode 100644 index 00000000..0f8e327a --- /dev/null +++ b/docs/content/cli/client/admin/config/list.md @@ -0,0 +1,24 @@ +# list + +List all dynamic configuration values. + +## Usage + +``` +piri client admin config list +``` + +## Example + +```bash +piri client admin config list +``` + +```json +{ + "pdp.aggregation.manager.batch_size": 10, + "pdp.aggregation.manager.poll_interval": "30s" +} +``` + +See [Dynamic Configuration](../../../../configuration/index.md#dynamic-configuration) for the list of available keys. diff --git a/docs/content/cli/client/admin/config/reload.md b/docs/content/cli/client/admin/config/reload.md new file mode 100644 index 00000000..9925c7fd --- /dev/null +++ b/docs/content/cli/client/admin/config/reload.md @@ -0,0 +1,25 @@ +# reload + +Reload configuration from the config file. + +## Usage + +``` +piri client admin config reload +``` + +## Example + +```bash +piri client admin config reload +``` + +``` +Configuration reloaded from file +{ + "pdp.aggregation.manager.batch_size": 10, + "pdp.aggregation.manager.poll_interval": "1m0s" +} +``` + +See [Dynamic Configuration](../../../../configuration/index.md#dynamic-configuration) for the list of available keys. diff --git a/docs/content/cli/client/admin/config/set.md b/docs/content/cli/client/admin/config/set.md new file mode 100644 index 00000000..0280530c --- /dev/null +++ b/docs/content/cli/client/admin/config/set.md @@ -0,0 +1,34 @@ +# set + +Set a dynamic configuration value. + +## Usage + +``` +piri client admin config set [flags] +``` + +## Arguments + +| Argument | Description | +|----------|-------------| +| `` | The configuration key to set | +| `` | The new value | + +## Flags + +| Flag | Description | +|------|-------------| +| `--persist` | Persist the change to the config file | + +## Example + +```bash +piri client admin config set pdp.aggregation.manager.poll_interval 1m +``` + +``` +pdp.aggregation.manager.poll_interval updated +``` + +See [Dynamic Configuration](../../../../configuration/index.md#dynamic-configuration) for the list of available keys. diff --git a/docs/content/cli/client/admin/index.md b/docs/content/cli/client/admin/index.md new file mode 100644 index 00000000..212603fe --- /dev/null +++ b/docs/content/cli/client/admin/index.md @@ -0,0 +1,19 @@ +# admin + +Administrative operations for managing a running Piri node. + +## Usage + +``` +piri client admin [command] +``` + +## Subcommands + +### [config](config/index.md) + +Manage dynamic configuration. + +### [log](log/index.md) + +Manage logging levels. diff --git a/docs/content/cli/client/admin/log/index.md b/docs/content/cli/client/admin/log/index.md new file mode 100644 index 00000000..b945966d --- /dev/null +++ b/docs/content/cli/client/admin/log/index.md @@ -0,0 +1,23 @@ +# log + +Manage logging levels for a running Piri node. + +## Usage + +``` +piri client admin log [command] +``` + +## Subcommands + +### [list](list.md) + +List all log systems and their current levels. + +### [set](set.md) + +Set a log level for one system or all systems. + +### [set-regex](set-regex.md) + +Set log level for subsystems matching a regex pattern. diff --git a/docs/content/cli/client/admin/log/list.md b/docs/content/cli/client/admin/log/list.md new file mode 100644 index 00000000..185c9f3c --- /dev/null +++ b/docs/content/cli/client/admin/log/list.md @@ -0,0 +1,25 @@ +# list + +List all log systems and their current levels. + +## Usage + +``` +piri client admin log list +``` + +## Example + +```bash +piri client admin log list +``` + +```json +{ + "piri/pdp": "info", + "piri/storage": "info", + "piri/retrieval": "debug", + "piri/claims": "info", + "piri/replicator": "warn" +} +``` diff --git a/docs/content/cli/client/admin/log/set-regex.md b/docs/content/cli/client/admin/log/set-regex.md new file mode 100644 index 00000000..53670860 --- /dev/null +++ b/docs/content/cli/client/admin/log/set-regex.md @@ -0,0 +1,30 @@ +# set-regex + +Set log level for subsystems matching a regex pattern. + +## Usage + +``` +piri client admin log set-regex +``` + +## Arguments + +| Argument | Description | +|----------|-------------| +| `` | Log level to set (debug, info, warn, error) | +| `` | Regular expression pattern to match system names | + +## Example + +```bash +piri client admin log set-regex debug "piri/pdp.*" +``` + +```json +{ + "piri/pdp": "debug", + "piri/pdp/scheduler": "debug", + "piri/pdp/aggregation": "debug" +} +``` diff --git a/docs/content/cli/client/admin/log/set.md b/docs/content/cli/client/admin/log/set.md new file mode 100644 index 00000000..beed16ad --- /dev/null +++ b/docs/content/cli/client/admin/log/set.md @@ -0,0 +1,28 @@ +# set + +Set a log level for one system or all systems. + +## Usage + +``` +piri client admin log set [system] +``` + +## Arguments + +| Argument | Description | +|----------|-------------| +| `` | Log level to set (debug, info, warn, error) | +| `[system]` | Specific system name. If omitted, sets level for all systems | + +## Example + +```bash +piri client admin log set debug piri/pdp +``` + +```json +{ + "piri/pdp": "debug" +} +``` diff --git a/docs/content/cli/client/index.md b/docs/content/cli/client/index.md new file mode 100644 index 00000000..e2573559 --- /dev/null +++ b/docs/content/cli/client/index.md @@ -0,0 +1,31 @@ +# client + +Interact with a running Piri node. + +All client commands require a running Piri server and authenticate using JWT tokens signed with your node's identity key. Pass the same `--config` file you used with `piri serve` - this provides both the server address and your identity key for authentication: + +```bash +piri --config=config.toml client admin config list +``` + +## Usage + +``` +piri client [command] [flags] +``` + +## Flags + +| Flag | Description | Default | +|------|-------------|---------| +| `--node-url ` | URL of a Piri node | `http://localhost:3000` | + +## Subcommands + +### [admin](admin/index.md) + +Administrative operations (logs, config). + +### [pdp](pdp/index.md) + +PDP (Provable Data Possession) operations. \ No newline at end of file diff --git a/docs/content/cli/client/pdp/index.md b/docs/content/cli/client/pdp/index.md new file mode 100644 index 00000000..79fb4aa4 --- /dev/null +++ b/docs/content/cli/client/pdp/index.md @@ -0,0 +1,15 @@ +# pdp + +PDP (Provable Data Possession) operations. + +## Usage + +``` +piri client pdp [command] +``` + +## Subcommands + +### [proofset](proofset/index.md) + +Manage proof sets. diff --git a/docs/content/cli/client/pdp/proofset/index.md b/docs/content/cli/client/pdp/proofset/index.md new file mode 100644 index 00000000..9f4886d4 --- /dev/null +++ b/docs/content/cli/client/pdp/proofset/index.md @@ -0,0 +1,15 @@ +# proofset + +Manage proof sets. + +## Usage + +``` +piri client pdp proofset [command] +``` + +## Subcommands + +### [state](state.md) + +Get the current state of a proof set. diff --git a/docs/content/cli/client/pdp/proofset/state.md b/docs/content/cli/client/pdp/proofset/state.md new file mode 100644 index 00000000..9c4cb8f5 --- /dev/null +++ b/docs/content/cli/client/pdp/proofset/state.md @@ -0,0 +1,50 @@ +# state + +Get the current state of a proof set. + +## Usage + +``` +piri client pdp proofset state [flags] +``` + +## Flags + +| Flag | Description | +|------|-------------| +| `--proof-set ` | The proof set ID to query | +| `--json` | Output in JSON format | + +## Example + +```bash +piri client pdp proofset state --proof-set 123 +``` + +``` +Proof Set State +=============== +Proof Set ID: 123 +Proving Period: 2880 epochs +Challenge Window: 60 epochs + +Owners +------ +Owner: 0x1234...abcd + +Status +------ +Initialized: true +Current Epoch: 4052160 + +Challenge State +--------------- +In Challenge Window: false +Has Proven: true +Next Challenge: 2025-01-28 14:30:00 UTC (estimated) + +On-Chain State +-------------- +Root Count: 42 +Total Data Size: 1.2 TiB +``` diff --git a/docs/content/cli/identity/generate.md b/docs/content/cli/identity/generate.md new file mode 100644 index 00000000..eb44c7ff --- /dev/null +++ b/docs/content/cli/identity/generate.md @@ -0,0 +1,25 @@ +# generate + +Generate a new PEM-encoded Ed25519 key pair for use with decentralized identities (DIDs). + +## Usage + +``` +piri identity generate +``` + +## Aliases + +- `gen` + +## Example + +```bash +piri identity generate > my-key.pem +``` + +``` +# did:key:z6MkhaXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX +``` + +The PEM-encoded key is written to stdout and can be redirected to a file. The DID is printed to stderr as a comment for convenience. diff --git a/docs/content/cli/identity/index.md b/docs/content/cli/identity/index.md new file mode 100644 index 00000000..1d2b45d8 --- /dev/null +++ b/docs/content/cli/identity/index.md @@ -0,0 +1,23 @@ +# identity + +Generate and manage Ed25519 keys for decentralized identities (DIDs). + +## Usage + +``` +piri identity [command] +``` + +## Aliases + +- `id` + +## Subcommands + +### [generate](generate.md) + +Generate a new PEM-encoded Ed25519 key pair. + +### [parse](parse.md) + +Parse a DID from a PEM file. diff --git a/docs/content/cli/identity/parse.md b/docs/content/cli/identity/parse.md new file mode 100644 index 00000000..1b1d6ee7 --- /dev/null +++ b/docs/content/cli/identity/parse.md @@ -0,0 +1,25 @@ +# parse + +Parse a DID from a PEM file containing an Ed25519 key. + +## Usage + +``` +piri identity parse +``` + +## Arguments + +| Argument | Description | +|----------|-------------| +| `` | Path to PEM file containing Ed25519 private key | + +## Example + +```bash +piri identity parse my-key.pem +``` + +``` +# did:key:z6MkhaXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX +``` diff --git a/docs/content/cli/index.md b/docs/content/cli/index.md new file mode 100644 index 00000000..b6dfdbbf --- /dev/null +++ b/docs/content/cli/index.md @@ -0,0 +1,49 @@ +# CLI Reference + +Piri provides a command-line interface for managing storage nodes on the Storacha network. + +## Usage + +``` +piri [command] [flags] +``` + +## Global Flags + +| Flag | Description | +|------|-------------| +| `--config ` | Config file path. Defaults to `~/.config/piri/config.toml` | +| `--log-level ` | Logging level (debug, info, warn, error) | +| `--data-dir ` | Storage service data directory | +| `--temp-dir ` | Storage service temp directory | +| `--key-file ` | Path to PEM file containing ed25519 private key | + +## Subcommands + +### [init](init.md) + +Initialize a new Piri node. + +### [serve](serve/index.md) + +Start the Piri server. + +### [client](client/index.md) + +Interact with a running Piri node. + +### [wallet](wallet/index.md) + +Manage wallets. + +### [identity](identity/index.md) + +Generate and manage node identity. + +### [status](status/index.md) + +Show node status. + +### [update](update.md) + +Check for and apply updates to Piri. diff --git a/docs/content/cli/init.md b/docs/content/cli/init.md new file mode 100644 index 00000000..3a5fc34b --- /dev/null +++ b/docs/content/cli/init.md @@ -0,0 +1,97 @@ +# init + +Initialize your Piri node and register it with the Storacha network. + +This is a **one-time setup command** that prepares your node to participate in the network. It performs several critical operations: + +1. **Validates your configuration** - Checks that all required files and endpoints are accessible +2. **Imports your wallet** - Adds your Filecoin delegated address to Piri's wallet for on-chain transactions +3. **Creates a proof set** - Registers a proof set on the PDP smart contract (this can take up to 5 minutes as it waits for on-chain confirmation) +4. **Registers with the network** - Signs up your node with the Storacha delegator service and receives authorization proofs +5. **Generates a configuration file** - Outputs a complete TOML config file that [`piri serve`](serve/index.md) uses to run your node + +After initialization completes, you use the generated config file with `piri serve` to start your node. You typically only run `piri init` once per node setup. + +This command is **idempotent** - running it multiple times with the same parameters is safe and will reuse existing resources (like your proof set) rather than creating duplicates. + +## Prerequisites + +Before running this command, you need: + +- A synced [Lotus node](../setup/prerequisites.md#lotus-node-setup) with ETH RPC enabled +- A [funded delegated wallet](../setup/prerequisites.md#funded-delegated-wallet) exported to hex format +- An [Ed25519 identity key](../setup/key-generation.md) in PEM format (generated with `piri identity generate`) +- A domain with [TLS configured](../setup/tls-termination.md) pointing to your server +- Directories created for data and temp storage + +See the [Setup Guide](../setup/piri-server.md) for the complete walkthrough. + +## Usage + +``` +piri init [flags] +``` + +## Flags + +All flags are required: + +| Flag | Description | +|------|-------------| +| `--network ` | Network to join (`forge-prod` for mainnet, `warm-staging` for calibration) | +| `--data-dir ` | Directory for permanent Piri data (blobs, database) | +| `--temp-dir ` | Directory for temporary data during processing | +| `--key-file ` | Path to PEM file containing your Ed25519 identity key | +| `--wallet-file ` | Path to hex file containing your delegated Filecoin wallet private key | +| `--lotus-endpoint ` | WebSocket URL of your Lotus node (e.g., `wss://lotus.example.com/rpc/v1`) | +| `--operator-email ` | Contact email for the Storacha team to reach you | +| `--public-url ` | Public HTTPS URL where your node will be accessible | + +## Example + +```bash +piri init \ + --network=forge-prod \ + --data-dir=/var/lib/piri/data \ + --temp-dir=/var/lib/piri/temp \ + --key-file=/etc/piri/service.pem \ + --wallet-file=/etc/piri/wallet.hex \ + --lotus-endpoint=wss://lotus.example.com/rpc/v1 \ + --operator-email=admin@example.com \ + --public-url=https://piri.example.com > config.toml +``` + +``` +Initializing your Piri node on the Storacha Network... + +[1/5] Validating configuration... +Configuration validated + +[2/5] Creating Piri node... +Node created with DID: did:key:z6MkhaXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX + +[3/5] Setting up proof set... +Creating new proof set... +Waiting for proof set creation to be confirmed on-chain... +Proof set created with ID: 123 + +[4/5] Registering with delegator service... +Successfully registered with delegator service +Received delegator proof + +[5/5] Generating configuration file... + +Initialization complete! +``` + +The config file is written to stdout. Use `> config.toml` to save it to a file, or place it directly in `~/.config/piri/config.toml` for automatic loading by `piri serve`. + +## What's Next + +After initialization, start your node: + +```bash +piri serve --config=config.toml +``` + +See [`piri serve`](serve/index.md) for details on running the server. diff --git a/docs/content/cli/serve/index.md b/docs/content/cli/serve/index.md new file mode 100644 index 00000000..630fa6f7 --- /dev/null +++ b/docs/content/cli/serve/index.md @@ -0,0 +1,76 @@ +# serve + +Start the Piri server and begin participating in the Storacha network. + +This is a **long-running process** that operates your storage node. Once started, Piri will: + +1. **Accept storage requests** - Receive and store data from clients via the UCAN protocol +2. **Generate PDP proofs** - Create Provable Data Possession proofs on a regular schedule to demonstrate you're storing data correctly +3. **Respond to challenges** - Submit proofs to the on-chain PDP smart contract during challenge windows +4. **Publish to IPNI** - Announce stored content to the InterPlanetary Network Indexer for discoverability +5. **Handle retrievals** - Serve stored data back to clients on request + +The server runs continuously, writing logs to the terminal. For production deployments, you should run it as a systemd service or use a process manager to ensure it restarts on failure. + +## Prerequisites + +Before running this command: + +1. Run [`piri init`](../init.md) to generate a configuration file - this registers your node with the network and creates your proof set +2. Ensure your [TLS reverse proxy](../../setup/tls-termination.md) is running (Nginx with SSL certificates) +3. Verify your Lotus node is synced and accessible + +## Configuration + +The server reads its configuration from a TOML file generated by `piri init`. You can specify the config file path with `--config`, or place it in the default location (`~/.config/piri/config.toml`) for automatic loading. + +Most flags below are already set in your config file - you typically only need `--config` to start the server. See the [Configuration](../../configuration/index.md) docs for details on all available settings. + +## Usage + +``` +piri serve [flags] +``` + +## Flags + +| Flag | Description | Default | +|------|-------------|---------| +| `--config ` | Path to configuration file | `~/.config/piri/config.toml` | +| `--host ` | Host to listen on | `localhost` | +| `--port ` | Port to listen on | `3000` | +| `--public-url ` | URL the node is publicly accessible at | | +| `--network ` | Network the node operates on | | +| `--proof-set ` | Proof set ID to use with PDP | | +| `--lotus-url ` | WebSocket URL for Lotus node | | +| `--owner-address
` | Ethereum address to submit PDP proofs with (must be in piri wallet) | | + +## Example + +```bash +piri serve --config=config.toml +``` + +``` +▗▄▄▖ ▄ ▄▄▄ ▄ ▗ +▐▌ ▐▌▄ █ ▄ █▌ +▐▛▀▘ █ █ █ ▗█▘ +▐▌ █ █ ▀▘ + +v0.2.1 +did:key:z6MkhaXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX +Piri Running on: localhost:3000 +Piri Public Endpoint: https://piri.example.com +``` + +## Stopping the Server + +Use `Ctrl+C` to gracefully stop the server. On shutdown, Piri will: + +- Wait for any active retrievals to finish +- Wait for any in-progress chain tasks (proving, message sends) to complete +- Force stop after 1 minute if tasks haven't completed + +Before stopping for maintenance or upgrades, check if it's safe using [`piri status`](../status/index.md) to ensure you're not in the middle of a challenge window. + +See the [Setup Guide](../../setup/piri-server.md) for complete setup instructions and the [Upgrading](../../operations/upgrading.md) guide for safe update procedures. diff --git a/docs/content/cli/status/index.md b/docs/content/cli/status/index.md new file mode 100644 index 00000000..50ea59a7 --- /dev/null +++ b/docs/content/cli/status/index.md @@ -0,0 +1,47 @@ +# status + +Check node status and health. + +This command requires a running Piri server. Pass the same `--config` file you used with `piri serve` so the command knows where to connect: + +```bash +piri --config=config.toml status +``` + +This command provides a quick overview of your node's current state, including whether it's healthy, in a challenge window, and safe to update. For more detailed information about your proof set (challenge timing, on-chain state, fee information), use [`piri client pdp proofset state`](../client/pdp/proofset/state.md). + +## Usage + +``` +piri status [command] [flags] +``` + +## Flags + +| Flag | Description | +|------|-------------| +| `--json` | Output in JSON format | + +## Subcommands + +### [upgrade-check](upgrade-check.md) + +Check if it's safe to upgrade the node. + +## Example + +```bash +piri status +``` + +``` +Node Status +=========== +Healthy: YES +Currently Proving: NO +In Challenge: NO +Has Proven: YES +In Fault State: NO +Safe to Update: YES +Next Challenge: 2025-01-28 14:30:00 UTC +``` diff --git a/docs/content/cli/status/upgrade-check.md b/docs/content/cli/status/upgrade-check.md new file mode 100644 index 00000000..2ee615e7 --- /dev/null +++ b/docs/content/cli/status/upgrade-check.md @@ -0,0 +1,29 @@ +# upgrade-check + +Check if it's safe to upgrade the node. + +## Usage + +``` +piri status upgrade-check +``` + +## Exit Codes + +| Code | Meaning | +|------|---------| +| `0` | Safe to upgrade | +| `1` | Not safe to upgrade | +| `2` | Unable to determine status | + +## Example + +```bash +piri status upgrade-check && piri update +``` + +``` +Safe to upgrade +``` + +This command is designed for use in scripts and automation. It returns exit codes to indicate whether an upgrade can safely proceed without interrupting proof generation. diff --git a/docs/content/cli/update.md b/docs/content/cli/update.md new file mode 100644 index 00000000..fe677a92 --- /dev/null +++ b/docs/content/cli/update.md @@ -0,0 +1,32 @@ +# update + +Check for and apply updates to Piri. + +## Usage + +``` +piri update [flags] +``` + +## Flags + +| Flag | Description | +|------|-------------| +| `--check` | Check for updates without applying them | +| `--force` | Skip safety checks and force update | +| `--version ` | Update to a specific version (e.g., v1.2.3) | + +## Example + +```bash +piri update +``` + +``` +Current version: v0.2.1 +Latest version: v0.2.2 +Downloading v0.2.2... +Update complete. Restart the service to apply changes. +``` + +This command downloads and installs the latest version but does not restart the service. Use `piri status upgrade-check` first to verify it's safe to update. diff --git a/docs/content/cli/wallet/import.md b/docs/content/cli/wallet/import.md new file mode 100644 index 00000000..6e02efca --- /dev/null +++ b/docs/content/cli/wallet/import.md @@ -0,0 +1,34 @@ +# import + +Import an address to the wallet. + +## Usage + +``` +piri wallet import +``` + +## Arguments + +| Argument | Description | +|----------|-------------| +| `` | Path to a file containing a delegated Filecoin address private key in hex format | + +## Example + +```bash +piri wallet import wallet.hex +``` + +``` +imported wallet f410fxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx successfully! +``` + +To create a wallet file using Lotus: + +```bash +lotus wallet new delegated +lotus wallet export > wallet.hex +``` + +The wallet must be imported before running `piri init` or `piri serve`. diff --git a/docs/content/cli/wallet/index.md b/docs/content/cli/wallet/index.md new file mode 100644 index 00000000..9a68deb2 --- /dev/null +++ b/docs/content/cli/wallet/index.md @@ -0,0 +1,21 @@ +# wallet + +Manage wallet addresses. + +## Usage + +``` +piri wallet [command] +``` + +## Subcommands + +### [list](list.md) + +List all wallet addresses. + +### [import](import.md) + +Import an address to the wallet. + +You must import a wallet before you can start a Piri node. The wallet is used as the owner address for submitting PDP proofs to the blockchain. diff --git a/docs/content/cli/wallet/list.md b/docs/content/cli/wallet/list.md new file mode 100644 index 00000000..1f326470 --- /dev/null +++ b/docs/content/cli/wallet/list.md @@ -0,0 +1,19 @@ +# list + +List all wallet addresses. + +## Usage + +``` +piri wallet list +``` + +## Example + +```bash +piri wallet list +``` + +``` +Address: f410fxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +``` diff --git a/docs/content/configuration/identity.md b/docs/content/configuration/identity.md new file mode 100644 index 00000000..93ced0cd --- /dev/null +++ b/docs/content/configuration/identity.md @@ -0,0 +1,20 @@ +# identity + +Node identity configuration. + +| Key | Default | Env | Dynamic | +|----------------------|---------|--------------------------|---------| +| `identity.key_file` | - | `PIRI_IDENTITY_KEY_FILE` | No | + +## Fields + +### `key_file` + +Path to ED25519 PEM private key file. Generate with `piri identity generate`. + +## TOML + +```toml +[identity] +key_file = "/etc/piri/service.pem" +``` diff --git a/docs/content/configuration/index.md b/docs/content/configuration/index.md new file mode 100644 index 00000000..65242613 --- /dev/null +++ b/docs/content/configuration/index.md @@ -0,0 +1,93 @@ +# Configuration + +Piri uses a TOML configuration file. The `piri init` command generates a complete config file for you - this reference helps you understand and tune it. + +## Priority + +Configuration is evaluated in this order (highest to lowest priority): + +1. CLI flags +2. Environment variables +3. Config file +4. Network preset defaults +5. Built-in defaults + +## Config File Locations + +Piri searches for config files in this order: + +1. Path specified via `--config` flag +2. `~/.config/piri/config.toml` +3. `piri-config.toml` in current directory + +## Environment Variables + +Override any config value with `PIRI_` prefix, replacing dots with underscores: + +``` +repo.data_dir → PIRI_REPO_DATA_DIR +``` + +## Dynamic Configuration + +Most settings require a restart. The following can be changed at runtime via the [admin config commands](../cli/client/admin/config/index.md): + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| [`pdp.aggregation.manager.poll_interval`](pdp/aggregation/manager.md#poll_interval) | duration | `30s` | How often the aggregation manager polls for new work | +| [`pdp.aggregation.manager.batch_size`](pdp/aggregation/manager.md#batch_size) | duration | `10` | Maximum number of items to process in a single batch | + +## Minimal Example + +```toml +network = "forge-prod" + +[identity] +key_file = "/path/to/service.pem" + +[repo] +data_dir = "/data/piri" +temp_dir = "/tmp/piri" + +[server] +port = 3000 +host = "0.0.0.0" +public_url = "https://piri.example.com" + +[pdp] +owner_address = "0x..." +lotus_endpoint = "wss://lotus.example.com/rpc/v1" + +[ucan] +proof_set = 123 +``` + +## Sections + +### [network](network.md) + +Network selection (`forge-prod`, `warm-staging`). + +### [identity](identity.md) + +Node identity configuration. + +### [repo](repo/index.md) + +Storage directory configuration. + +### [server](server.md) + +HTTP server configuration. + +### [pdp](pdp/index.md) + +PDP (Provable Data Possession) configuration. + +### [ucan](ucan.md) + +UCAN service configuration. + +### [telemetry](telemetry.md) + +Observability configuration. diff --git a/docs/content/configuration/network.md b/docs/content/configuration/network.md new file mode 100644 index 00000000..837a6ffe --- /dev/null +++ b/docs/content/configuration/network.md @@ -0,0 +1,24 @@ +# network + +Network selection for connecting to Storacha. + +| Key | Default | Env | Dynamic | +|-----|---------|-----|---------| +| `network` | - | `PIRI_NETWORK` | No | + +## Overview + +The `network` field selects which Storacha network to connect to. This automatically configures service URLs, contract addresses, and chain IDs. + +## Values + +| Network | Description | +|---------|-------------| +| `forge-prod` | Production network (recommended) | +| `warm-staging` | Staging environment for testing | + +## TOML + +```toml +network = "forge-prod" +``` diff --git a/docs/content/configuration/pdp/aggregation/aggregator.md b/docs/content/configuration/pdp/aggregation/aggregator.md new file mode 100644 index 00000000..92fbea9f --- /dev/null +++ b/docs/content/configuration/pdp/aggregation/aggregator.md @@ -0,0 +1,49 @@ +# Aggregator + +Piece aggregation configuration. + +| Key | Default | Env | Dynamic | +|-----|---------|-----|---------| +| `pdp.aggregation.aggregator.job_queue.workers` | `runtime.NumCPU()` | `PIRI_PDP_AGGREGATION_AGGREGATOR_JOB_QUEUE_WORKERS` | No | +| `pdp.aggregation.aggregator.job_queue.retries` | `50` | `PIRI_PDP_AGGREGATION_AGGREGATOR_JOB_QUEUE_RETRIES` | No | +| `pdp.aggregation.aggregator.job_queue.retry_delay` | `10s` | `PIRI_PDP_AGGREGATION_AGGREGATOR_JOB_QUEUE_RETRY_DELAY` | No | + +## Overview + +The aggregator groups pieces into aggregates between 128MB and 256MB. +When a piece's CommP hash is calculated, it enters this queue to be combined with other pieces into an aggregate. + +**How aggregation works:** + +- Pieces are buffered until total size reaches 128MB minimum +- Pieces larger than 128MB are submitted immediately as single-piece aggregates +- The maximum aggregate size is 256MB + +**Performance Note:** Aggregate creation involves building merkle trees from piece commitments (32-byte hashes). +Memory usage is minimal since only the CommP hashes are held in memory, not the actual blob data. + +## Fields + +### `job_queue.workers` + +Number of concurrent aggregation operations. Defaults to the number of CPU cores. + +- **Higher values**: Faster aggregate creation when many pieces complete CommP calculation simultaneously +- **Lower values**: Reduced concurrency, but memory impact is minimal since only piece hashes are processed + +### `job_queue.retries` + +Maximum retry attempts before a piece is moved to the dead-letter queue. + +### `job_queue.retry_delay` + +Wait time between retry attempts after a failure. + +## TOML + +```toml +[pdp.aggregation.aggregator.job_queue] +workers = 2 +retries = 50 +retry_delay = "10s" +``` diff --git a/docs/content/configuration/pdp/aggregation/commp.md b/docs/content/configuration/pdp/aggregation/commp.md new file mode 100644 index 00000000..01462090 --- /dev/null +++ b/docs/content/configuration/pdp/aggregation/commp.md @@ -0,0 +1,43 @@ +# CommP + +CommP (Piece Commitment) configuration. + +| Key | Default | Env | Dynamic | +|-----|---------|-----|---------| +| `pdp.aggregation.commp.job_queue.workers` | `runtime.NumCPU()` | `PIRI_PDP_AGGREGATION_COMMP_JOB_QUEUE_WORKERS` | No | +| `pdp.aggregation.commp.job_queue.retries` | `50` | `PIRI_PDP_AGGREGATION_COMMP_JOB_QUEUE_RETRIES` | No | +| `pdp.aggregation.commp.job_queue.retry_delay` | `10s` | `PIRI_PDP_AGGREGATION_COMMP_JOB_QUEUE_RETRY_DELAY` | No | + +## Overview + +CommP calculation is the first stage of the aggregation pipeline. When a blob is uploaded, it enters this queue to have +its Piece Commitment hash calculated. This hash is required for creating aggregates that can be submitted to the +blockchain. + +**Performance Note:** CommP calculation is CPU-intensive. Each worker consumes significant CPU resources while processing a blob. + +## Fields + +### `job_queue.workers` + +Number of concurrent CommP calculations. Defaults to the number of CPU cores. + +- **Higher values**: Faster throughput when many blobs arrive simultaneously, but higher CPU usage +- **Lower values**: Reduced CPU load, but blobs queue longer during high ingest periods + +### `job_queue.retries` + +Maximum retry attempts before a blob is moved to the dead-letter queue. + +### `job_queue.retry_delay` + +Wait time between retry attempts after a failure. + +## TOML + +```toml +[pdp.aggregation.commp.job_queue] +workers = 4 # Limit to 4 cores for shared environments +retries = 50 +retry_delay = "10s" +``` diff --git a/docs/content/configuration/pdp/aggregation/index.md b/docs/content/configuration/pdp/aggregation/index.md new file mode 100644 index 00000000..1d34f34e --- /dev/null +++ b/docs/content/configuration/pdp/aggregation/index.md @@ -0,0 +1,22 @@ +# Aggregation + +The aggregation pipeline processes accepted blobs through four stages before submitting proofs to the blockchain: + +1. **CommP Calculation** - Computes the Piece Commitment/PieceCID. +2. **Aggregation** - Groups pieces into 128-256MB Aggregates. +3. **Batch Management** - Batches aggregates for efficient chain submission. +4. **Chain Submission** - Submits roots to the PDP smart contract(s). + +## Subsections + +### [commp](commp.md) + +CommP calculation job queue configuration. + +### [aggregator](aggregator.md) + +Piece aggregation job queue configuration. + +### [manager](manager.md) + +Batch management and chain submission configuration. diff --git a/docs/content/configuration/pdp/aggregation/manager.md b/docs/content/configuration/pdp/aggregation/manager.md new file mode 100644 index 00000000..d9ab0a5b --- /dev/null +++ b/docs/content/configuration/pdp/aggregation/manager.md @@ -0,0 +1,96 @@ +# Manager + +Aggregation manager configuration. + +| Key | Default | Env | Dynamic | +|-------------------------------------------------|--------------------|------------------------------------------------------|---------| +| `pdp.aggregation.manager.poll_interval` | `30s` | `PIRI_PDP_AGGREGATION_MANAGER_POLL_INTERVAL` | Yes | +| `pdp.aggregation.manager.batch_size` | `10` | `PIRI_PDP_AGGREGATION_MANAGER_BATCH_SIZE` | Yes | +| `pdp.aggregation.manager.job_queue.workers` | `runtime.NumCPU()` | `PIRI_PDP_AGGREGATION_MANAGER_JOB_QUEUE_WORKERS` | No | +| `pdp.aggregation.manager.job_queue.retries` | `50` | `PIRI_PDP_AGGREGATION_MANAGER_JOB_QUEUE_RETRIES` | No | +| `pdp.aggregation.manager.job_queue.retry_delay` | `10s` | `PIRI_PDP_AGGREGATION_MANAGER_JOB_QUEUE_RETRY_DELAY` | No | + +## Overview + +The manager is the final stage of the aggregation pipeline before chain submission. +It buffers completed aggregates and submits them to the blockchain in batches. + +**How it works:** + +- Aggregates are buffered as they complete +- When `poll_interval` elapses OR `batch_size` is reached, the buffer is submitted +- Submissions are enqueued to a job queue for reliable delivery + +**Dynamic Configuration:** Both `poll_interval` and `batch_size` can be changed at runtime using the admin API. +Changes take effect immediately. + +## Fields + +### `poll_interval` +The frequency in which the aggregation manager submits roots to the chain. + +### `batch_size` +The maximum number of roots to submit in a single message. + +### `job_queue.workers` +The number of workers spawned by the manager controlling the number of roots that may be submitted in parallel. + +### `job_queue.retries` +The number of times to retry submitting a root before the operation is considered failed. + +### `job_queue.retry_delay` + +The duration to wait between successive failures. + +## Gas Usage + +The manager configuration directly affects gas costs for chain submission. + +### How batching reduces gas + +Each blockchain transaction has: + +- **Fixed costs**: Transaction overhead, signature verification +- **Variable costs**: Per-root data storage + +By batching multiple roots into a single transaction, you amortize fixed costs across more roots. + +### Configuration trade-offs + +| Setting | Effect | Gas Impact | +|---------|--------|------------| +| Higher `batch_size` | More roots per transaction | Lower gas per root | +| Lower `batch_size` | Fewer roots per transaction | Higher gas per root | +| Higher `poll_interval` | More time to fill batches | Better batching, lower gas | +| Lower `poll_interval` | Less time to fill batches | Faster finality, potentially higher gas | + +### Recommendations + +**High-volume nodes** (many blobs/hour): + +- Use default or higher `batch_size` (10+) +- Use moderate `poll_interval` (30s-60s) +- Batches will naturally fill, optimizing gas + +**Low-volume nodes** (few blobs/hour): + +- Consider higher `poll_interval` (60s-300s) to allow batches to fill +- Or accept higher per-root gas cost for faster finality + +**Cost-sensitive deployments**: + +- Maximize `batch_size` and `poll_interval` +- Trade finality speed for gas efficiency + +## TOML + +```toml +[pdp.aggregation.manager] +poll_interval = "30s" +batch_size = 10 + +[pdp.aggregation.manager.job_queue] +workers = 3 +retries = 50 +retry_delay = "1m" +``` diff --git a/docs/content/configuration/pdp/index.md b/docs/content/configuration/pdp/index.md new file mode 100644 index 00000000..e74d2ade --- /dev/null +++ b/docs/content/configuration/pdp/index.md @@ -0,0 +1,81 @@ +# pdp + +PDP (Provable Data Possession) service configuration. + +| Key | Default | Env | Dynamic | +|-----|---------|-----|---------| +| `pdp.owner_address` | - | `PIRI_PDP_OWNER_ADDRESS` | No | +| `pdp.lotus_endpoint` | - | `PIRI_PDP_LOTUS_ENDPOINT` | No | + +## Fields + +### `owner_address` + +Ethereum address that owns the proof set. + +### `lotus_endpoint` + +Filecoin Lotus node WebSocket endpoint. + +## TOML + +```toml +[pdp] +owner_address = "0x1234..." +lotus_endpoint = "wss://lotus.example.com/rpc/v1" +``` + +### [aggregation](aggregation/index.md) + +Aggregation system configuration. + +
+Preset-Managed Fields + +These fields are automatically configured by the `network` preset. You typically don't need to set them manually. See [presets](../../presets.md) for details. + +| Key | Default | Env | Dynamic | +|-----|---------|-----|---------| +| `pdp.chain_id` | - | `PIRI_PDP_CHAIN_ID` | No | +| `pdp.payer_address` | - | `PIRI_PDP_PAYER_ADDRESS` | No | + +### `chain_id` + +Filecoin chain ID. `314` for mainnet, `314159` for calibration. + +### `payer_address` + +Ethereum address that pays storage providers. + +## [pdp.signing_service] + +Configure transaction signing. Use either remote signing service OR local private key (not both). + +**Remote signing (recommended for production):** + +```toml +[pdp.signing_service] +did = "did:web:signer.forge.storacha.network" +url = "https://signer.forge.storacha.network" +``` + +**Local signing (development only):** + +```toml +[pdp.signing_service] +private_key = "0x..." # Hex-encoded ECDSA private key +``` + +## [pdp.contracts] + +Smart contract addresses. + +```toml +[pdp.contracts] +verifier = "0x..." +provider_registry = "0x..." +service = "0x..." +service_view = "0x..." +``` + +
diff --git a/docs/content/configuration/repo/blob-storage.md b/docs/content/configuration/repo/blob-storage.md new file mode 100644 index 00000000..44749ae5 --- /dev/null +++ b/docs/content/configuration/repo/blob-storage.md @@ -0,0 +1,14 @@ +# blob-storage + +Configure external S3-compatible storage for blobs. This section is optional. + +```toml +[repo.blob_storage.minio] +endpoint = "s3.example.com:9000" +bucket = "piri-blobs" +insecure = false + +[repo.blob_storage.minio.credentials] +access_key_id = "..." +secret_access_key = "..." +``` diff --git a/docs/content/configuration/repo/index.md b/docs/content/configuration/repo/index.md new file mode 100644 index 00000000..06521c24 --- /dev/null +++ b/docs/content/configuration/repo/index.md @@ -0,0 +1,31 @@ +# repo + +Storage directory configuration. + +| Key | Default | Env | Dynamic | +|-----|-------------------|-----|---------| +| `repo.data_dir` | `$HOME/.storacha/` | `PIRI_REPO_DATA_DIR` | No | +| `repo.temp_dir` | - | `PIRI_REPO_TEMP_DIR` | No | + +## Fields + +### `data_dir` + +Directory for persistent data (databases, blobs, claims). + +### `temp_dir` + +Directory for temporary files during processing. + +### [`blob-storage`](blob-storage.md) + +External S3-compatible storage configuration. + +## TOML + +```toml +[repo] +data_dir = "/data/piri" +temp_dir = "/tmp/piri" +``` + diff --git a/docs/content/configuration/server.md b/docs/content/configuration/server.md new file mode 100644 index 00000000..4fa5554a --- /dev/null +++ b/docs/content/configuration/server.md @@ -0,0 +1,32 @@ +# server + +HTTP server configuration. + +| Key | Default | Env | Dynamic | +|---------------------|------------------------|--------------------------|---------| +| `server.port` | `3000` | `PIRI_SERVER_PORT` | No | +| `server.host` | `0.0.0.0` | `PIRI_SERVER_HOST` | No | +| `server.public_url` | `http://{host}:{port}` | `PIRI_SERVER_PUBLIC_URL` | No | + +## Fields + +### `port` + +HTTP listening port (1-65535). + +### `host` + +HTTP listening host/interface. + +### `public_url` + +Externally accessible URL. Defaults to `http://{host}:{port}` if not set. + +## TOML + +```toml +[server] +port = 3000 +host = "0.0.0.0" +public_url = "https://piri.example.com" +``` diff --git a/docs/content/configuration/telemetry.md b/docs/content/configuration/telemetry.md new file mode 100644 index 00000000..e5b6d6ab --- /dev/null +++ b/docs/content/configuration/telemetry.md @@ -0,0 +1,40 @@ +# telemetry + +Observability configuration. + +| Key | Default | Env | Dynamic | +|-----|---------|-----|---------| +| `telemetry.disable_storacha_analytics` | `false` | `PIRI_TELEMETRY_DISABLE_STORACHA_ANALYTICS` | No | + +## Fields + +### `disable_storacha_analytics` + +Disable sending analytics to Storacha. + +## [telemetry.metrics] + +Configure metrics collectors (array). + +```toml +[telemetry.metrics] +endpoint = "https://otel.example.com:4317" +insecure = false +publish_interval = "30s" + +[telemetry.metrics.headers] +Authorization = "Bearer ..." +``` + +## [telemetry.traces] + +Configure trace collectors (array). + +```toml +[telemetry.traces] +endpoint = "https://otel.example.com:4317" +insecure = false + +[telemetry.traces.headers] +Authorization = "Bearer ..." +``` diff --git a/docs/content/configuration/ucan.md b/docs/content/configuration/ucan.md new file mode 100644 index 00000000..4a2b05b1 --- /dev/null +++ b/docs/content/configuration/ucan.md @@ -0,0 +1,88 @@ +# ucan + +UCAN service configuration. + +| Key | Default | Env | Dynamic | +|-----|---------|-----|---------| +| `ucan.proof_set` | - | `PIRI_UCAN_PROOF_SET` | No | + +## Fields + +### `proof_set` + +Proof set ID from initialization. + +## TOML + +```toml +[ucan] +proof_set = 123 +``` + +
+Preset-Managed Fields + +These fields are automatically configured by the `network` preset. You typically don't need to set them manually. See [presets](../presets.md) for details. + +## [ucan.services] + +External service connections. + +### [ucan.services.indexer] + +```toml +[ucan.services.indexer] +did = "did:web:indexer.forge.storacha.network" +url = "https://indexer.forge.storacha.network/claims" +proof = "..." # Optional delegation proof +``` + +### [ucan.services.upload] + +```toml +[ucan.services.upload] +did = "did:web:up.forge.storacha.network" +url = "https://up.forge.storacha.network" +``` + +### [ucan.services.etracker] + +Egress tracker for usage reporting. + +| Key | Default | Env | Dynamic | +|-----|---------|-----|---------| +| `ucan.services.etracker.max_batch_size_bytes` | `104857600` | `PIRI_UCAN_SERVICES_ETRACKER_MAX_BATCH_SIZE_BYTES` | No | + +Constraints: 10MiB - 1GiB + +```toml +[ucan.services.etracker] +did = "did:web:etracker.forge.storacha.network" +url = "https://etracker.forge.storacha.network" +receipts_endpoint = "https://..." +max_batch_size_bytes = 104857600 # 100 MiB +proof = "..." # Optional +``` + +### [ucan.services.publisher] + +IPNI announcement configuration. + +```toml +[ucan.services.publisher] +ipni_announce_urls = [ + "https://cid.contact/announce", + "https://ipni.forge.storacha.network" +] +``` + +### [ucan.services.principal_mapping] + +Maps service DIDs to principal DIDs. + +```toml +[ucan.services] +principal_mapping = { "did:web:up.forge.storacha.network" = "did:key:z6Mk..." } +``` + +
diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 4985f67b..5309b89b 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -17,6 +17,7 @@ theme: name: material logo: images/logo-header.png features: + - navigation.indexes - navigation.top - navigation.footer - navigation.tracking @@ -52,8 +53,61 @@ nav: - Key Generation: setup/key-generation.md - TLS Termination: setup/tls-termination.md - Piri Server: setup/piri-server.md + - Configuration: + - configuration/index.md + - network: configuration/network.md + - identity: configuration/identity.md + - repo: + - configuration/repo/index.md + - blob-storage: configuration/repo/blob-storage.md + - server: configuration/server.md + - pdp: + - configuration/pdp/index.md + - aggregation: + - configuration/pdp/aggregation/index.md + - commp: configuration/pdp/aggregation/commp.md + - aggregator: configuration/pdp/aggregation/aggregator.md + - manager: configuration/pdp/aggregation/manager.md + - ucan: configuration/ucan.md + - telemetry: configuration/telemetry.md - Operations: - Inspect Proof Set: operations/inspect-proof-set.md - Best Practices: operations/best-practices.md - Upgrading: operations/upgrading.md - Telemetry: operations/telemetry.md + - CLI Reference: + - cli/index.md + - init: cli/init.md + - serve: cli/serve/index.md + - status: + - cli/status/index.md + - upgrade-check: cli/status/upgrade-check.md + - update: cli/update.md + - wallet: + - cli/wallet/index.md + - list: cli/wallet/list.md + - import: cli/wallet/import.md + - identity: + - cli/identity/index.md + - generate: cli/identity/generate.md + - parse: cli/identity/parse.md + - client: + - cli/client/index.md + - admin: + - cli/client/admin/index.md + - config: + - cli/client/admin/config/index.md + - list: cli/client/admin/config/list.md + - get: cli/client/admin/config/get.md + - set: cli/client/admin/config/set.md + - reload: cli/client/admin/config/reload.md + - log: + - cli/client/admin/log/index.md + - list: cli/client/admin/log/list.md + - set: cli/client/admin/log/set.md + - set-regex: cli/client/admin/log/set-regex.md + - pdp: + - cli/client/pdp/index.md + - proofset: + - cli/client/pdp/proofset/index.md + - state: cli/client/pdp/proofset/state.md From d44f547431fdfcefe7d20e026e275a81009fa908 Mon Sep 17 00:00:00 2001 From: frrist Date: Tue, 17 Feb 2026 12:48:40 -0800 Subject: [PATCH 2/2] final touch, include telemetry details --- docs/content/cli/client/admin/config/get.md | 4 +- docs/content/cli/client/admin/config/index.md | 54 ++++++ docs/content/cli/client/admin/config/list.md | 4 +- docs/content/concepts/database.md | 87 +++++++++ docs/content/concepts/index.md | 19 ++ docs/content/concepts/networks.md | 64 +++++++ docs/content/concepts/telemetry.md | 173 ++++++++++++++++++ docs/content/configuration/pdp/index.md | 8 +- .../configuration/repo/blob-storage.md | 14 -- docs/content/configuration/repo/database.md | 59 ++++++ docs/content/configuration/repo/index.md | 4 - docs/content/configuration/telemetry.md | 54 ++++-- docs/mkdocs.yml | 7 +- .../storage/handlers/replica/telemetry.go | 7 +- .../storage/handlers/replica/transfer.go | 12 +- 15 files changed, 519 insertions(+), 51 deletions(-) create mode 100644 docs/content/concepts/database.md create mode 100644 docs/content/concepts/index.md create mode 100644 docs/content/concepts/networks.md create mode 100644 docs/content/concepts/telemetry.md delete mode 100644 docs/content/configuration/repo/blob-storage.md create mode 100644 docs/content/configuration/repo/database.md diff --git a/docs/content/cli/client/admin/config/get.md b/docs/content/cli/client/admin/config/get.md index 2a478250..c1cdbcb2 100644 --- a/docs/content/cli/client/admin/config/get.md +++ b/docs/content/cli/client/admin/config/get.md @@ -1,6 +1,8 @@ # get -Get a specific dynamic configuration value. +Get the current runtime value of a dynamic configuration key. + +This is the value Piri is using right now, which may differ from the config file if a runtime override has been applied via `set`. ## Usage diff --git a/docs/content/cli/client/admin/config/index.md b/docs/content/cli/client/admin/config/index.md index caea485c..cc34d157 100644 --- a/docs/content/cli/client/admin/config/index.md +++ b/docs/content/cli/client/admin/config/index.md @@ -2,6 +2,60 @@ Manage dynamic configuration values at runtime. +## How It Works + +The config file is the source of truth. Runtime changes are ephemeral by default—they override behavior without touching the file. A reload restores the file's authority. The `--persist` flag exists for operators who mean it: change the behavior and update the file so the two agree. + +This prevents the most common configuration surprise: a system whose running state has silently diverged from its config file, discovered during a restart nobody planned for. + +**Example workflow:** + +A Piri node starts with this config file: + +```toml +[pdp.aggregation.manager] +poll_interval = "30s" +``` + +The node uses a poll interval of 30 seconds. + +**Runtime override (ephemeral):** + +``` +piri client admin config set pdp.aggregation.manager.poll_interval 5m +``` + +The node now uses 5 minutes. The config file is unchanged. + +**Reload from file:** + +``` +piri client admin config reload +``` + +The node returns to 30 seconds. The file wins; runtime overrides do not survive a reload. + +**Edit the file, then reload:** + +```toml +[pdp.aggregation.manager] +poll_interval = "5m" +``` + +``` +piri client admin config reload +``` + +The node uses 5 minutes. The file still wins—it just agrees with you now. + +**Runtime override with `--persist`:** + +``` +piri client admin config set pdp.aggregation.manager.poll_interval 10m --persist +``` + +The node uses 10 minutes and the config file is updated to match. Future reloads will preserve the change. + ## Usage ``` diff --git a/docs/content/cli/client/admin/config/list.md b/docs/content/cli/client/admin/config/list.md index 0f8e327a..4a7fe91c 100644 --- a/docs/content/cli/client/admin/config/list.md +++ b/docs/content/cli/client/admin/config/list.md @@ -1,6 +1,8 @@ # list -List all dynamic configuration values. +List the current runtime values of all dynamic configuration keys. + +These are the values Piri is using right now. They may differ from the config file if runtime overrides have been applied via `set`. ## Usage diff --git a/docs/content/concepts/database.md b/docs/content/concepts/database.md new file mode 100644 index 00000000..9f64154b --- /dev/null +++ b/docs/content/concepts/database.md @@ -0,0 +1,87 @@ +# Database + +Piri uses databases to manage operational state, job queues, and task scheduling. This page explains the database architecture and helps you choose the right backend for your deployment. + +## Logical Databases + +Piri maintains four logical databases, each serving a distinct purpose: + +| Database | Purpose | +|----------|---------| +| **Scheduler** | Task engine state and PDP proof scheduling | +| **Replicator** | Data replication job tracking | +| **Aggregator** | CommP hash aggregation job queue | +| **Egress Tracker** | Data egress operation tracking | + +## SQLite Mode (Default) + +SQLite is the default database backend, requiring no additional configuration. + +### File Locations + +When using SQLite, Piri creates separate database files under your configured `data_dir`: + +``` +{data_dir}/ +├── replicator/ +│ └── replicator.db +├── pdp/ +│ └── state/ +│ └── state.db +├── aggregator/ +│ └── jobqueue/ +│ └── jobqueue.db +└── egress_tracker/ + └── jobqueue/ + └── jobqueue.db +``` + +### Characteristics + +- **Write-Ahead Logging (WAL)**: Enables concurrent read access while writing +- **Single Writer**: SQLite only supports one writer at a time (1 connection per database) +- **Zero Configuration**: Works out of the box with just `data_dir` set + +## PostgreSQL Mode + +PostgreSQL mode uses a single database instance with separate schemas for each logical database. + +### Schema Layout + +All four logical databases share one PostgreSQL database, isolated by schema: + +- `replicator` schema +- `scheduler` schema +- `aggregator` schema +- `egress_tracker` schema + +### Characteristics + +- **Connection Pooling**: Configurable max connections and idle pool size +- **Concurrent Writers**: Supports multiple simultaneous writers +- **External Management**: Database runs separately from Piri + +### Configuration + +See [Database Configuration](../configuration/repo/database.md) for PostgreSQL setup. + +## Choosing a Backend + +| Consideration | SQLite | PostgreSQL | +|---------------|--------|------------| +| Setup complexity | None | Requires external database | +| Concurrent writers | Single | Multiple | +| Scaling | Single node | Multiple Piri instances | +| Operational overhead | Low | Higher | +| Backup strategy | File copy | Database dumps | + +**Recommended:** + +- **SQLite**: Development, testing, single-node deployments +- **PostgreSQL**: Production environments, high availability requirements, multiple Piri instances sharing state + +## Backend Switching + +> **Warning**: You cannot switch database backends after initial setup. There is no migration path between SQLite and PostgreSQL. + +Choose your database backend before storing any data. If you need to switch backends later, you must start with a fresh Piri installation. diff --git a/docs/content/concepts/index.md b/docs/content/concepts/index.md new file mode 100644 index 00000000..e58d74e2 --- /dev/null +++ b/docs/content/concepts/index.md @@ -0,0 +1,19 @@ +# Concepts + +Understanding how Piri works internally. + +This section covers the architectural concepts and design decisions behind Piri's components. + +## Topics + +### [Database](database.md) + +How Piri uses databases for operational state, the difference between SQLite and PostgreSQL backends, and guidance on choosing the right backend for your deployment. + +### [Networks](networks.md) + +Storacha networks that Piri operates on, including service endpoints, smart contract addresses, and chain configuration. + +### [Telemetry](telemetry.md) + +OpenTelemetry-based observability: metrics and traces emitted by Piri, and how to integrate with your monitoring infrastructure. diff --git a/docs/content/concepts/networks.md b/docs/content/concepts/networks.md new file mode 100644 index 00000000..e28a75a6 --- /dev/null +++ b/docs/content/concepts/networks.md @@ -0,0 +1,64 @@ +# Networks + +Piri operates on Storacha networks, which define the services and smart contracts it communicates with. + +## forge-prod + +Production network on Filecoin Mainnet. + +**Chain ID:** 314 + +### Services + +| Service | URL | DID | +|----------------|--------------------------------------------|-------------------------------------------| +| Upload | `https://up.forge.storacha.network` | `did:web:up.forge.storacha.network` | +| Indexing | `https://indexer.forge.storacha.network` | `did:web:indexer.forge.storacha.network` | +| Egress Tracker | `https://etracker.forge.storacha.network` | `did:web:etracker.forge.storacha.network` | +| Signing | `https://signer.forge.storacha.network` | `did:web:signer.forge.storacha.network` | +| Registrar | `https://registrar.forge.storacha.network` | - | +| IPNI | `https://ipni.forge.storacha.network` | - | + +### Smart Contracts + +| Contract | Address | +|-------------------|------------------------------------------------------------------------------------------------------------------------------------| +| PDP Verifier | [`0xBADd0B92C1c71d02E7d520f64c0876538fa2557F`](https://filecoin.blockscout.com/address/0xBADd0B92C1c71d02E7d520f64c0876538fa2557F) | +| Provider Registry | [`0xf55dDbf63F1b55c3F1D4FA7e339a68AB7b64A5eB`](https://filecoin.blockscout.com/address/0xf55dDbf63F1b55c3F1D4FA7e339a68AB7b64A5eB) | +| PDP Service | [`0x56e53c5e7F27504b810494cc3b88b2aa0645a839`](https://filecoin.blockscout.com/address/0x56e53c5e7F27504b810494cc3b88b2aa0645a839) | +| PDP Service View | [`0x778Bbb8F50d759e2AA03ab6FAEF830Ca329AFF9D`](https://filecoin.blockscout.com/address/0x778Bbb8F50d759e2AA03ab6FAEF830Ca329AFF9D) | +| Payments | [`0x23b1e018F08BB982348b15a86ee926eEBf7F4DAa`](https://filecoin.blockscout.com/address/0x23b1e018F08BB982348b15a86ee926eEBf7F4DAa) | +| USDFC Token | [`0x80B98d3aa09ffff255c3ba4A241111Ff1262F045`](https://filecoin.blockscout.com/address/0x80B98d3aa09ffff255c3ba4A241111Ff1262F045) | + +**Payer Address (Storacha) :** [`0x3c1ae7a70a2b51458fcb7927fd77aae408a1b857`](https://filecoin.blockscout.com/address/0x3c1ae7a70a2b51458fcb7927fd77aae408a1b857) + +## warm-staging + +Staging network on Filecoin Calibration testnet. Use this for testing before deploying to production. + +**Chain ID:** 314159 + +### Services + +| Service | URL | DID | +|----------------|---------------------------------------------------|--------------------------------------------------| +| Upload | `https://staging.up.warm.storacha.network` | `did:web:staging.up.warm.storacha.network` | +| Indexing | `https://staging.indexer.warm.storacha.network` | `did:web:staging.indexer.warm.storacha.network` | +| Egress Tracker | `https://staging.etracker.warm.storacha.network` | `did:web:staging.etracker.warm.storacha.network` | +| Signing | `https://staging.signer.warm.storacha.network` | `did:web:staging.signer.warm.storacha.network` | +| Registrar | `https://staging.registrar.warm.storacha.network` | - | +| IPNI | `https://staging.ipni.warm.storacha.network` | - | + +### Smart Contracts + +| Contract | Address | +|-------------------|--------------------------------------------------------------------------------------------------------------------------------------------| +| PDP Verifier | [`0x85e366Cf9DD2c0aE37E963d9556F5f4718d6417C`](https://filecoin-testnet.blockscout.com/address/0x85e366Cf9DD2c0aE37E963d9556F5f4718d6417C) | +| Provider Registry | [`0x6A96aaB210B75ee733f0A291B5D8d4A053643979`](https://filecoin-testnet.blockscout.com/address/0x6A96aaB210B75ee733f0A291B5D8d4A053643979) | +| PDP Service | [`0x0c6875983B20901a7C3c86871f43FdEE77946424`](https://filecoin-testnet.blockscout.com/address/0x0c6875983B20901a7C3c86871f43FdEE77946424) | +| PDP Service View | [`0xEAD67d775f36D1d2894854D20e042C77A3CC20a5`](https://filecoin-testnet.blockscout.com/address/0xEAD67d775f36D1d2894854D20e042C77A3CC20a5) | +| Payments | [`0x09a0fDc2723fAd1A7b8e3e00eE5DF73841df55a0`](https://filecoin-testnet.blockscout.com/address/0x09a0fDc2723fAd1A7b8e3e00eE5DF73841df55a0) | +| USDFC Token | [`0xb3042734b608a1B16e9e86B374A3f3e389B4cDf0`](https://filecoin-testnet.blockscout.com/address/0xb3042734b608a1B16e9e86B374A3f3e389B4cDf0) | + +**Payer Address (Storacha):** [`0x8d3d7cE4F43607C9d0ac01f695c7A9caC135f9AD`](https://filecoin-testnet.blockscout.com/address/0x8d3d7cE4F43607C9d0ac01f695c7A9caC135f9AD) + diff --git a/docs/content/concepts/telemetry.md b/docs/content/concepts/telemetry.md new file mode 100644 index 00000000..51cdd5e6 --- /dev/null +++ b/docs/content/concepts/telemetry.md @@ -0,0 +1,173 @@ +# Telemetry + +Piri uses [OpenTelemetry](https://opentelemetry.io/) to emit metrics and traces for observability. You can configure custom collectors to send this data to your own monitoring infrastructure. + +## Metrics + +Piri emits metrics via OTLP (OpenTelemetry Protocol) that can be consumed by any compatible collector. + +### Host Metrics + +System-level metrics for monitoring node health: + +| Metric | Type | Unit | Description | +|------------------------------------------|-------|-------|-------------------------------------| +| `system_cpu_utilization` | Gauge | 0-1 | System-wide CPU utilization | +| `system_memory_used_bytes` | Gauge | bytes | System memory in use | +| `system_memory_total_bytes` | Gauge | bytes | Total system memory | +| `piri_datadir_used_bytes` | Gauge | bytes | Disk space used by data directory | +| `piri_datadir_free_bytes` | Gauge | bytes | Free disk space for data directory | +| `piri_datadir_total_bytes` | Gauge | bytes | Total disk space for data directory | + +### Job Queue Metrics + +Track task execution in internal job queues: + +| Metric | Type | Description | +|-----------------------------|---------------|----------------------------------| +| `active_jobs` | UpDownCounter | Currently running jobs | +| `queued_jobs` | UpDownCounter | Jobs waiting in queue | +| `failed_jobs` | Counter | Permanently failed jobs | +| `job_duration` | Histogram | Job execution duration (seconds) | + +**Labels:** + +| Label | Description | +|-------|-------------| +| `queue` | Name of the job queue (e.g., `replicator`, `aggregator`, `egress_tracker`) | +| `job` | Type of job being executed | +| `status` | Job outcome (`success` or `failure`) | +| `attempt` | Retry attempt number (1-based) | +| `failure_reason` | Reason for permanent failure (only on `failed_jobs`) | + +### HTTP Server Metrics + +Standard OpenTelemetry HTTP instrumentation: + +| Metric | Type | Description | +|-----------------------------------------------|-----------|--------------------| +| `http.server.request.duration` | Histogram | Request latency | +| `http.server.request.body.size` | Histogram | Request body size | +| `http.server.response.body.size` | Histogram | Response body size | + +### PDP Metrics + +Provable Data Possession task metrics: + +| Metric | Type | Description | +|--------------------------------------------------|---------|-----------------------------------------| +| `chain_current_epoch` | Gauge | Current Filecoin chain epoch | +| `next_challenge_window_start_epoch` | Gauge | Epoch when next challenge window starts | +| `pdp_next_failure` | Counter | Next proving period task failures | +| `pdp_prove_failure` | Counter | Proof generation task failures | +| `message_send_failure` | Counter | Blockchain message send failures | +| `message_estimate_gas_failure` | Counter | Gas estimation failures | + +### Replication Metrics + +| Metric | Type | Description | +|----------------------------------|-----------|-------------------------------------| +| `transfer_duration` | Histogram | Replica transfer operation duration | + +**Labels:** + +| Label | Description | +|-------|-------------| +| `source` | Origin endpoint where data is pulled from | +| `sink` | Destination endpoint where data is written to (this node) | + +### Server Info + +Build and runtime information: + +| Metric | Type | Description | +|---------------------------------|------|-----------------| +| `piri_server_info` | Info | Server metadata | + +**Labels:** + +| Label | Description | +|-------|-------------| +| `version` | Piri software version | +| `commit` | Git commit hash of the build | +| `built_by` | Build system identifier | +| `build_date` | When the binary was compiled | +| `start_time_unix` | Server start time (Unix timestamp) | +| `server_type` | Server mode (`full` or `ucan`) | +| `did` | Server's Decentralized Identifier | +| `owner_address` | Ethereum address of node owner | +| `public_url` | Server's publicly accessible URL | +| `proof_set` | PDP proof set ID | + +## Traces + +Distributed tracing provides end-to-end visibility into operations: + +| Span | Description | +|---------------------------------------|------------------------------| +| `blob.accept` | Blob acceptance operations | +| `blob.allocate` | Blob allocation operations | +| `space.content.retrieve` | Content retrieval operations | +| `AddRoots` | PDP root addition operations | + +Traces use parent-based sampling and integrate with W3C Trace Context propagation. + +## Integration + +### Prometheus + +Use an [OpenTelemetry Collector](https://opentelemetry.io/docs/collector/) with a Prometheus exporter: + +```yaml +# otel-collector-config.yaml +receivers: + otlp: + protocols: + http: + endpoint: "0.0.0.0:4317" + +exporters: + prometheus: + endpoint: "0.0.0.0:9090" + +service: + pipelines: + metrics: + receivers: [otlp] + exporters: [prometheus] +``` + +Configure Piri to send metrics to your collector: + +```toml +[[telemetry.metrics]] +endpoint = "http://localhost:4317" +insecure = true +publish_interval = "30s" +``` + +### Jaeger + +For distributed tracing, configure a Jaeger backend with OTLP support: + +```toml +[[telemetry.traces]] +endpoint = "http://jaeger:4317" +insecure = true +``` + +### Grafana + +Connect your Prometheus datasource and create dashboards using the metrics above. Key metrics to monitor: + +- **System health**: `system_cpu_utilization`, `system_memory_used_bytes`, `piri_datadir_free_bytes` +- **Job queue health**: `active_jobs`, `failed_jobs`, `job_duration` +- **API performance**: `http.server.request.duration` (p95, p99) + +## Configuration + +See [Configuration > telemetry](../configuration/telemetry.md) for collector setup options. + +## Analytics + +Piri can optionally send anonymized analytics to Storacha to help improve the software. See [Operations > Telemetry](../operations/telemetry.md) for details and opt-out instructions. diff --git a/docs/content/configuration/pdp/index.md b/docs/content/configuration/pdp/index.md index e74d2ade..4ec28e25 100644 --- a/docs/content/configuration/pdp/index.md +++ b/docs/content/configuration/pdp/index.md @@ -29,10 +29,10 @@ lotus_endpoint = "wss://lotus.example.com/rpc/v1" Aggregation system configuration. -
+
Preset-Managed Fields -These fields are automatically configured by the `network` preset. You typically don't need to set them manually. See [presets](../../presets.md) for details. +These fields are automatically configured by the `network` preset. You almost never should set them manually. See [Networks](../../concepts/networks.md) for details. | Key | Default | Env | Dynamic | |-----|---------|-----|---------| @@ -47,7 +47,7 @@ Filecoin chain ID. `314` for mainnet, `314159` for calibration. Ethereum address that pays storage providers. -## [pdp.signing_service] +## pdp.signing_service Configure transaction signing. Use either remote signing service OR local private key (not both). @@ -66,7 +66,7 @@ url = "https://signer.forge.storacha.network" private_key = "0x..." # Hex-encoded ECDSA private key ``` -## [pdp.contracts] +## pdp.contracts Smart contract addresses. diff --git a/docs/content/configuration/repo/blob-storage.md b/docs/content/configuration/repo/blob-storage.md deleted file mode 100644 index 44749ae5..00000000 --- a/docs/content/configuration/repo/blob-storage.md +++ /dev/null @@ -1,14 +0,0 @@ -# blob-storage - -Configure external S3-compatible storage for blobs. This section is optional. - -```toml -[repo.blob_storage.minio] -endpoint = "s3.example.com:9000" -bucket = "piri-blobs" -insecure = false - -[repo.blob_storage.minio.credentials] -access_key_id = "..." -secret_access_key = "..." -``` diff --git a/docs/content/configuration/repo/database.md b/docs/content/configuration/repo/database.md new file mode 100644 index 00000000..e6969617 --- /dev/null +++ b/docs/content/configuration/repo/database.md @@ -0,0 +1,59 @@ +# database + +Database backend configuration. + +| Key | Default | Env | Dynamic | +|-----|---------|-----|---------| +| `repo.database.type` | `sqlite` | `PIRI_REPO_DATABASE_TYPE` | No | +| `repo.database.postgres.url` | - | `PIRI_REPO_DATABASE_POSTGRES_URL` | No | +| `repo.database.postgres.max_open_conns` | `5` | `PIRI_REPO_DATABASE_POSTGRES_MAX_OPEN_CONNS` | No | +| `repo.database.postgres.max_idle_conns` | `5` | `PIRI_REPO_DATABASE_POSTGRES_MAX_IDLE_CONNS` | No | +| `repo.database.postgres.conn_max_lifetime` | `30m` | `PIRI_REPO_DATABASE_POSTGRES_CONN_MAX_LIFETIME` | No | + +## Fields + +### `type` + +Database backend: `sqlite` (default) or `postgres`. + +> **Important**: Database type cannot be changed after initial setup. Data is not migrated between backends. See [Database Concepts](../../concepts/database.md) for details. + +### `postgres.url` + +PostgreSQL connection string. Required when `type` is `postgres`. + +Format: `postgres://user:password@host:port/dbname?sslmode=disable` + +### `postgres.max_open_conns` + +Maximum number of open connections to the database. + +### `postgres.max_idle_conns` + +Maximum number of idle connections in the pool. + +### `postgres.conn_max_lifetime` + +Maximum lifetime for a connection. Accepts Go duration strings (e.g., `30m`, `1h`). + +## TOML + +SQLite (default - no configuration needed): + +```toml +[repo] +data_dir = "/data/piri" +``` + +PostgreSQL: + +```toml +[repo.database] +type = "postgres" + +[repo.database.postgres] +url = "postgres://piri:secret@localhost:5432/piri?sslmode=disable" +max_open_conns = 10 +max_idle_conns = 5 +conn_max_lifetime = "30m" +``` diff --git a/docs/content/configuration/repo/index.md b/docs/content/configuration/repo/index.md index 06521c24..405b60a9 100644 --- a/docs/content/configuration/repo/index.md +++ b/docs/content/configuration/repo/index.md @@ -17,10 +17,6 @@ Directory for persistent data (databases, blobs, claims). Directory for temporary files during processing. -### [`blob-storage`](blob-storage.md) - -External S3-compatible storage configuration. - ## TOML ```toml diff --git a/docs/content/configuration/telemetry.md b/docs/content/configuration/telemetry.md index e5b6d6ab..b4652011 100644 --- a/docs/content/configuration/telemetry.md +++ b/docs/content/configuration/telemetry.md @@ -1,40 +1,54 @@ # telemetry -Observability configuration. +Observability configuration for metrics and distributed tracing. -| Key | Default | Env | Dynamic | -|-----|---------|-----|---------| -| `telemetry.disable_storacha_analytics` | `false` | `PIRI_TELEMETRY_DISABLE_STORACHA_ANALYTICS` | No | +| Key | Default | Env | Dynamic | +|----------------------------------------|---------|---------------------------------------------|---------| +| `telemetry.disable_storacha_analytics` | `false` | `PIRI_TELEMETRY_DISABLE_STORACHA_ANALYTICS` | No | ## Fields ### `disable_storacha_analytics` -Disable sending analytics to Storacha. +Disable sending analytics to Storacha. See [Operations > Telemetry](../operations/telemetry.md) for details on what data is collected. -## [telemetry.metrics] +### `metrics` -Configure metrics collectors (array). +Array of OTLP metrics collector configurations. Each entry supports: -```toml -[telemetry.metrics] -endpoint = "https://otel.example.com:4317" -insecure = false -publish_interval = "30s" +| Field | Required | Description | +|--------------------|----------|------------------------------------------------------------| +| `endpoint` | Yes | OTLP HTTP endpoint (e.g., `https://otel.example.com:4317`) | +| `insecure` | No | Use HTTP instead of HTTPS (default: `false`) | +| `publish_interval` | No | How often to publish metrics (Go duration, e.g., `30s`) | +| `headers` | No | Custom HTTP headers (e.g., for authentication) | -[telemetry.metrics.headers] -Authorization = "Bearer ..." -``` +### `traces` + +Array of OTLP trace collector configurations. Each entry supports: -## [telemetry.traces] +| Field | Required | Description | +|------------|----------|----------------------------------------------| +| `endpoint` | Yes | OTLP HTTP endpoint | +| `insecure` | No | Use HTTP instead of HTTPS (default: `false`) | +| `headers` | No | Custom HTTP headers | -Configure trace collectors (array). +See [Concepts > Telemetry](../concepts/telemetry.md) for details on available metrics and traces. + +## TOML ```toml -[telemetry.traces] +[telemetry] +disable_storacha_analytics = false + +[[telemetry.metrics]] endpoint = "https://otel.example.com:4317" insecure = false +publish_interval = "30s" +headers = { Authorization = "Bearer ..." } -[telemetry.traces.headers] -Authorization = "Bearer ..." +[[telemetry.traces]] +endpoint = "https://otel.example.com:4317" +insecure = false +headers = { Authorization = "Bearer ..." } ``` diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 5309b89b..d4eb9e7e 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -59,7 +59,7 @@ nav: - identity: configuration/identity.md - repo: - configuration/repo/index.md - - blob-storage: configuration/repo/blob-storage.md + - database: configuration/repo/database.md - server: configuration/server.md - pdp: - configuration/pdp/index.md @@ -75,6 +75,11 @@ nav: - Best Practices: operations/best-practices.md - Upgrading: operations/upgrading.md - Telemetry: operations/telemetry.md + - Concepts: + - concepts/index.md + - Database: concepts/database.md + - Networks: concepts/networks.md + - Telemetry: concepts/telemetry.md - CLI Reference: - cli/index.md - init: cli/init.md diff --git a/pkg/service/storage/handlers/replica/telemetry.go b/pkg/service/storage/handlers/replica/telemetry.go index 0d010b59..7c797fab 100644 --- a/pkg/service/storage/handlers/replica/telemetry.go +++ b/pkg/service/storage/handlers/replica/telemetry.go @@ -51,9 +51,12 @@ func NewMetrics() (*Metrics, error) { }, nil } -func (m *Metrics) startDuration(sink string) *telemetry.StopWatch { +func (m *Metrics) startDuration(source, sink string) *telemetry.StopWatch { if m == nil || m.durationTimer == nil { return nil } - return m.durationTimer.Start(attribute.String("sink", sink)) + return m.durationTimer.Start( + attribute.String("source", source), + attribute.String("sink", sink), + ) } diff --git a/pkg/service/storage/handlers/replica/transfer.go b/pkg/service/storage/handlers/replica/transfer.go index 4140dffd..b6434ed4 100644 --- a/pkg/service/storage/handlers/replica/transfer.go +++ b/pkg/service/storage/handlers/replica/transfer.go @@ -184,7 +184,7 @@ func Transfer(ctx context.Context, service TransferService, request *TransferReq forks []fx.Effect ) - stopwatch := metrics.startDuration(sinkLabel(request.Sink)) + stopwatch := metrics.startDuration(sourceLabel(&request.Source.URL), sinkLabel(request.Sink)) defer func() { success := true if err != nil { @@ -247,12 +247,16 @@ func sinkLabel(sink *url.URL) string { if sink == nil { return "none" } - if sink.Host != "" { - return sink.Host - } return sink.String() } +func sourceLabel(source *url.URL) string { + if source == nil { + return "none" + } + return source.String() +} + // checkBlobExists checks if the blob already exists in either PDP or Blobs store func checkBlobExists(ctx context.Context, service TransferService, blob types.Blob) (bool, error) { var err error