diff --git a/static/.well-known/mcp.json b/static/.well-known/mcp.json new file mode 100644 index 0000000..d405e92 --- /dev/null +++ b/static/.well-known/mcp.json @@ -0,0 +1,36 @@ +{ + "name": "Bacalhau Documentation", + "description": "MCP server for Bacalhau distributed compute documentation - CLI reference, API docs, examples, and tutorials", + "version": "1.0.0", + "server": { + "url": "https://mcp.expanso.io", + "transport": "https" + }, + "capabilities": { + "tools": true, + "resources": true + }, + "tools": [ + { + "name": "search_bacalhau_docs", + "description": "Search Bacalhau documentation for answers about distributed compute, CLI commands, and API usage" + }, + { + "name": "get_cli_help", + "description": "Get detailed help for Bacalhau CLI commands" + }, + { + "name": "get_example", + "description": "Get code examples for running jobs with Bacalhau" + } + ], + "documentation": { + "main": "https://docs.bacalhau.org", + "github": "https://github.com/bacalhau-project/bacalhau", + "llms_txt": "https://docs.bacalhau.org/llms.txt" + }, + "contact": { + "discord": "https://bit.ly/bacalhau-project-slack", + "github": "https://github.com/bacalhau-project/bacalhau/issues" + } +} diff --git a/static/llms.txt b/static/llms.txt new file mode 100644 index 0000000..c37e595 --- /dev/null +++ b/static/llms.txt @@ -0,0 +1,43 @@ +# Bacalhau Documentation + +> Bacalhau is an open-source distributed compute platform that enables processing data where it lives. Run Docker containers and WebAssembly across edge, on-premises, and multi-cloud environments. + +## Quick Start + +```bash +# Install Bacalhau +curl -sL https://get.bacalhau.org/install.sh | bash + +# Run your first job +bacalhau docker run ubuntu echo "Hello World" + +# Check job status +bacalhau job list +``` + +## Documentation Sections + +- [Getting Started](/llms/getting-started.txt): Installation, quickstart, first jobs +- [CLI Reference](/llms/cli.txt): All bacalhau commands and options +- [API Reference](/llms/api.txt): REST API documentation +- [Examples](/llms/examples.txt): Code examples and tutorials +- [Concepts](/llms/concepts.txt): Architecture and core concepts + +## Key Concepts + +- **Compute Over Data**: Run compute where data lives, not the other way around +- **Docker Support**: Run existing Docker containers without modification +- **WASM Support**: Lightweight WebAssembly execution for edge devices +- **Distributed**: Scale from laptop to thousands of nodes +- **Open Source**: Apache 2.0 licensed, community-driven + +## Resources + +- GitHub: https://github.com/bacalhau-project/bacalhau +- Discord: https://bit.ly/bacalhau-project-slack +- Website: https://bacalhau.org + +## Related + +- [Expanso Platform](https://docs.expanso.io/llms.txt): Enterprise features +- [Examples](https://examples.expanso.io/llms.txt): Code examples diff --git a/static/llms/api.txt b/static/llms/api.txt new file mode 100644 index 0000000..f10b589 --- /dev/null +++ b/static/llms/api.txt @@ -0,0 +1,1012 @@ +# Bacalhau API Reference + +## README + +# Overview + +:::warning +Note that in version 1.4.0 API logic and endpoints have changed. Check out the release notes and updated API description in the API documentation section. +::: + +Welcome to the official API documentation for Bacalhau. This guide provides a detailed insight into Bacalhau's RESTful HTTP APIs and demonstrates how to make the most out of them. + +## Overview + +Bacalhau prioritizes an "API-first" design, enabling users to interact with their deployed systems programmatically. In the `v1.4.0` the API model was changed to include only two endpoints, focused on orchestrating, querying and managing your network nodes and jobs. Each endpoint has a clear, separate environment and goal, allowing to manage coordination between nodes, jobs, and executions more effectively. + +- **Endpoint Prefix**: All APIs are versioned and prefixed with `/api/v1`. +- **Default Port**: By default, Bacalhau listens on port `1234`. + +## **API endpoints** + +### **Orchestrator** + +The Majority of Bacalhau’s functionality is channeled through the `Orchestrator` endpoint and its operations. It handles user requests and schedules and it is critical for creating, managing, monitoring, and analyzing jobs within Bacalhau. It also provides mechanisms to query information about the nodes in the cluster. + +```bash +api/v1/orchestrator/ +``` + +Here’s the job submission format, where you can tag a YAML file with the job specifications or input the commands with your CLI + +```bash +# Submit a job +curl -X PUT \ + -H "Content-Type: application/json" \ + -d '{ + "Job": { + "Name": "test-job", + "Namespace": "default", + "Type": "batch", + "Count": 1, + "Labels": { + "foo": "bar", + "env": "dev" + }, + "Tasks": [ + { + "Name": "task1", + "Engine": { + "Type": "docker", + "Params": { + "Image": "ubuntu:latest", + "Entrypoint": [ + "echo", + "hello" + ] + } + }, + "Publisher": { + "Type": "noop", + "Params": {} + }, + "ResourcesConfig": { + "CPU": "0.1", + "Memory": "10mb" + }, + "Network": { + "Type": "None" + }, + "Timeouts": { + "ExecutionTimeout": 30 + } + } + ] + } + }' \ + http://0.0.0.0:20000/api/v1/orchestrator/jobs + +{"JobID":"28c08f7f-6fb0-48ed-912d-a2cb6c3a4f3a","EvaluationID":"996b12e4-bcc5-4d74-ac21-0c421dafb7de"} +``` + +### Agent + +This endpoint offers a convenient route to collate detailed information about the Bacalhau node you're interacting with, whether it's acting as the orchestrator or a compute node. It provides you with insights into the node's health, capabilities, and the deployed Bacalhau version. + +```bash +api/v1/agent/node +``` + +Here’s the command structure for querying your current node. You can check on its status and collate information on its health and capabilities: + +```bash +# Is alive +curl 0.0.0.0:20000/api/v1/agent/alive +``` + +## Features + +### Pagination + +To handle large datasets, Bacalhau supports pagination. Users can define the `limit` in their request and then utilize the `next_token` from the response to fetch subsequent data chunks. + +### Ordering + +To sort the results of list-based queries, use the `order_by` parameter. By default, the list will be sorted in ascending order. If you want to reverse it, use the `reverse` parameter. Note that the fields available for sorting might vary depending on the specific API endpoint. + +### Pretty JSON Output + +By default, Bacalhau's APIs provide a minimized JSON response. If you want to view the output in a more readable format, append `pretty` to the query string. + +### HTTP Methods + +Being RESTful in nature, Bacalhau's API endpoints rely on standard HTTP methods to perform various actions: + +1. **GET**: Fetch data. +2. **PUT**: Update or create data. +3. **DELETE**: Remove data. + +The behavior of an API depends on its HTTP method. For example, `/api/v1/orchestrator/jobs`: + +1. **GET**: Lists all jobs. +2. **PUT**: Submits a new job. +3. **DELETE**: Stops a job. + +### HTTP Response Codes + +Understanding HTTP response codes is crucial. A `2xx` series indicates a successful operation, `4xx` indicates client-side errors, and `5xx` points to server-side issues. Always refer to the message accompanying the code for more information. + +:::warning +Since `/api/v1/requester/*` was changed to `/api/v1/orchestrator/` in `v1.4.0`, all `/api/v1/requester/*` requests will result in 410 error. +::: + + +## agent + +# Agent + +The Bacalhau Agent APIs provide a convenient means to retrieve information about the Bacalhau node you are communicating with, whether it serves as the orchestrator or functions as a compute node. These APIs offer valuable insights into the node's health, capabilities, and deployed Bacalhau version. + +## Is Alive + +**Endpoint:** `GET /api/v1/agent/alive` + +This API can be used to determine if the agent is operational and responding as expected. + +**Response**: + +```json +{ + "Status": "OK" +} +``` + +## Deployed Bacalhau Version + +**Endpoint:** `GET /api/v1/agent/version` + +This API provides details about the Bacalhau version, including major and minor version numbers, Git version, Git commit, build date, and platform information. + +**Response**: + +```json +{ + "Major": "1", + "Minor": "1", + "GitVersion": "v1.1.0", + "GitCommit": "970e1a0f23c7eb739a097aa8212f7964434bcd97", + "BuildDate": "2023-09-25T07:59:00Z", + "GOOS": "linux", + "GOARCH": "amd64" +} +``` + +## Node Info + +**Endpoint:** `GET /api/v1/agent/node` + +This API provides detailed information about the node, including its peer ID and network addresses, node type (e.g., Compute), labels, compute node capabilities, and the deployed Bacalhau version. + +**Response**: + +```json +{ + "PeerInfo": { + "ID": "QmdZQ7ZbhnvWY1J12XYKGHApJ6aufKyLNSvf8jZBrBaAVL", + "Addrs": [ + "/ip4/35.245.245.245/tcp/1235" + ] + }, + "NodeType": "Compute", + "Labels": { + "Architecture": "amd64", + "Operating-System": "linux", + "git-lfs": "True", + "owner": "bacalhau" + }, + "ComputeNodeInfo": { + "ExecutionEngines": [ + "docker", + "wasm" + ], + "Publishers": [ + "noop", + "ipfs", + "s3" + ], + "StorageSources": [ + "repoclonelfs", + "s3", + "ipfs", + "urldownload", + "inline", + "repoclone" + ], + "MaxCapacity": { + "CPU": 12.8, + "Memory": 53931121049, + "Disk": 721417073459, + "GPU": 0 + }, + "AvailableCapacity": { + "CPU": 12.8, + "Memory": 53931121049, + "Disk": 721417073459, + "GPU": 0 + }, + "MaxJobRequirements": { + "CPU": 12.8, + "Memory": 53931121049, + "Disk": 721417073459, + "GPU": 0 + }, + "RunningExecutions": 0, + "EnqueuedExecutions": 0 + }, + "BacalhauVersion": { + "Major": "1", + "Minor": "1", + "GitVersion": "v1.1.0", + "GitCommit": "970e1a0f23c7eb739a097aa8212f7964434bcd97", + "BuildDate": "2023-09-25T07:59:00Z", + "GOOS": "linux", + "GOARCH": "amd64" + } +} +``` + + +## jobs + +# Jobs + +## Describe Job + +**Endpoint:** `GET /api/v1/orchestrator/jobs/:jobID` + +Retrieve the specification and current status of a particular job. + +**Parameters**: + +1. `jobID`: Identifier of the job to describe. This can be full ID of the job (e.g. `j-28c08f7f-6fb0-48ed-912d-a2cb6c3a4f3a`) or just the short format (e.g. `j-28c08f7f`) if it's unique. + +**Response**: + +**Job**: Specification for the requested [job](/specifications/job/README.md). + +**Example**: + +```bash +curl 127.0.0.1:1234/api/v1/orchestrator/jobs/j-d586d2cc-6fc9-42c4-9dd9-a78df1d7cd01 +{ + "Job": { + "ID": "j-d586d2cc-6fc9-42c4-9dd9-a78df1d7cd01", + "Name": "A sample job", + "Namespace": "default", + "Type": "batch", + "Priority": 0, + "Count": 1, + "Constraints": [], + "Meta": { + "bacalhau.org/requester.id": "QmdZQ7ZbhnvWY1J12XYKGHApJ6aufKyLNSvf8jZBrBaAVL", + "bacalhau.org/requester.publicKey": "CAASpgIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDVRKPgCfY2fgfrkHkFjeWcqno+MDpmp8DgVaY672BqJl/dZFNU9lBg2P8Znh8OTtHPPBUBk566vU3KchjW7m3uK4OudXrYEfSfEPnCGmL6GuLiZjLf+eXGEez7qPaoYqo06gD8ROdD8VVse27E96LlrpD1xKshHhqQTxKoq1y6Rx4DpbkSt966BumovWJ70w+Nt9ZkPPydRCxVnyWS1khECFQxp5Ep3NbbKtxHNX5HeULzXN5q0EQO39UN6iBhiI34eZkH7PoAm3Vk5xns//FjTAvQw6wZUu8LwvZTaihs+upx2zZysq6CEBKoeNZqed9+Tf+qHow0P5pxmiu+or+DAgMBAAE=" + }, + "Labels": { + "env": "prod", + "name": "demo" + }, + "Tasks": [ + { + "Name": "main", + "Engine": { + "Type": "docker", + "Params": { + "Entrypoint": [ + "/bin/bash" + ], + "Image": "ubuntu:latest", + "Parameters": [ + "-c", + "echo hello world" + ] + } + }, + "Publisher": { + "Type": "", + "Params": {} + }, + "Env": {}, + "Meta": {}, + "InputSources": [], + "ResultPaths": [], + "Resources": { + "CPU": "", + "Memory": "", + "Disk": "", + "GPU": "" + }, + "Network": { + "Type": "None" + }, + "Timeouts": { + "ExecutionTimeout": 1800 + } + } + ], + "State": { + "StateType": "Completed", + "Message": "" + }, + "Version": 0, + "Revision": 2, + "CreateTime": 1695883778909107178, + "ModifyTime": 1695883779369191994 + } +} +``` + +## List Jobs + +**Endpoint:** `GET /api/v1/orchestrator/jobs` + +Retrieve a list of jobs. + +**Parameters**: + +1. `namespace`: Specify a namespace to filter the jobs. Use `*` to display jobs from all namespaces. +2. `labels`: Use label-based criteria to filter jobs. See [Label Filtering](/specifications/job/label.md) for usage details. +3. `limit`: Set the maximum number of jobs to return. Default is set to 10. +4. `next_token`: Utilize this parameter for pagination continuation. +5. `order_by`: Determine the ordering of jobs. Choose between `id` or `create_time` (default is `create_time`). +6. `reverse`: Opt to reverse the default order of displayed jobs. + +**Response**: + +- [**Jobs**](/specifications/job/README.md): List of matching jobs. +- **NextToken** `(string)`: Pagination token. + +**Example**: + +List jobs with limit set to 3: + +```bash +curl 127.0.0.1:1234/api/v1/orchestrator/jobs?limit=3 +{ + "Jobs": [ + { + "ID": "j-f6331e9a-727d-4175-8350-095b6b372408", + # ... + }, + { + "ID": "j-f7853204-a553-4991-a1a3-816b88fdbfc7", + # ... + }, + { + "ID": "j-f791ad14-af5b-4c26-8c93-15cc23dca811", + # ... + } + ], + "NextToken": "" +} +``` + +List with label filtering + +```bash +curl --get 127.0.0.1:1234/api/v1/orchestrator/jobs --data-urlencode 'labels=env in (prod,dev)' +``` + +## Create Job + +**Endpoint:** `PUT /api/v1/orchestrator/jobs` + +Submit a new job for execution. + +**Request Body**: + +- [**Job**](../specifications/job/): JSON definition of the job. + +**Response**: + +1. **JobID** `(string)`: Identifier for the new job. +2. **EvaluationID** `(string)`: Identifier for the evaluation to schedule the job. +3. **Warnings** `(string[])`: Any warnings during job submission. + +**Example**: + +```bash +curl -X PUT \ + -H "Content-Type: application/json" \ + -d '{ + "Job": { + "Name": "test-job", + "Type": "batch", + "Count": 1, + "Labels": { + "foo": "bar", + "env": "dev" + }, + "Tasks": [ + { + "Name": "task1", + "Engine": { + "Type": "docker", + "Params": { + "Image": "ubuntu:latest", + "Entrypoint": [ + "echo", + "hello" + ] + } + }, + "Publisher": { + "Type": "noop" + } + } + ], + "CreateTime": 1234 + } + }' \ + 127.0.0.1:1234/api/v1/orchestrator/jobs + + { + "JobID": "j-9809ae4b-d4fa-47c6-823b-86c924e60604", + "EvaluationID": "5dac9fe0-2358-4ec7-bec9-6747dfa2b33e", + "Warnings": [ + "job create time is ignored when submitting a job" + ] +} +``` + +## Stop Job + +**Endpoint:** `DELETE /api/v1/orchestrator/jobs/:jobID` + +Terminate a specific job asynchronously. + +**Parameters**: + +1. `:jobID`: Identifier of the job to describe. This can be full ID of the job (e.g. `j-28c08f7f-6fb0-48ed-912d-a2cb6c3a4f3a`) or just the short format (e.g. `j-28c08f7f`) if it's unique. +2. `reason`: A message for debugging and traceability. + +**Response**: + +- **EvaluationID** `(string)`: Identifier for the evaluation to stop the job. + +**Example**: + +```bash +curl -X DELETE 127.0.0.1:1234/api/v1/orchestrator/jobs/j-50ee38d5-2812-4365-aceb-7b47b8f3858e +{ + "EvaluationID": "1316fdfe-97c4-43bc-8e0b-50a7f02f18bb" +} +``` + +## Job History + +**Endpoint:** `GET /api/v1/orchestrator/jobs/:jobID/history` + +Retrieve historical events for a specific job. + +**Parameters**: + +1. `since`: Timestamp to start (default: 0). +2. `event_type`: Filter by event type: `job`, `execution`, or `all` (default). +3. `execution_id`: Filter by execution ID. +4. `node_id`: Filter by node ID. +5. `limit`: Maximum events to return. +6. `next_token`: For pagination. + +**Response**: + +- **History**: List of matching historical events. +- **NextToken** `(string)\*\*: Pagination token. + +**Example**: + +List events for a specific execution + +```bash +curl 127.0.0.1:1234/api/v1/orchestrator/jobs/j-4cd1566f-84cb-4830-a96b-1349f5b54b1b/history\?execution_id=e-82f7813f-58da-4323-8261-886af35284c4 +{ + "NextToken": "", + "History": [ + { + "Type": "ExecutionLevel", + "JobID": "j-4cd1566f-84cb-4830-a96b-1349f5b54b1b", + "NodeID": "QmUDAXvv31WPZ8U9CzuRTMn9iFGiopGE7rHiah1X8a6PkT", + "ExecutionID": "e-82f7813f-58da-4323-8261-886af35284c4", + "JobState": null, + "ExecutionState": { + "Previous": 1, + "New": 1 + }, + "NewRevision": 1, + "Comment": "", + "Time": "2023-09-28T07:23:01.352803607Z" + }, + { + "Type": "ExecutionLevel", + "JobID": "j-4cd1566f-84cb-4830-a96b-1349f5b54b1b", + "NodeID": "QmUDAXvv31WPZ8U9CzuRTMn9iFGiopGE7rHiah1X8a6PkT", + "ExecutionID": "e-82f7813f-58da-4323-8261-886af35284c4", + "JobState": null, + "ExecutionState": { + "Previous": 1, + "New": 2 + }, + "NewRevision": 2, + "Comment": "", + "Time": "2023-09-28T07:23:01.446196661Z" + }, + { + "Type": "ExecutionLevel", + "JobID": "j-4cd1566f-84cb-4830-a96b-1349f5b54b1b", + "NodeID": "QmUDAXvv31WPZ8U9CzuRTMn9iFGiopGE7rHiah1X8a6PkT", + "ExecutionID": "e-82f7813f-58da-4323-8261-886af35284c4", + "JobState": null, + "ExecutionState": { + "Previous": 2, + "New": 3 + }, + "NewRevision": 3, + "Comment": "", + "Time": "2023-09-28T07:23:01.604862596Z" + }, + { + "Type": "ExecutionLevel", + "JobID": "j-4cd1566f-84cb-4830-a96b-1349f5b54b1b", + "NodeID": "QmUDAXvv31WPZ8U9CzuRTMn9iFGiopGE7rHiah1X8a6PkT", + "ExecutionID": "e-82f7813f-58da-4323-8261-886af35284c4", + "JobState": null, + "ExecutionState": { + "Previous": 3, + "New": 3 + }, + "NewRevision": 4, + "Comment": "", + "Time": "2023-09-28T07:23:01.611816334Z" + }, + { + "Type": "ExecutionLevel", + "JobID": "j-4cd1566f-84cb-4830-a96b-1349f5b54b1b", + "NodeID": "QmUDAXvv31WPZ8U9CzuRTMn9iFGiopGE7rHiah1X8a6PkT", + "ExecutionID": "e-82f7813f-58da-4323-8261-886af35284c4", + "JobState": null, + "ExecutionState": { + "Previous": 3, + "New": 5 + }, + "NewRevision": 5, + "Comment": "", + "Time": "2023-09-28T07:23:01.705013737Z" + }, + { + "Type": "ExecutionLevel", + "JobID": "j-4cd1566f-84cb-4830-a96b-1349f5b54b1b", + "NodeID": "QmUDAXvv31WPZ8U9CzuRTMn9iFGiopGE7rHiah1X8a6PkT", + "ExecutionID": "e-82f7813f-58da-4323-8261-886af35284c4", + "JobState": null, + "ExecutionState": { + "Previous": 5, + "New": 7 + }, + "NewRevision": 6, + "Comment": "", + "Time": "2023-09-28T07:23:02.483265228Z" + } + ] +} +``` + +## Job Executions + +**Endpoint:** `GET /api/v1/orchestrator/jobs/:jobID/executions` + +Retrieve all executions for a particular job. + +**Parameters**: + +1. `limit`: Maximum executions to return. +2. `next_token`: For pagination. +3. `order_by`: Order by `modify_time` (default), `create_time`, `id`, `state`. +4. `reverse`: Reverse the order. + +**Response**: + +- **Executions**: List of relevant executions. +- **NextToken** `(string)\*\*: Pagination token. + +**Example** + +List executions for a batch job with 3 executions (i.e. `count=3`) + +```bash +curl 127.0.0.1:1234/api/v1/orchestrator/jobs/j-412c34b4-da77-4a46-886c-76e03615a04e/executions +{ + "NextToken": "", + "Executions": [ + { + "ID": "e-cdd9fb3e-3183-4069-8bc9-679b6bcce4db", + "Namespace": "default", + "EvalID": "", + "Name": "", + "NodeID": "QmYgxZiySj3MRkwLSL4X2MF5F9f2PMhAE3LV49XkfNL1o3", + "JobID": "j-412c34b4-da77-4a46-886c-76e03615a04e", + "AllocatedResources": { + "Tasks": {} + }, + "DesiredState": { + "StateType": 2, + "Message": "execution completed" + }, + "ComputeState": { + "StateType": 7, + "Message": "" + }, + "PublishedResult": { + "Type": "", + "Params": null + }, + "RunOutput": { + "stdout": "hello world\n", + "stdouttruncated": false, + "stderr": "", + "stderrtruncated": false, + "exitCode": 0, + "runnerError": "" + }, + "PreviousExecution": "", + "NextExecution": "", + "FollowupEvalID": "", + "Revision": 6, + "CreateTime": 1695886565851709698, + "ModifyTime": 1695886566370340241 + }, + { + "ID": "e-836a4a50-f6cd-479f-a20d-2a12ff7fea64", + "Namespace": "default", + "EvalID": "", + "Name": "", + "NodeID": "QmXaXu9N5GNetatsvwnTfQqNtSeKAD6uCmarbh3LMRYAcF", + "JobID": "j-412c34b4-da77-4a46-886c-76e03615a04e", + "AllocatedResources": { + "Tasks": {} + }, + "DesiredState": { + "StateType": 2, + "Message": "execution completed" + }, + "ComputeState": { + "StateType": 7, + "Message": "" + }, + "PublishedResult": { + "Type": "", + "Params": null + }, + "RunOutput": { + "stdout": "hello world\n", + "stdouttruncated": false, + "stderr": "", + "stderrtruncated": false, + "exitCode": 0, + "runnerError": "" + }, + "PreviousExecution": "", + "NextExecution": "", + "FollowupEvalID": "", + "Revision": 6, + "CreateTime": 1695886565855906980, + "ModifyTime": 1695886566505560693 + }, + { + "ID": "e-b7e7adc7-b28c-4af0-9002-a7fdce303634", + "Namespace": "default", + "EvalID": "", + "Name": "", + "NodeID": "QmUDAXvv31WPZ8U9CzuRTMn9iFGiopGE7rHiah1X8a6PkT", + "JobID": "j-412c34b4-da77-4a46-886c-76e03615a04e", + "AllocatedResources": { + "Tasks": {} + }, + "DesiredState": { + "StateType": 2, + "Message": "execution completed" + }, + "ComputeState": { + "StateType": 7, + "Message": "" + }, + "PublishedResult": { + "Type": "", + "Params": null + }, + "RunOutput": { + "stdout": "hello world\n", + "stdouttruncated": false, + "stderr": "", + "stderrtruncated": false, + "exitCode": 0, + "runnerError": "" + }, + "PreviousExecution": "", + "NextExecution": "", + "FollowupEvalID": "", + "Revision": 6, + "CreateTime": 1695886565853878926, + "ModifyTime": 1695886566583711985 + } + ] +} +``` + +## Job Results + +**Endpoint:** `GET /api/v1/orchestrator/jobs/:jobID/results` + +Fetch results published by all executions for the defined job. Applicable only for `batch` and `ops` jobs. + +**Response**: + +- **Results**: List of all published results. +- **NextToken** `(string)\*\*: Pagination token. + +**Example**: + +Result of a job that used the [S3 Publisher](/components/publishers/s3.md): + +```bash +curl 127.0.0.1:1234/api/v1/orchestrator/jobs/j-479d160f-f9ab-4e32-aec9-a45554126450/results +{ + "NextToken": "", + "Results": [ + { + "Type": "s3", + "Params": { + "Bucket": "bacalhau-test-datasets", + "Key": "my-prefix/my-result-file.tar.gz", + "Region": "eu-west-1", + "ChecksumSHA256": "qKAFvkLvSc+QqHE4hFiy4qVEmXhr423lQaRBfJecsgo=", + "VersionID": "bNS92VdFudVI7NPsXF51Qn.RPw31TKNG" + } + } + ] +} +``` + + +## nodes + +# Nodes + +## Describe Node + +**Endpoint:** `GET /api/v1/orchestrator/nodes/:nodeID` + +Retrieve information about a specific node. + +**Parameters**: + +1. `:nodeID`: Identifier of the node to describe. (e.g. `QmUDAXvv31WPZ8U9CzuRTMn9iFGiopGE7rHiah1X8a6PkT`) + +**Response**: + +* **Node**: Detailed information about the requested node. + +**Example**: + +```bash +curl 127.0.0.1:1234/api/v1/orchestrator/nodes/QmUDAXvv31WPZ8U9CzuRTMn9iFGiopGE7rHiah1X8a6PkT +{ + "Node": { + "PeerInfo": { + "ID": "QmUDAXvv31WPZ8U9CzuRTMn9iFGiopGE7rHiah1X8a6PkT", + "Addrs": [ + "/ip4/34.34.247.247/tcp/1235" + ] + }, + "NodeType": "Compute", + "Labels": { + "Architecture": "amd64", + "Operating-System": "linux", + "git-lfs": "True", + "owner": "bacalhau" + }, + "ComputeNodeInfo": { + "ExecutionEngines": [ + "docker", + "wasm" + ], + "Publishers": [ + "s3", + "noop", + "ipfs" + ], + "StorageSources": [ + "urldownload", + "inline", + "repoclone", + "repoclonelfs", + "s3", + "ipfs" + ], + "MaxCapacity": { + "CPU": 3.2, + "Memory": 12561049190, + "Disk": 582010404864, + "GPU": 1 + }, + "AvailableCapacity": { + "CPU": 3.2, + "Memory": 12561049190, + "Disk": 582010404864, + "GPU": 1 + }, + "MaxJobRequirements": { + "CPU": 3.2, + "Memory": 12561049190, + "Disk": 582010404864, + "GPU": 1 + }, + "RunningExecutions": 0, + "EnqueuedExecutions": 0 + }, + "BacalhauVersion": { + "Major": "1", + "Minor": "1", + "GitVersion": "v1.1.0", + "GitCommit": "970e1a0f23c7eb739a097aa8212f7964434bcd97", + "BuildDate": "2023-09-25T07:59:00Z", + "GOOS": "linux", + "GOARCH": "amd64" + } + } +} +``` + +## List Nodes + +**Endpoint:** `GET /api/v1/orchestrator/nodes` + +Retrieve a list of nodes. + +**Parameters**: + +1. `labels`: Use label-based criteria to filter nodes. See [Label Filtering](/guides/labels-and-constraints.md) for usage details. +2. `limit`: Set the maximum number of jobs to return. Default is set to 10. +3. `next_token`: Utilize this parameter for pagination continuation. +4. `order_by`: Determine the ordering of jobs. Choose between `id`, `type`, `available_cpu`, `available_memory`, `available_disk` or `available_gpu`. (default is `id`). +5. `reverse`: Opt to reverse the default order of displayed jobs. + +**Response**: + +* **Nodes**: List of matching nodes. +* **NextToken** `(string)`: Pagination token. + +**Example**: + +Find two linux nodes with most available Memory + +```bash +curl --get "127.0.0.1:1234/api/v1/orchestrator/nodes?limit=2&order_by=available_memory" --data-urlencode 'labels=Operating-System=linux' +{ + "NextToken": "", + "Nodes": [ + { + "PeerInfo": { + "ID": "QmcC3xifiiCuGGQ9rpvefUoary9tY65x2HaNxSdeMTvM9U", + "Addrs": [ + "/ip4/212.248.248.248/tcp/1235" + ] + }, + "NodeType": "Compute", + "Labels": { + "Architecture": "amd64", + "Operating-System": "linux", + "env": "prod", + "git-lfs": "False", + "name": "saturnia_len20" + }, + "ComputeNodeInfo": { + "ExecutionEngines": [ + "wasm", + "docker" + ], + "Publishers": [ + "noop", + "ipfs" + ], + "StorageSources": [ + "urldownload", + "inline", + "ipfs" + ], + "MaxCapacity": { + "CPU": 102, + "Memory": 858993459200, + "Disk": 562967789568, + "GPU": 2 + }, + "AvailableCapacity": { + "CPU": 102, + "Memory": 858993459200, + "Disk": 562967789568, + "GPU": 2 + }, + "MaxJobRequirements": { + "CPU": 96, + "Memory": 858993459200, + "Disk": 562967789568, + "GPU": 2 + }, + "RunningExecutions": 0, + "EnqueuedExecutions": 0 + }, + "BacalhauVersion": { + "Major": "1", + "Minor": "1", + "GitVersion": "v1.1.0", + "GitCommit": "970e1a0f23c7eb739a097aa8212f7964434bcd97", + "BuildDate": "2023-09-25T07:59:00Z", + "GOOS": "linux", + "GOARCH": "amd64" + } + }, + { + "PeerInfo": { + "ID": "QmXaXu9N5GNetatsvwnTfQqNtSeKAD6uCmarbh3LMRYAcF", + "Addrs": [ + "/ip4/35.245.245.245/tcp/1235" + ] + }, + "NodeType": "Compute", + "Labels": { + "Architecture": "amd64", + "Operating-System": "linux", + "git-lfs": "True", + "owner": "bacalhau" + }, + "ComputeNodeInfo": { + "ExecutionEngines": [ + "docker", + "wasm" + ], + "Publishers": [ + "noop", + "ipfs", + "s3" + ], + "StorageSources": [ + "s3", + "ipfs", + "urldownload", + "inline", + "repoclone", + "repoclonelfs" + ], + "MaxCapacity": { + "CPU": 12.8, + "Memory": 53931124326, + "Disk": 718749414195, + "GPU": 0 + }, + "AvailableCapacity": { + "CPU": 12.8, + "Memory": 53931124326, + "Disk": 718749414195, + "GPU": 0 + }, + "MaxJobRequirements": { + "CPU": 12.8, + "Memory": 53931124326, + "Disk": 718749414195, + "GPU": 0 + }, + "RunningExecutions": 0, + "EnqueuedExecutions": 0 + }, + "BacalhauVersion": { + "Major": "1", + "Minor": "1", + "GitVersion": "v1.1.0", + "GitCommit": "970e1a0f23c7eb739a097aa8212f7964434bcd97", + "BuildDate": "2023-09-25T07:59:00Z", + "GOOS": "linux", + "GOARCH": "amd64" + } + } + ] +} +``` + diff --git a/static/llms/cli.txt b/static/llms/cli.txt new file mode 100644 index 0000000..0c08ff7 --- /dev/null +++ b/static/llms/cli.txt @@ -0,0 +1,1841 @@ +# Bacalhau CLI Reference + +## README + +# Overview + +This page explains how the Bacalhau CLI is structured and which global flags are most commonly used. Understanding these fundamentals will help you work efficiently with all Bacalhau commands. + +## What You'll Learn + +- The general structure and organization of Bacalhau commands +- How global flags affect command behavior regardless of the specific command +- How to customize output formats and control connection settings +- How to specify configuration files and data directories +- Where to find more detailed command references + +## CLI Structure + +Bacalhau commands follow a consistent pattern that makes them intuitive and predictable: + +```bash +bacalhau [flags] +``` + +## Top-Level Commands + +Bacalhau's CLI groups commands into logical categories: + +- `agent`: Client-side commands for checking health, version, and node information +- `job`: Core job management (create, list, describe, stop, retrieve logs, etc.) +- `node`: Cluster node management and inspection +- `config`: Client configuration management +- `docker`: Imperative command for running Docker-based jobs + +## Command Examples + +```bash +# Get version information +bacalhau agent version + +# List recent jobs +bacalhau job list + +# Submit a Docker job (imperative style) +bacalhau docker run ubuntu:latest -- echo "Hello World" + +# View compute nodes in the cluster +bacalhau node list +``` + +## Global Flags + +These flags work with any command and provide consistent behavior across the CLI. They're especially useful for scripting and automation. + +### Connection Settings + +| Flag | Description | Default | +| ------------------- | ----------------------------- | ----------- | +| `--api-host string` | Hostname for the Bacalhau API | `localhost` | +| `--api-port int` | Port for the Bacalhau API | `1234` | + +Example: + +```bash +bacalhau job list --api-host mycluster.example.com --api-port 8080 +``` + +### Configuration Management + +| Flag | Description | Default | +| --------------------- | -------------------------------------------------------- | ------------- | +| `-c, --config string` | Config file(s) or dot separated path(s) to config values | - | +| `--data-dir string` | The filesystem path where Bacalhau stores its data | `~/.bacalhau` | + +Examples: + +```bash +# Use a custom config file +bacalhau job list --config /path/to/custom/config.yaml + +# Specify a custom data directory +bacalhau job list --data-dir /path/to/bacalhau/data + +# Configure a specific value using dot notation +bacalhau serve --orchestrator -c WebUI.Enabled=true +``` + +### Output Formatting + +| Flag | Description | Example Values | +| ----------------- | ------------------------------------------ | ----------------------- | +| `--output format` | Output format style | `json`, `yaml`, `table` | +| `--pretty` | Format JSON or YAML output for readability | - | + +Examples: + +```bash +# Get machine-readable job list +bacalhau job list --output json + +# Get pretty-printed JSON +bacalhau job list --output json --pretty + +# Get YAML output +bacalhau job list --output yaml +``` + +## Getting Help + +For detailed information about any command's available flags: + +```bash +bacalhau --help +``` + +This will show all available options, including both global flags and command-specific flags. + +**Tip:** For full details on each command's available flags, see the CLI Reference or type `bacalhau --help`. + + +## README + +# Config + +The `bacalhau config` command is a parent command that offers sub-commands to modify and query information about the Bacalhau config. This can be useful for debugging, monitoring, or managing the nodes configuration. + +## Usage + +```bash +bacalhau config [command] +``` + +## Available Commands + +1. [**list**](list.md): + + - Description: Lists the configuration keys and values of the bacalhau node. This command is useful for understanding how configuration keys map to their respective values, aiding in the use of the `bacalhau config set` command. + - Usage: + + ```bash + bacalhau config list + ``` + +2. [**set**](set.md): + + - Description: Sets a value in the bacalhau node's configuration file. This command is used to modify the configuration file that the bacalhau node will reference for its settings. + - Usage: + + ```bash + bacalhau config set + ``` + + +## README + +# Agent + +The `bacalhau agent` command is a parent command that offers sub-commands to query information about the Bacalhau agent. This can be useful for debugging, monitoring, or managing the agent's behavior and health. + +## Usage + +```bash +bacalhau agent [command] +``` + +## Available Commands + +1. [**alive**](alive.md): + + - Description: Retrieves the agent's liveness and health information. This can be helpful to determine if the agent is running and healthy. + - Usage: + + ```bash + bacalhau agent alive + ``` + +2. [**node**](node.md): + + - Description: Gathers the agent's node-related information. This might include details about the machine or environment where the agent is running, available resources, supported engines, etc. + - Usage: + + ```bash + bacalhau agent node + ``` + +3. [**version**](version.md): + + - Description: Retrieves the Bacalhau version of the agent. This can be beneficial for ensuring compatibility or checking for updates. + - Usage: + + ```bash + bacalhau agent version + ``` + +For more detailed information on any of the sub-commands, you can use the command: + +```bash +bacalhau agent [command] --help +``` + +## Flags + +- `-h`, `--help`: + - Description: Displays help information for the `agent` command. + +## Global Flags + +- `--api-host string`: + - Description: Specifies the host used for RESTful communication between the client and server. The flag is disregarded if the `BACALHAU_API_HOST` environment variable is set. + - Default: `bootstrap.production.bacalhau.org` +- `--api-port int`: + - Description: Specifies the port for REST communication. If the `BACALHAU_API_PORT` environment variable is set, this flag will be ignored. + - Default: `1234` +- `--log-mode logging-mode`: + - Description: Sets the desired log format. Options are: `default`, `station`, `json`, `combined`, and `event`. + - Default: `default` +- `--repo string`: + - Description: Defines the path to the bacalhau repository. + - Default: \`\`$HOME/.bacalhau\` + + +## README + +# Job + +The `bacalhau job` command provides a suite of sub-commands to submit, query, and manage jobs within Bacalhau. Users can deploy jobs, obtain job details, track execution logs, and more. + +## Usage + +```bash +bacalhau job [command] +``` + +## Available Commands + +1. [**describe**](./describe.md): + + - Description: Retrieves detailed information of a job using its ID. + - Usage: + + ```bash + bacalhau job describe + ``` + +2. [**executions**](./executions.md): + + - Description: Lists all executions associated with a job, identified by its ID. + - Usage: + + ```bash + bacalhau job executions + ``` + +3. [**get**](./get.md): + + - Description: Get the results of a job, identified by its ID. + - Usage: + + ```bash + bacalhau job get + ``` + +4. [**history**](./history.md): + + - Description: Enumerates the historical events related to a job, identified by its ID. + - Usage: + + ```bash + bacalhau job history + ``` +5. [**list**](./list.md): + + - Description: Provides an overview of all submitted jobs. + - Usage: + + ```bash + bacalhau job list + ``` + +6. [**logs**](./logs.md): + + - Description: Fetches and streams the logs from a currently executing job. + - Usage: + + ```bash + bacalhau job logs + ``` + +7. [**run**](./run.md): + + - Description: Submits a job for execution using either a JSON or YAML configuration file. + - Usage: + + ```bash + bacalhau job run + ``` + +8. [**stop**](./stop.md): + + - Description: Halts a previously submitted job. + - Usage: + + ```bash + bacalhau job stop + ``` + +For comprehensive details on any of the sub-commands, run: + +```bash +bacalhau job [command] --help +``` + +## Flags + +- `-h`, `--help`: + - Description: Shows the help information for the `job` command. + +## Global Flags + +1. `--api-host string`: + - Description: Determines the host for RESTful communication between the client and server. This flag is overlooked if the `BACALHAU_API_HOST` environment variable is set. + - Default: `bootstrap.production.bacalhau.org` +2. `--api-port int`: + - Description: Designates the port for RESTful communication. This flag is bypassed if the `BACALHAU_API_PORT` environment variable is active. + - Default: `1234` +3. `--log-mode logging-mode`: + - Description: Chooses the preferred log format. Available choices are: `default`, `station`, `json`, `combined`, and `event`. + - Default: `default` +4. `--repo string`: + - Description: Specifies the path to the bacalhau repository. + - Default: `$HOME/.bacalhau` + + +## README + +# Node + +The `bacalhau node` command provides a set of sub-commands to query and manage node-related information within Bacalhau. With these tools, users can access specific details about nodes, list all network nodes, and more. + +## Usage + +```bash +bacalhau node [command] +``` + +## Available Commands + +1. [**approve**](approve.md): + + - Description: Approves a single node to join the cluster. + - Usage: + + ```bash + bacalhau node approve + ``` + +2. [**delete**](delete.md): + + - Description: Deletes a node from the cluster using its ID. + - Usage: + + ```bash + bacalhau node delete + ``` + +3. [**describe**](describe.md): + + - Description: Retrieves detailed information of a node using its ID. + - Usage: + + ```bash + bacalhau node describe + ``` + +4. [**list**](list.md): + + - Description: Lists the details of all nodes present in the network. + - Usage: + + ```bash + bacalhau node list + ``` + +5. [**reject**](reject.md): + +- Description: Reject a specific node's request to join the cluster. +- Usage: + + ```bash + bacalhau node reject + ``` + +For comprehensive details on any of the sub-commands, run: + +```bash +bacalhau node [command] --help +``` + +## Flags + +- `-h`, `--help`: + - Description: Shows the help information for the `node` command. + +## Global Flags + +- `--api-host string`: + - Description: Specifies the host for RESTful communication between the client and server. The flag will be ignored if the `BACALHAU_API_HOST` environment variable is set. + - Default: `bootstrap.production.bacalhau.org` +- `--api-port int`: + - Description: Designates the port for RESTful communication. The flag will be bypassed if the `BACALHAU_API_PORT` environment variable is active. + - Default: `1234` +- `--log-mode logging-mode`: + - Description: Chooses the preferred log format. Available choices are: `default`, `station`, `json`, `combined`, and `event`. + - Default: `default` +- `--repo string`: + - Description: Specifies the path to the bacalhau repository. + - Default: `/Users/walid/.bacalhau` + + +## alive + +# Alive + +## Description + +The `bacalhau agent alive` command provides information about the agent's liveness and health. This is essential for monitoring and ensuring that the agent is active and functioning correctly. + +## Usage + +```bash +bacalhau agent alive [flags] +``` + +## Flags + +- `-h`, `--help`: + - Description: Displays help information for the `alive` sub-command. +- `--output format`: + - Description: Determines the format in which the output is displayed. Available formats include JSON and YAML. + - Options: `json`, `yaml` + - Default: `yaml` +- `--pretty`: + - Description: Formats the output for enhanced readability. This flag is relevant only when using JSON or YAML output formats. + +## Global Flags + +- `--api-host string`: + - Description: Specifies the host used for RESTful communication between the client and server. The flag is disregarded if the `BACALHAU_API_HOST` environment variable is set. + - Default: `bootstrap.production.bacalhau.org` +- `--api-port int`: + - Description: Specifies the port for REST communication. If the `BACALHAU_API_PORT` environment variable is set, this flag will be ignored. + - Default: `1234` +- `--log-mode logging-mode`: + - Description: Sets the desired log format. Options are: `default`, `station`, `json`, `combined`, and `event`. + - Default: `default` +- `--repo string`: + - Description: Defines the path to the bacalhau repository. + - Default: \`\`$HOME/.bacalhau\` + +## Examples + +### Checking the Agent's Liveness and Health Info + +1. **Basic Usage**: + + **Command**: + + ```bash + bacalhau agent alive + ``` + + **Output**: + + ``` + status: OK + ``` + +2. **Output in JSON format**: + + **Command**: + + ```bash + bacalhau agent alive --output json --pretty + ``` + + **Output**: + + ```json + { + "Status": "OK" + } + ``` + + +## approve + +# Approve + +The `bacalhau node approve` command offers administrators the ability to approve the cluster membership for a node using its name. + +## Description + +Using the `approve` sub-command under the `bacalhau node` umbrella, users can allow a node in the pending state to join the cluster and receive work. This feature is crucial for system administrators to manage the cluster. + +## Usage + +```bash +bacalhau node approve [id] [flags] +``` + +## Flags + +- `[id]`: + - The unique identifier of the node you wish to describe. +- `-h`, `--help`: + - Displays the help documentation for the `describe` command. +- `-m message`: + - A message to be attached to the approval action. + +## Global Flags + +- `--api-host string`: + - Specifies the host for client-server communication through REST. This flag is overridden if the `BACALHAU_API_HOST` environment variable is set. + - Default: `"bootstrap.production.bacalhau.org"` +- `--api-port int`: + - Designates the port for REST-based communication between client and server. This flag is overlooked if the `BACALHAU_API_PORT` environment variable is defined. + - Default: `1234` +- `--log-mode logging-mode`: + - Determines the log format preference. + - Options: `'default','station','json','combined','event'` + - Default: `'default'` +- `--repo string`: + - Points to the bacalhau repository's path. + - Default: `"`$HOME/.bacalhau"\` + +## Examples + +1. Approve a Node with ID `nodeID123`: + + ```bash + bacalhau node approve nodeID123 + ``` + +2. Approve a Node with an audit message: + + ```bash + bacalhau node approve nodeID123 -m "okay" + ``` + + +## delete + +# Delete + +The `bacalhau node delete` command offers administrators the ability to remove a node from the cluster using its name. + +## Description + +Using the `delete` sub-command, administrators can remove a node from the list of available compute nodes in the cluster. This feature is necessary for the management of the infrastructure. + +## Usage + +```bash +bacalhau node delete [id] [flags] +``` + +## Flags + +- `[id]`: + - The unique identifier of the node you wish to describe. +- `-h`, `--help`: + - Displays the help documentation for the `describe` command. +- `-m message`: + - A message to be attached to the deletion action. + +## Global Flags + +- `--api-host string`: + - Specifies the host for client-server communication through REST. This flag is overridden if the `BACALHAU_API_HOST` environment variable is set. + - Default: `"bootstrap.production.bacalhau.org"` +- `--api-port int`: + - Designates the port for REST-based communication between client and server. This flag is overlooked if the `BACALHAU_API_PORT` environment variable is defined. + - Default: `1234` +- `--log-mode logging-mode`: + - Determines the log format preference. + - Options: `'default','station','json','combined','event'` + - Default: `'default'` +- `--repo string`: + - Points to the bacalhau repository's path. + - Default: `"`$HOME/.bacalhau"\` + +## Examples + +1. Delete the Node with ID `nodeID123`: + + ```bash + bacalhau node delete nodeID123 + ``` + +2. Delete a Node with an audit message: + + ```bash + bacalhau node delete nodeID123 -m "bad actor" + ``` + + +## describe + +# Describe + +## Description + +The `bacalhau job describe` command provides a detailed description of a specific job in YAML format. This description can be particularly useful when wanting to understand the attributes and current status of a specific job. To list all available jobs, the `bacalhau job list` command can be used. + +## Usage + +```bash +bacalhau job describe [id] [flags] +``` + +## Flags + +- `-h`, `--help`: + - Description: Display help for the `describe` command. +- `--output format`: + - Description: Specifies the desired output format for the command. Supported values are `json` and `yaml`. + - Default: `yaml` +- `--pretty`: + - Description: Pretty prints the output. This option is applicable only to `json` and `yaml` output formats. + +## Global Flags + +- `--api-host string`: + - Description: Specifies the host for the client and server to communicate through via REST. If the `BACALHAU_API_HOST` environment variable is set, this flag will be ignored. + - Default: `bootstrap.production.bacalhau.org` +- `--api-port int`: + - Description: Determines the port for the client and server to communicate on using REST. If the `BACALHAU_API_PORT` environment variable is set, this flag will be ignored. + - Default: `1234` +- `--log-mode logging-mode`: + - Description: Specifies the desired log format. Supported values include `default`, `station`, `json`, `combined`, and `event`. + - Default: `default` +- `--repo string`: + - Description: Defines the path to the bacalhau repository. + - Default: `$HOME/.bacalhau` + +## Examples + +1. **Describe a Job with Full ID**: + + ```bash + bacalhau job describe j-e3f8c209-d683-4a41-b840-f09b88d087b9 + ``` + +2. **Describe a Job with Shortened ID**: + + ```bash + bacalhau job describe j-47805f5c + ``` + +3. **Describe a Job with JSON Output**: + + ```bash + bacalhau job describe --output json --pretty j-b6ad164a + ``` + + +## describe + +# Describe + +The `bacalhau node describe` command offers users the ability to retrieve detailed information about a specific node using its unique identifier. + +## Description + +Using the `describe` sub-command under the `bacalhau node` umbrella, users can get comprehensive details of a node by providing its ID. This information is crucial for system administrators and network managers to understand the state, specifications, and other attributes of nodes in their infrastructure. + +## Usage + +```bash +bacalhau node describe [id] [flags] +``` + +## Flags + +- `[id]`: + - The unique identifier of the node you wish to describe. +- `-h`, `--help`: + - Displays the help documentation for the `describe` command. +- `--output format`: + - Defines the desired format for the command's output. + - Options: `"json"` or `"yaml"` + - Default: `"yaml"` +- `--pretty`: + - When this flag is used, the command will pretty print the output. This is applicable only for outputs in `json` and `yaml` formats. + +## Global Flags + +- `--api-host string`: + - Specifies the host for client-server communication through REST. This flag is overridden if the `BACALHAU_API_HOST` environment variable is set. + - Default: `"bootstrap.production.bacalhau.org"` +- `--api-port int`: + - Designates the port for REST-based communication between client and server. This flag is overlooked if the `BACALHAU_API_PORT` environment variable is defined. + - Default: `1234` +- `--log-mode logging-mode`: + - Determines the log format preference. + - Options: `'default','station','json','combined','event'` + - Default: `'default'` +- `--repo string`: + - Points to the bacalhau repository's path. + - Default: `"`$HOME/.bacalhau"\` + +## Examples + +1. Describing a Node with ID `nodeID123`: + + ```bash + bacalhau node describe nodeID123 + ``` + +2. Describing a Node with Output in JSON Format: + + ```bash + bacalhau node describe nodeID123 --output json + ``` + +3. Pretty Printing the Description of a Node: + + ```bash + bacalhau node describe nodeID123 --pretty + ``` + + +## executions + +# Executions + +## Description + +The `bacalhau job executions` command retrieves a list of executions for a specific job based on its ID. This can be essential when tracking the various runs and their respective states for a particular job. + +## Usage + +```bash +bacalhau job executions [id] [flags] +``` + +## Flags + +- `-h`, `--help`: + - Description: Display help for the `executions` command. +- `--hide-header`: + - Description: Do not print the column headers when displaying the results. +- `--limit uint32`: + - Description: Restricts the number of results returned. + - Default: `20` +- `--next-token string`: + - Description: Uses the specified token for pagination. Useful for fetching the next set of results. +- `--no-style`: + - Description: Removes all styling from the table output, displaying raw data. +- `--order-by string`: + - Description: Orders results based on a specific field. Valid fields are: `modify_time`, `create_time`, `id`, and `state`. +- `--order-reversed`: + - Description: Reverses the order of the results. Useful in conjunction with `--order-by`. +- `--output format`: + - Description: Specifies the desired output format for the command. Supported values are `table`, `csv`, `json`, and `yaml`. + - Default: `table` +- `--pretty`: + - Description: Pretty prints the output. This option is applicable only to `json` and `yaml` output formats. +- `--wide`: + - Description: Prints full values in the table results without truncating any information. + +## Global Flags + +- `--api-host string`: + - Description: Specifies the host for the client and server to communicate through via REST. If the `BACALHAU_API_HOST` environment variable is set, this flag will be ignored. + - Default: `bootstrap.production.bacalhau.org` +- `--api-port int`: + - Description: Determines the port for the client and server to communicate on using REST. If the `BACALHAU_API_PORT` environment variable is set, this flag will be ignored. + - Default: `1234` +- `--log-mode logging-mode`: + - Description: Specifies the desired log format. Supported values include `default`, `station`, `json`, `combined`, and `event`. + - Default: `default` +- `--repo string`: + - Description: Defines the path to the bacalhau repository. + - Default: `$HOME/.bacalhau` + +## Examples + +1. **List executions for a specific Job**: + + ```bash + bacalhau job executions j-6f2bf0ea-ebcd-4490-899a-9de9d8d95881 + ``` + + Expected output: + + ```bash + CREATED MODIFIED ID NODE ID REV. COMPUTE DESIRED COMMENT + STATE STATE + 16:46:03 16:46:04 e-99362435 QmTSJgdN 6 Completed Stopped + 16:46:03 16:46:04 e-75dd20bb QmXRdLru 6 Completed Stopped + 16:46:03 16:46:04 e-03870df5 QmVXwmdZ 6 Completed Stopped + ``` + +2. **Order executions by state for a specific job**: + + Execute the command: + + ```bash + bacalhau job executions j-6f2bf0ea-ebcd-4490-899a-9de9d8d95881 --order-by state + ``` + + Expected output: + + ```bash + CREATED MODIFIED ID NODE ID REV. COMPUTE DESIRED COMMENT + STATE STATE + 16:46:03 16:46:04 e-03870df5 QmVXwmdZ 6 Completed Stopped + 16:46:03 16:46:04 e-75dd20bb QmXRdLru 6 Completed Stopped + 16:46:03 16:46:04 e-99362435 QmTSJgdN 6 Completed Stopped + ``` + +3. **List executions with YAML output**: + + ```bash + bacalhau job executions j-6f2bf0ea-ebcd-4490-899a-9de9d8d95881 --output yaml + ``` + + Expected output: + + ```yaml + ... [The YAML formatted output] ... + ``` + + +## get + +# Get + +## Description + +The `bacalhau job get` command is used to get the results of the job, including stdout and stderr. + +## Usage + +```bash +bacalhau job get [id] [flags] +``` + +## Flags + +- `--download-timeout-secs duration`: + - Description: Timeout duration for IPFS downloads. (default 5m0s) +- `-h`, `--help`: + - Description: Display help for the `list` command. +- `output-dir string`: + - Description: Directory to write the output to. +- `--raw`: + - Description: Download raw result CIDs instead of merging multiple CIDs into a single result. + +## Global Flags + +- `--api-host string`: + - Description: Defines the host for client-server communication via REST. Overridden by the `BACALHAU_API_HOST` environment variable, if set. + - Default: `bootstrap.production.bacalhau.org` +- `--api-port int`: + - Description: Sets the port for RESTful communication between the client and server. The `BACALHAU_API_PORT` environment variable takes precedence if set. + - Default: `1234` +- `--log-mode logging-mode`: + - Description: Designates the desired log format. Options include `default`, `station`, `json`, `combined`, and `event`. + - Default: `default` +- `--repo string`: + - Description: Points to the bacalhau repository location. + - Default: `$HOME/.bacalhau` + +## Examples + +1. **Get the results of a job**: + + `bacalhau job get 51225160-807e-48b8-88c9-28311c7899e1` + +2. **Get the results of a job, using a short ID**: + + `bacalhau job get 51225160` + + +## history + +# History + +## Description + +The `bacalhau job history` command lists the history events of a specific job based on its ID. This feature allows users to track changes, executions, and other significant milestones associated with a particular job. + +## Usage + +```bash +bacalhau job history [id] [flags] +``` + +## Flags + +- `--event-type string`: + - Description: Specifies the type of history events to retrieve. Available options include `all`, `job`, and `execution`. + - Default: `all` +- `--execution-id string`: + - Description: Filters results by a specific execution ID. +- `-h`, `--help`: + - Description: Display help for the `history` command. +- `--hide-header`: + - Description: Opts out of printing the column headers in the results. +- `--limit uint32`: + - Description: Limits the number of results returned. +- `--next-token string`: + - Description: Uses the provided token for pagination. +- `--no-style`: + - Description: Strips all styling from the table output. +- `--node-id string`: + - Description: Filters the results by a specific node ID. +- `--order-by string`: + - Description: Organizes results based on a chosen field. +- `--order-reversed`: + - Description: Reverses the order of the displayed results. +- `--output format`: + - Description: Dictates the desired output format for the command. Options are `table`, `csv`, `json`, and `yaml`. + - Default: `table` +- `--pretty`: + - Description: Offers a more visually pleasing output for `json` and `yaml` formats. +- `--wide`: + - Description: Presents full values in the table results, preventing truncation. + +## Global Flags + +- `--api-host string`: + - Description: Defines the host for client-server communication via REST. Overridden by the `BACALHAU_API_HOST` environment variable, if set. + - Default: `bootstrap.production.bacalhau.org` +- `--api-port int`: + - Description: Sets the port for RESTful communication between the client and server. The `BACALHAU_API_PORT` environment variable takes precedence if set. + - Default: `1234` +- `--log-mode logging-mode`: + - Description: Designates the desired log format. Options include `default`, `station`, `json`, `combined`, and `event`. + - Default: `default` +- `--repo string`: + - Description: Points to the bacalhau repository location. + - Default: `$HOME/.bacalhau` + +## Examples + +1. **Retrieve the history of a specific job**: + + Execute the command to get the job history: + + ```bash + bacalhau job history j-6f2bf0ea-ebcd-4490-899a-9de9d8d95881 + ``` + + Expected output: + + ```plaintext + TIME LEVEL EXEC. ID ... NEW STATE COMMENT + ... [The output rows like the ones you've shown] ... + 16:46:04 JobLevel 2 Pending Completed + ``` + +2. **Filter the history by event type**: + + Filter the job history by the event type: + + ```bash + bacalhau job history j-6f2bf0ea-ebcd-4490-899a-9de9d8d95881 --event-type job + ``` + + Expected output: + + ```plaintext + TIME LEVEL EXEC. ID NODE ID REV. PREVIOUS STATE NEW STATE COMMENT + 16:46:03 JobLevel 1 Pending Pending Job created + 16:46:04 JobLevel 2 Pending Completed + ``` + +3. **Filter the history by execution ID**: + + Filter the job history by a specific execution ID: + + ```bash + bacalhau job history j-6f2bf0ea-ebcd-4490-899a-9de9d8d95881 --execution-id e-99362435 + ``` + + Expected output: + + ```plaintext + TIME LEVEL EXEC. ID ... NEW STATE COMMENT + ... [The output rows for the specific execution ID] ... + 16:46:04 ExecutionLevel e-99362435 QmTSJgdN 6 BidAccepted Completed + ``` + +4. **Retrieve the history in YAML format**: + + Get the job history in YAML format: + + ```bash + bacalhau job history j-6f2bf0ea-ebcd-4490-899a-9de9d8d95881 --output yaml + ``` + + Expected output: + + ```yaml + ... [The YAML formatted output] ... + ``` + + +## list + +# Config List + +## Description + +The `bacalhau config list` command lists the configuration keys and values of the bacalhau node. This command is useful for understanding how configuration keys map to their respective values, aiding in the use of the `bacalhau config set` command. + +Note: Configuration values displayed by this command represent the settings that will be applied when the bacalhau node is next restarted. It is important to note that these values may not reflect the current operational configuration of an active bacalhau node. The displayed configuration is relevant and accurate for a node that is either not currently running or that has been restarted after the execution of this command. + +## Usage + +```bash +bacalhau config list [flags] +``` + +## Flags + +- `-h`, `--help`: + - Description: Displays help information for the `list` sub-command. +- `--hide-header`: + - Description: Do not print the column headers when displaying the results. + - Default: `false` +- `--no-style`: + - Description: Removes all styling from the table output, displaying raw data. + - Default: `false` +- `--output format`: + - Description: Determines the format in which the output is displayed. Available formats include Table, JSON, and YAML. + - Options: `json`, `yaml`, `table` + - Default: `table` +- `--pretty`: + - Description: Formats the output for enhanced readability. This flag is relevant only when using JSON or YAML output formats. + - Default: `true` +- `--wide`: + - Description: Prints full values in the table results without truncating any information. + - Default: `false` + +## Examples + +### Listing the Bacalhau nodes configuration settings + +1. **Basic Usage**: + + **Command**: + + ```bash + bacalhau config list + ``` + + **Output**: + + ```bash + KEY VALUE + + ... + ``` + +2. **Output in JSON format**: + + **Command**: + + ```bash + bacalhau config list --output json --pretty + ``` + + **Output**: + + ```json + [ + { + "Key": "", + "Value": + }, + ... + ] + ``` + + +## list + +# List + +## Description + +The `bacalhau job list` command provides a listing of all submitted jobs. This command offers an overview of all tasks and processes registered in the system, allowing users to monitor and manage their jobs effectively. + +## Usage + +```bash +bacalhau job list [flags] +``` + +## Flags + +- `-h`, `--help`: + - Description: Display help for the `list` command. +- `--hide-header`: + - Description: Opts out of printing the column headers in the results. +- `--labels string`: + - Description: Filters jobs by labels. It's designed to function similar to Kubernetes label selectors. + - Default: `bacalhau_canary != true` +- `--limit uint32`: + - Description: Limits the number of results returned. + - Default: `10` +- `--next-token string`: + - Description: Uses the provided token for pagination. +- `--no-style`: + - Description: Strips all styling from the table output. +- `--order-by string`: + - Description: Organizes results based on a chosen field. Valid fields are `id` and `created_at`. +- `--order-reversed`: + - Description: Reverses the order of the displayed results. +- `--output format`: + - Description: Dictates the desired output format for the command. Options are `table`, `csv`, `json`, and `yaml`. + - Default: `table` +- `--pretty`: + - Description: Offers a more visually pleasing output for `json` and `yaml` formats. +- `--wide`: + - Description: Presents full values in the table results, preventing truncation. + +## Global Flags + +- `--api-host string`: + - Description: Defines the host for client-server communication via REST. Overridden by the `BACALHAU_API_HOST` environment variable, if set. + - Default: `bootstrap.production.bacalhau.org` +- `--api-port int`: + - Description: Sets the port for RESTful communication between the client and server. The `BACALHAU_API_PORT` environment variable takes precedence if set. + - Default: `1234` +- `--log-mode logging-mode`: + - Description: Designates the desired log format. Options include `default`, `station`, `json`, `combined`, and `event`. + - Default: `default` +- `--repo string`: + - Description: Points to the bacalhau repository location. + - Default: `$HOME/.bacalhau` + +## Examples + +1. **List all jobs**: + + Execute the command to list all the jobs: + + ```bash + bacalhau job list + ``` + + Expected output: + + ```plaintext + CREATED ID JOB TYPE STATE + 08:19:07 d78a4cb4 docker batch Completed + 04:17:21 e45f31a7 docker batch Completed + 04:53:50 f4993f62 docker batch Completed + ... (trimmed for brevity) ... + ``` + +2. **Limit the list to the last two jobs**: + + Limit the list to display only the last two jobs: + + ```bash + bacalhau job list --limit 2 + ``` + + Expected output: + + ```plaintext + CREATED ID JOB TYPE STATE + 03:14:16 19a26187 docker batch Completed + 21:47:21 2a53a13b docker batch Completed + ``` + +3. **Order the list by creation date in descending order**: + + Order the jobs by their creation date in a descending manner: + + ```bash + bacalhau job list --order-by created_at --order-reversed + ``` + + Expected output: + + ```plaintext + CREATED ID JOB TYPE STATE + 17:44:16 90e14efd docker batch Completed + 17:44:08 8204570c docker batch Completed + 17:43:50 f196521d docker batch Completed + ... (trimmed for brevity) ... + ``` + +4. **Filter the jobs by specific labels**: + + Display jobs that have specific labels: + + ```bash + bacalhau job list --labels "region in (us-east-1, us-east-2),env = prod" + ``` + + Expected output: + + ```plaintext + ... (filtered jobs) ... + ``` + +5. **Display the list in JSON format with pretty printing**: + + Get a limited list of jobs in a formatted JSON output: + + ```bash + bacalhau job list --limit 3 --output json --pretty + ``` + + Expected output: + + ```plaintext + ... [The JSON formatted output] ... + ``` + + +## list + +# List + +The `bacalhau node list` command is designed to provide users with a comprehensive list of network nodes along with details based on specified flags. + +## Description + +The `list` sub-command under the `bacalhau node` category enumerates information about nodes in the network. It supports various filtering, ordering, and output formatting options, allowing users to tailor the output to their needs. + +## Usage + +```bash +bacalhau node list [flags] +``` + +## Flags + +- `-h`, `--help`: + - Show the help message for the `list` command. +- `--hide-header`: + - Do not display the column headers in the output. +- `--filter-approval`: + - Only show nodes with the specified approval status. Valid values are: `approved`, `pending`, `rejected`. +- `--filter-status`: + - Only show nodes with the specified state. Valid values are: `healthy`, `unhealthy`, `unknown`. +- `--labels string`: + - Filter nodes based on labels. This follows the filtering format provided by Kubernetes, as shown in their documentation about labels. +- `--limit uint32`: + - Restrict the number of results displayed. +- `--next-token string`: + - Provide the next token for pagination. +- `--no-style`: + - Output the table without any style. +- `--order-by string`: + - Sort the results based on a specific field. Valid sorting fields are: `id`, `type`, `available_cpu`, `available_memory`, `available_disk`, `available_gpu`. +- `--order-reversed`: + - Display the results in reverse order. +- `--output format`: + - Choose the output format. Available options: `table`, `csv`, `json`, `yaml`. + - Default: `table`. +- `--pretty`: + - Enhance the visual appeal of the output. This is applicable only to `json` and `yaml` formats. +- `--show strings`: + - Determine the column groups to be displayed. Acceptable values are: `labels`, `version`, `features`, `capacity`. + - Default: `labels`, `capacity`. +- `--wide`: + - Display full values in the output table, without truncation. + +## Global Flags + +- `--api-host string`: + - Specify the host for client-server communication via REST. This gets ignored if the `BACALHAU_API_HOST` environment variable is defined. + - Default: `"bootstrap.production.bacalhau.org"`. +- `--api-port int`: + - Specify the port for RESTful communication between client and server. Gets overlooked if the `BACALHAU_API_PORT` environment variable is set. + - Default: `1234`. +- `--log-mode logging-mode`: + - Choose the desired log format. + - Options: `'default', 'station', 'json', 'combined', 'event'`. + - Default: `'default'`. +- `--repo string`: + - Point to the directory path of the bacalhau repository. + - Default: `"`$HOME/.bacalhau"\`. + +## Examples + +1. **Retrieve the list of nodes**: + + Execute the command to get a list of all nodes: + + ```bash + bacalhau node list + ``` + + Expected output: + + ```plaintext + ID TYPE LABELS CPU MEMORY DISK GPU + QmTSJgdN Compute Architecture=amd64 Operating-System=linux 3.2 / 11.7 GB / 77.8 GB / 1 / + git-lfs=True owner=bacalhau 3.2 11.7 GB 77.8 GB 1 + QmVXwmdZ Compute Architecture=amd64 Operating-System=linux 3.2 / 12.5 GB / 77.8 GB / 0 / + git-lfs=True owner=bacalhau 3.2 12.5 GB 77.8 GB 0 + QmXRdLru Compute Architecture=amd64 Operating-System=linux 3.2 / 12.5 GB / 78.0 GB / 0 / + git-lfs=True owner=bacalhau 3.2 12.5 GB 78.0 GB 0 + ... [Additional nodes information] ... + ``` + +2. **Filter the list of nodes by labels**: + + Execute the command to get a list of nodes with specific labels: + + ```bash + bacalhau node list --labels "Operating-System=linux,owner=bacalhau" + ``` + + Expected output: + + ```plaintext + ID TYPE LABELS CPU MEMORY DISK GPU + QmTSJgdN Compute Architecture=amd64 Operating-System=linux 3.2 / 11.7 GB / 77.8 GB / 1 / + git-lfs=True owner=bacalhau 3.2 11.7 GB 77.8 GB 1 + ... [Additional nodes information] ... + ``` + +3. **Order the list of nodes by available memory**: + + Execute the command to get the list of nodes ordered by available memory: + + ```bash + bacalhau node list --order-by available_memory + ``` + + Expected output: + + ```plaintext + ID TYPE LABELS CPU MEMORY DISK GPU + QmVXwmdZ Compute Architecture=amd64 Operating-System=linux 3.2 / 12.5 GB / 77.8 GB / 0 / + git-lfs=True owner=bacalhau 3.2 12.5 GB 77.8 GB 0 + ... [Additional nodes information] ... + ``` + +4. **Limit the number of nodes displayed and output in JSON format**: + + Execute the command to get a limited list of nodes in JSON format: + + ```bash + bacalhau node list --limit 3 --output json --pretty + ``` + + Expected output: + + ```json + [ + { + "PeerInfo": { + "ID": "QmTSJgdN7zCPAqBCkmdsdpFbiJV8bJ6zhoxK9N5xfar1sz", + ... [Additional node details] ... + }, + ... [Other nodes] ... + } + ] + ``` + + +## logs + +# Logs + +## Description + +The `bacalhau job logs` command allows users to retrieve logs from a job that has been previously submitted. This command is useful for tracking and debugging the progress and state of a running or completed job. + +## Usage + +```bash +bacalhau job logs [id] [flags] +``` + +## Flags + +1. `-f`, `--follow`: + - Description: This flag allows the user to follow the logs in real-time after fetching the current logs. It provides a continuous stream of log updates, similar to `tail -f` in Unix-like systems. +2. `-h`, `--help`: + - Description: Display help information for the `logs` command. + +## Global Flags + +1. `--api-host string`: + - Description: Specifies the host for the client and server to communicate through REST. This flag is disregarded if the `BACALHAU_API_HOST` environment variable is set. + - Default: `bootstrap.production.bacalhau.org` +2. `--api-port int`: + - Description: Sets the port for RESTful communication between the client and server. If the `BACALHAU_API_PORT` environment variable is available, this flag is ignored. + - Default: `1234` +3. `--log-mode logging-mode`: + - Description: Determines the desired log format. Available options include `default`, `station`, `json`, `combined`, and `event`. + - Default: `default` +4. `--repo string`: + - Description: Specifies the path to the bacalhau repository. + - Default: `$HOME/.bacalhau` + +## Examples + +1. **Display Logs for a Previously Submitted Job with Full ID**: + + **Command:** + + ```bash + bacalhau job logs j-51225160-807e-48b8-88c9-28311c7899e1 + ``` + + **Expected Output:** + + ```bash + [2023-09-24 09:01:32] INFO - Application started successfully. + [2023-09-24 09:01:33] DEBUG - Initializing database connections. + [2023-09-24 09:01:35] WARN - API rate limit approaching. + [2023-09-24 09:02:01] ERROR - Failed to retrieve data from endpoint: /api/v1/data. + [2023-09-24 09:05:00] INFO - Data sync completed with 4500 new records. + ``` + +2. **Follow Logs in Real-Time**: + + **Command:** + + ```bash + bacalhau job logs --follow j-51225160-807e-48b8-88c9-28311c7899e1 + ``` + + **Expected Output**: + + ```bash + [2023-09-24 11:30:02] INFO - User 'john_doe' logged in successfully. + [2023-09-24 11:30:15] DEBUG - Fetching data from cache for key: userSettings_john_doe. + [2023-09-24 11:31:05] WARN - High memory usage detected: 85% of allocated resources. + ... [Logs continue to appear in real-time] ... + ``` + +3. **Display Logs Using a Shortened ID**: + + **Command:** + + ```bash + bacalhau job logs j-ebd9bf2f + ``` + + **Expected Output:** + + ```bash + [2023-09-24 10:15:12] INFO - Application initialization sequence started. + [2023-09-24 10:15:13] DEBUG - Loading configurations from /config/app.json. + [2023-09-24 10:15:14] INFO - Connected to message broker successfully. + [2023-09-24 10:16:00] ERROR - Failed to send email notification to user@example.com. + ``` + + +## node + +# Node + +## Description + +The `bacalhau agent node` command retrieves information about the agent's node, providing insights into the agent's environment and aiding in debugging. + +## Usage + +```bash +bacalhau agent node [flags] +``` + +## Flags + +- `-h`, `--help`: + - Displays help information for the `node` sub-command. +- `--output format`: + - Defines the output format (either JSON or YAML). + - Options: `json`, `yaml` + - Default: `yaml` +- `--pretty`: + - Beautifies the output when using JSON or YAML formats. + +## Global Flags + +- `--api-host string`: + - The host for REST communication. Overrides the `BACALHAU_API_HOST` environment variable. + - Default: `bootstrap.production.bacalhau.org` +- `--api-port int`: + - The port for REST communication. Overridden if `BACALHAU_API_PORT` environment variable is set. + - Default: `1234` +- `--log-mode logging-mode`: + - Specifies the log format. Choices are: `default`, `station`, `json`, `combined`, `event`. + - Default: `default` +- `--repo string`: + - Path to the bacalhau repository. + - Default: \`\`$HOME/.bacalhau\` + +## Examples + +1. **Retrieve Node Information in Default Format (YAML)** + + ```bash + bacalhau agent node + ``` + +2. **Retrieve Node Information in JSON Format** + + ```bash + bacalhau agent node --output json + ``` + +3. **Retrieve Node Information in Pretty-printed JSON Format** + + ```bash + bacalhau agent node --output json --pretty + ``` + + +## reject + +# Reject + +The `bacalhau node reject` command offers administrators the ability to reject a compute node's request to join the cluster. + +## Description + +Using the `reject` sub-command, administrators can reject a node in the pending state from joining the cluster and receiving work. This feature is crucial for system administrators to manage the cluster and will stop the node from taking part in the cluster until approved. + +## Usage + +```bash +bacalhau node rejected [id] [flags] +``` + +## Flags + +- `[id]`: + - The unique identifier of the node you wish to describe. +- `-h`, `--help`: + - Displays the help documentation for the `describe` command. +- `-m message`: + - A message to be attached to the rejection action. + +## Global Flags + +- `--api-host string`: + - Specifies the host for client-server communication through REST. This flag is overridden if the `BACALHAU_API_HOST` environment variable is set. + - Default: `"bootstrap.production.bacalhau.org"` +- `--api-port int`: + - Designates the port for REST-based communication between client and server. This flag is overlooked if the `BACALHAU_API_PORT` environment variable is defined. + - Default: `1234` +- `--log-mode logging-mode`: + - Determines the log format preference. + - Options: `'default','station','json','combined','event'` + - Default: `'default'` +- `--repo string`: + - Points to the bacalhau repository's path. + - Default: `"`$HOME/.bacalhau"\` + +## Examples + +1. Reject a Node with ID `nodeID123`: + + ```bash + bacalhau node reject nodeID123 + ``` + +2. Reject a Node with an audit message: + + ```bash + bacalhau node reject nodeID123 -m "potentially bad" + ``` + + +## run + +[Content truncated - see full docs] + + +## set + +# Config Set + +## Description: + +The `bacalhau config set` command sets a value in the bacalhau node's configuration file. This command is used to modify the configuration file that the bacalhau node will reference for its settings. Key names in the configuration are case-insensitive. Additionally, the command validates the value being set based on the type of the configuration key, ensuring that only appropriate and valid configurations are applied. + +:::note +Changes made using this command will be applied to the configuration file, but they do not immediately affect the running configuration of an active bacalhau node. The modifications will take effect only after the node is restarted. +::: + +## Usage + +```bash +bacalhau config set +``` + +## Flags + +- `-h`, `--help`: + - Description: Displays help information for the `set` sub-command. + +## Examples + +### Configuring the Server API Port Value + +```bash +bacalhau config set node.serverapi.port 9999 + +bacalhau config list | grep serverapi.port + node.serverapi.port 9999 + +cat ~/.bacalhau/config.yaml +node: + serverapi: + port: 9999 +``` + +### Configuring the Logging Mode Value + +```bash +bacalhau config set node.loggingmode json + +bacalhau config list | grep loggingmode + node.loggingmode json + +cat ~/.bacalhau/config.yaml +node: + loggingmode: json +``` + +### Multiple Set commands append to the file + +```bash +bacalhau config set node.serverapi.port 9999 +bacalhau config set node.serverapi.host 0.0.0.0 +bacalhau config set node.loggingmode json + +cat ~/.bacalhau/config.yaml +node: + loggingmode: json + serverapi: + host: 0.0.0.0 + port: 9999 +``` + +### Set command value validation + +#### Example of invalid logging mode value + +```bash +bacalhau config set node.loggingmode some-invalid-value + +Error: setting "node.loggingmode": "some-invalid-value" is an invalid log-mode (valid modes: ["default" "station" "json" "combined" "event"]) +``` + +#### Example of invalid time duration value + +```bash +bacalhau config set node.volumesizerequesttimeout 10days + +Error: setting "node.volumesizerequesttimeout": time: unknown unit "days" in duration "10days" +``` + + +## stop + +# Stop + +## Description + +The `bacalhau job stop` command allows users to terminate a previously submitted job. This is useful in scenarios where there's a need to halt a running job, perhaps due to misconfiguration or changed priorities. + +## Usage + +```bash +bacalhau job stop [id] [flags] +``` + +## Flags + +- `--quiet`: + - Description: If provided, the command will not display any output, neither to the standard output (stdout) nor to the standard error (stderr). +- `-h`, `--help`: + - Description: Displays help information for the `stop` command. + +## Global Flags + +- `--api-host string`: + - Description: Specifies the host used for RESTful communication between the client and server. The flag is disregarded if `BACALHAU_API_HOST` environment variable is set. + - Default: `bootstrap.production.bacalhau.org` +- `--api-port int`: + - Description: Determines the port for REST communication. If `BACALHAU_API_PORT` environment variable is set, this flag will be ignored. + - Default: `1234` +- `--log-mode logging-mode`: + - Description: Selects the desired log format. Options include: `default`, `station`, `json`, `combined`, and `event`. + - Default: `default` +- `--repo string`: + - Description: Defines the path to the bacalhau repository. + - Default: `$HOME/.bacalhau` + +## Examples + +1. **Stop a Specific Job**: + + If you wish to halt the execution of a job, you can utilize the `stop` command. Here's how you can achieve that: + + **Command:** + + ```bash + bacalhau job stop j-10eb97de-14cd-4db4-96ec-561bb943309a + ``` + + **Expected Output:** + + ```plaintext + Checking job status + + Connecting to network ................ done ✅ 0.0s + Verifying job state ................ done ✅ 0.2s + Stopping job ................ done ✅ 0.1s + + Job stop successfully submitted with evaluation ID: 397fd425-8b1a-491e-952a-0632492e7ece + ``` + +2. **Silently Stop a Job**: + + If you prefer to terminate a job without seeing any verbose feedback or messages, the `--quiet` option can be used. + + **Command:** + + ```bash + bacalhau job stop j-63b5ec0c-b5bf-4398-a152-b46c07abe52a --quiet + ``` + + **Expected Output:** + + ```plaintext + [No output displayed as the operation is run quietly.] + ``` + + +## version + +# Version + +The `bacalhau agent version` command is used to obtain the version of the bacalhau agent. + +## Description: + +Using this command, users can quickly retrieve the version of the agent, allowing them to confirm the specific release of the software they are using. + +## Usage: + +```bash +bacalhau agent version [flags] +``` + +## Flags: + +- **`-h`, `--help`**: + - Show help for the `version` command. +- **`--output format`**: + - Defines the output format of the command's results. Accepted formats include "json" and "yaml". +- **`--pretty`**: + - Used for pretty printing the output, enhancing readability. This flag is applicable only for the "json" and "yaml" output formats. + +## Global Flags: + +- **`--api-host string`**: + - Designates the host for client-server communication via REST. If the `BACALHAU_API_HOST` environment variable is present, this flag will be disregarded. + - Default: `"bootstrap.production.bacalhau.org"` +- **`--api-port int`**: + - Defines the port for client-server communication through REST. This flag becomes irrelevant if the `BACALHAU_API_PORT` environment variable is specified. + - Default: `1234` +- **`--log-mode logging-mode`**: + - Specifies the desired logging format. + - Options: `'default','station','json','combined','event'` + - Default: `'default'` +- **`--repo string`**: + - Indicates the path to the bacalhau repository. + - Default: `"`$HOME/.bacalhau"\` + +## Examples + +1. **Retrieve the agent version**: + + Execute the command to get the agent version: + + ```bash + bacalhau agent version + ``` + + Expected output: + + ```bash + Bacalhau v0.0.0-xxxxxxx + BuildDate 2023-09-22 16:03:44 +0000 UTC + GitCommit 0fe81cb488f666845ac72c73a4b804aaa658e511 + ``` + +2. **Retrieve the agent version in JSON format**: + + ```bash + bacalhau agent version --output json + ``` + + Expected output: + + ```bash + {"major":"0","minor":"0","gitversion":"v0.0.0-xxxxxxx","gitcommit":"0fe81cb488f666845ac72c73a4b804aaa658e511","builddate":"2023-09-22T16:03:44Z","goos":"linux","goarch":"amd64"} + ``` + +3. **Retrieve the agent version in Pretty-printed JSON format**: + + ```bash + bacalhau agent version --output json --pretty + ``` + + Expected output: + + ```bash + { + "major": "0", + "minor": "0", + "gitversion": "v0.0.0-xxxxxxx", + "gitcommit": "0fe81cb488f666845ac72c73a4b804aaa658e511", + "builddate": "2023-09-22T16:03:44Z", + "goos": "linux", + "goarch": "amd64" + } + ``` + diff --git a/static/llms/concepts.txt b/static/llms/concepts.txt new file mode 100644 index 0000000..7e1cc69 --- /dev/null +++ b/static/llms/concepts.txt @@ -0,0 +1,159 @@ +# Bacalhau Concepts & Architecture + +## architecture + +# Architecture + +Bacalhau employs a distributed, node-based architecture that brings compute operations closer to data. Built around a single self-contained binary that serves multiple roles, Bacalhau makes it remarkably simple to deploy and scale a distributed compute network. + +## High-Level Overview + +1. **User Submits a Job**: The user, through the Bacalhau CLI or API, sends a job definition to the Orchestrator. Jobs can be submitted in two ways: + * **Imperative**: `bacalhau docker run ...` with command-line arguments + * **Declarative**: `bacalhau job run ` using a YAML specification file +2. **Orchestrator Schedules Tasks**: Based on resource availability, data location, and job requirements, the Orchestrator assigns tasks to Compute Nodes. +3. **Compute Nodes Execute Tasks**: Each Compute Node pulls the necessary image(s), mounts or fetches input data (local, S3, etc.), and runs the task in an isolated environment. +4. **Results Publication**: Once the task completes, outputs are published to configured storage. The Orchestrator updates the job's status accordingly. + +All of these components run from the same Bacalhau binary, just in different modes, making deployment remarkably simple. + +## Core Components + +### Orchestrators + +* **Core Role**: Receives job submissions, maintains job state, and coordinates scheduling +* **NATS Server**: Acts as a messaging infrastructure hub +* **Communication**: Broadcasts scheduling decisions and listens for status updates + +### Compute Nodes + +* **Primary Function**: Execute containerized or WASM-based workloads +* **Resource Management**: Advertise available CPU, memory, GPU, and storage capacity +* **Data Handling**: Fetch or mount input data from various sources and publish results +* **Isolation**: Run tasks in Docker containers or WASM environments + +## Modular Architecture + +Bacalhau features a pluggable architecture with well-defined interfaces that enable extension without modifying core code: + +* **Execution Engine Interface**: Currently supports Docker and WebAssembly (WASM) workloads, with a clean API for adding new execution environments +* **Storage Provider Interface**: Plug in various storage backends (S3, HTTP/HTTPS, local paths, IPFS) for both input and output handling +* **Publisher Interface**: Easily add new ways to publish and share computation results + +## Data-Centric Design + +A key differentiator of Bacalhau is its data-centric approach: + +* **Data Locality**: The system intelligently schedules jobs on nodes with local access to data +* **Minimal Transfer**: Moves computation to data rather than moving large datasets +* **Data Sovereignty**: Process sensitive data within security boundaries without requiring it to leave premises +* **Cross-Organizational Computation**: Enable collaborative analysis on protected datasets without exposing raw data + +## Network Resilience + +Bacalhau's architecture is designed to maintain operations even during network disruptions: + +* **Event-Driven State**: All system events are stored in local ledgers and shared during normal operation +* **Independent Operation**: Nodes continue functioning during network outages +* **State Reconciliation**: When network partitions heal, nodes exchange missed events +* **Local Decision Making**: Orchestrators can make scheduling decisions with available information + +## Deployment Models + +Bacalhau's single-binary architecture supports flexible deployment configurations: + +* **Single Node**: Run orchestrator and compute services on one machine (ideal for development) +* **Regional Cluster**: Distributed compute nodes within a single geographic region +* **Global Cluster**: Compute network spanning multiple regions and data centers + +## Security Considerations + +* **Execution Environments**: Tasks run in Docker containers or WASM environments with appropriate resource limits +* **Access Control**: Each node requires valid credentials for accessing private data sources +* **Data Boundaries**: Computation happens within defined security perimeters, protecting sensitive information + +## Observability + +* **Metrics & Logging**: Each node can expose metrics on resource usage and job performance +* **Event Tracking**: Orchestrators record job lifecycle events for monitoring and auditing + + +## key-concepts + +# Key Concepts + +Bacalhau is built around a few core ideas and terminologies. If you're new to Bacalhau, here's what you need to know: + +## Distributed Compute Orchestration + +Bacalhau coordinates computing workloads across a network of machines, intelligently matching jobs to resources. + +* Bacalhau acts as a dispatcher: You submit jobs (e.g., container workloads), and it finds the best node to run them based on available resources, data location, and constraints. + +## Bring the Compute to the Data + +Instead of moving data to compute, Bacalhau moves compute to where data lives, reducing network overhead and improving efficiency. + +* Traditionally, big data solutions shuffle large datasets across networks to a central compute cluster. +* Bacalhau inverts this approach: it places compute tasks where the data already resides—whether in local storage, an S3 bucket, or other storage providers—reducing unnecessary data movement. + +## Jobs & Executions + +Bacalhau organizes work in a hierarchy that enables efficient resource allocation and parallelization. + +* A **Job** defines the overall workflow (e.g., "run a Docker image with these arguments"). +* A job can be broken into multiple **Executions** that run in parallel across different compute nodes. +* Bacalhau optimizes these executions based on data locality and available resources. + +## Job Types + +Bacalhau supports various execution patterns to accommodate different workload requirements: + +* **Batch Jobs**: One-time execution of a workload, typically for data processing tasks that run to completion. +* **Ops Jobs**: Administrative or operational tasks, often for system maintenance or monitoring. +* **Daemon Jobs**: Long-running background processes that perform ongoing work. +* **Service Jobs**: Web services or APIs that need to remain available and respond to requests. + +## Node Types + +The Bacalhau network consists of specialized components, each with specific responsibilities: + +* **Orchestrator Node**: Receives job submissions, schedules executions, and monitors state. Started with `bacalhau serve --orchestrator`. +* **Compute Node**: Executes workloads locally, typically requiring Docker or another runtime. Started with `bacalhau serve --compute`. +* **Hybrid Node**: Serves both roles at once—often used for local dev or small setups. Started with `bacalhau serve --orchestrator --compute`. + +## Execution Engines + +Bacalhau runs your code through pluggable runtime environments: + +* Bacalhau supports multiple execution engines through its modular architecture: + * **Docker**: For container-based workloads + * **WebAssembly (WASM)**: For lightweight, sandboxed execution +* The framework is designed to accommodate additional engines as needed. + +## Storage Providers + +Bacalhau can access data from various sources through a clean, extensible interface: + +* Bacalhau can mount data from various sources through its flexible storage provider interface: + * S3-compatible storage + * HTTP/HTTPS URLs + * Local filesystems + * IPFS + * And more via storage provider plugins + +## Publisher + +After execution, Bacalhau ensures your results are accessible where you need them: + +* After a job finishes, its results can be published to a specific backend—like local disk, S3 or IPFS—so they're easy to retrieve. + +## Communication Layer + +A reliable messaging system allows Bacalhau components to coordinate effectively: + +* Bacalhau uses NATS.io as its communication backbone: + * Orchestrators act as NATS servers + * Compute nodes connect as NATS clients + * This provides reliable, scalable messaging between components + diff --git a/static/llms/getting-started.txt b/static/llms/getting-started.txt new file mode 100644 index 0000000..5f9e648 --- /dev/null +++ b/static/llms/getting-started.txt @@ -0,0 +1,283 @@ +# Getting Started with Bacalhau + +## installation + +# Installation + +This section explains how to install Bacalhau on your machine, verify it's working, and understand basic requirements. Bacalhau is distributed as a single self-contained binary that can function as a client, orchestrator node, and compute node—greatly simplifying deployment and management of your distributed compute network. + +## Install the Bacalhau Binary + +To install the CLI, choose your environment, and run the command(s) below. + + + +### Verify the Installation + +```bash +bacalhau version +``` + +This should print: + +- The local binary version +- The version of the orchestrator this client is connected to, if any +- The latest available version of bacalhau in case you are running an outdated version. + +If you get `command not found`, verify your PATH includes the Bacalhau binary. + +### Upgrading Bacalhau + +To upgrade Bacalhau to the latest version, run the installation script. If Bacalhau is already installed, this will update it to the most recent version available. + +## Requirements & Tips + +- **Docker**: + - Must be installed and running on any **compute** node to handle Docker-based jobs. +- **AWS Credentials** (if you're using S3): + - For S3 inputs or outputs, the node needs valid AWS credentials (e.g., environment variables). +- **Running an Orchestrator & Compute**: + - See [Quick Start](./quick-start.mdx) for how to run a local or hybrid node with `bacalhau serve --orchestrator --compute`. +- **`bacalhau devstack`**: + - Perfect for local development or running tests. + +## Next Steps + +- Head over to Basic CLI Usage to learn how to submit, describe, and stop jobs. +- Check Common Workflows for steps on mounting data (S3, local folders) and publishing outputs. +- Explore References for advanced node management (Docker Compose, devstack, multi-node clusters). + + +## network-setup + +# Network Setup + +This guide explains how to set up Bacalhau networks for various deployment scenarios, from development environments to production deployments. + +## Introduction + +Bacalhau's architecture consists of two primary node types: + +- **Orchestrator nodes** that schedule and manage jobs +- **Compute nodes** that execute workloads + +Compute nodes connect to orchestrators, but don't need to be reachable by orchestrators or other compute nodes, making deployment simpler. + +## Getting Started with Bacalhau + +Choose the setup option that best matches your needs: + +| Setup Option | Best For | Key Benefit | +| ------------------- | ---------------------------------- | -------------------------------------------- | +| DevStack | Development & testing | Quick local setup with minimal configuration | +| Self-Hosted Network | Custom infrastructure requirements | Complete control over all components | + +### Option 1: Self-Hosted Network + +If you need to host your own orchestrator, follow these steps for a custom deployment. + +#### Setting Up an Orchestrator Node + +On your designated orchestrator machine: + +```bash +# Start an orchestrator-only node +bacalhau serve --orchestrator +``` + +Take note of this machine's IP address or hostname - you'll need it to connect compute nodes. + +#### Adding Compute Nodes + +On each machine that will execute jobs: + +```bash +# Start a compute-only node connected to your orchestrator +bacalhau serve --compute -c Compute.Orchestrators=:4222 +``` + +Replace `` with the actual IP address or hostname of your orchestrator. + +#### Verifying Your Cluster + +Check that all nodes are connected: + +```bash +# List all nodes in your network +bacalhau node list +``` + +You should see your orchestrator and all compute nodes listed. + +:::note +The setup described above creates an open network suitable for testing in trusted environments. For securing your network, refer to the Security Best Practices in the Reference section. +::: + +### Option 2: DevStack + +DevStack provides a pre-configured local environment perfect for development and testing. + +```bash +# Launch a complete development environment +bacalhau devstack +``` + +This pre-configures a transient orchestrator and compute nodes by default, giving you a complete environment for testing with minimal setup. + +You can submit jobs to your DevStack just like any other Bacalhau network: + +```bash +bacalhau docker run ubuntu:latest -- echo "Hello from DevStack!" +``` + +### Alternative Setup Methods + +These methods provide additional ways to set up Bacalhau for specific use cases. + +#### Single Hybrid Node + +For the simplest local setup, you can run a single node that acts as both orchestrator and compute: + +```bash +# Launch a combined orchestrator and compute node +bacalhau serve --orchestrator --compute +``` + +This starts Bacalhau in "hybrid mode" where: + +- The orchestrator handles job scheduling +- The compute service executes containers +- Both components run in the same process + +This option is useful for initial testing or for very small deployments. + +#### Docker Deployment + +Run Bacalhau in Docker for easier management: + +```bash +# Run an orchestrator node +docker run -p 4222:4222 ghcr.io/bacalhau-project/bacalhau:latest serve --orchestrator + +# Run a compute node using Docker-in-Docker +docker run --privileged -p 4222:4222 \ + ghcr.io/bacalhau-project/bacalhau:latest-dind \ + serve --compute -c Compute.Orchestrators=:4222 +``` + +The `bacalhau:latest-dind` image includes Docker-in-Docker capabilities required for compute nodes. + +#### Docker Compose Setup + +For a quick multi-node setup, Bacalhau provides Docker Compose examples that create a complete network suitable for testing: + +1. **Clone Network Setups Repository**\ + Clone the repository containing the network setups: + + ```bash + git clone https://github.com/bacalhau-project/bacalhau-network-setups.git + ``` + +2. **Navigate to a Specific Setup**\ + Change directory to your desired setup under `docker-compose`: + + ```bash + cd bacalhau-network-setups/docker-compose/ + ``` + +3. **Start the Network**\ + Use Docker Compose to bring up the network: + + ```bash + docker compose up + ``` + +These setups enable deployment and testing of Bacalhau across multiple nodes, including an orchestrator and persistent data storage. + +### Next Steps + +- Secure your network with our Security Guide +- Learn how to submit jobs to your network +- Explore common workflows for different use cases + + +## quick-start + +# Quick Start + +This Quick Start guide shows how to run your first Bacalhau job with minimal setup. Bacalhau's design as a single self-contained binary makes it incredibly easy to set up your own distributed compute network in minutes. + +## Prerequisites + +- Docker installed on any machine that runs a compute node +- Bacalhau CLI installed (see below) + +## 1. Installation + +1. Install Bacalhau using the one-liner below (Linux/macOS) or see the [installation guide](./installation.mdx) for Windows and Docker options. + +```bash +curl -sL https://get.bacalhau.org/install.sh | bash +``` + +2. Once installed, verify with: + +```bash +bacalhau version +``` + +## 2. Start a Hybrid Node + +Open a terminal and run: + +```bash +bacalhau serve --orchestrator --compute +``` + +- This command launches both an orchestrator and a compute node in one process +- Keep it running; you'll see logs indicating it's ready + +## 3. Submit a Data Analysis Job + +Bacalhau supports two primary methods of job submission: Imperative (CLI) and Declarative (YAML). We'll demonstrate a word count job on the classic novel Moby Dick. + + + +- The job downloads a sample dataset and processes it locally +- Bacalhau will display job progress until completion +- You'll receive a Job ID once the job is submitted + +## 4. Inspect the Job + +```bash +bacalhau job describe +``` + +- Replace `` with the actual ID printed in step 2 +- You can run `bacalhau job logs ` to just get the execution logs + +## 5. Retrieve Results + +Download and view your job results: + +```bash +# Download the results +bacalhau job get + +# View the analysis output +head job-*/outputs/moby-dick-counts.txt +``` + +:::note +You should see a word frequency analysis of the Moby Dick text file! +::: + +## 🎉 Success! + +You've just: + +1. Started a local Bacalhau network +2. Submitted a job using both imperative and declarative methods +3. Tracked job progress with detailed descriptions +4. Retrieved and viewed job results + diff --git a/static/llms/other.txt b/static/llms/other.txt new file mode 100644 index 0000000..b738275 --- /dev/null +++ b/static/llms/other.txt @@ -0,0 +1,1694 @@ +# Additional Bacalhau Documentation + +## Publishing Results + +# Publishing & Retrieving Results + +This guide explains how to configure output publishing and retrieve results from Bacalhau jobs across different storage systems. Proper output handling is essential for building effective data pipelines and workflows. + +## What You'll Learn + +- How Bacalhau's Publishers mechanism works +- How to configure different output destination types +- How to retrieve outputs from various storage systems +- How to choose the right publisher for your use case + +## Understanding Publishers and Result Paths + +In Bacalhau, you need to configure two key components for handling outputs: + +1. A **Publisher** defines where your job's output files are stored after execution +2. **Result Paths** specify which directories should be captured as job results. + + + +### Retrieving Local Outputs + +After your job completes, retrieve outputs using the `bacalhau job get` command: + +```bash +bacalhau job get +```` + +This will download all published outputs to your current directory. + +:::note + +- If you define a publisher without specifying result paths, only stdout and stderr will be uploaded to the chosen publisher +- If you define result paths without a publisher, the job will fail +- You can have multiple result paths, each capturing different directories + ::: + +## Publisher Types + +Bacalhau supports multiple publisher types to accommodate different needs and infrastructure requirements. + +### S3 Publisher + +The S3 Publisher uploads outputs to an Amazon S3 bucket or any S3-compatible storage service, such as MinIO. The compute node must have permission to write to the bucket, and the orchestrator must have permission to provide pre-signed URLs to download the results. + + + +### IPFS Publisher + +The IPFS Publisher uploads outputs to the InterPlanetary File System. Both the client (downloading the result) and the compute node must be connected to an IPFS daemon. + + + +### Local Publisher + +The Local Publisher saves outputs to the local filesystem of the compute node that ran your job. This is intended for **local testing only**, as it requires the client downloading the results to be on the same network as the compute node. + + + +If you are using the local publish, make SURE you have set the path to be available to your job. + +For example, in your config file for your node, you probably want to mount in the local file system: + +``` + +Compute: +AllowListedLocalPaths: - /outputs:rw - /etc/\*.conf:ro + +``` + +You can read more about that here: [Understanding Publishers and Results](publishing-results.mdx#understanding-publishers-and-result-paths) + +## Troubleshooting + +### No Outputs Found + +If you don't see expected outputs: + +1. Check that your job wrote to the directories specified in your `ResultPaths` +2. Verify the job completed successfully with `bacalhau job describe ` +3. Check for errors in the logs with `bacalhau job logs ` + +### S3 Publishing Issues + +For S3 publisher problems: + +1. Ensure compute nodes have proper IAM roles or credentials to write to the bucket +2. Check that the orchestrator has permissions to generate pre-signed URLs + +### IPFS Publishing Issues + +For IPFS publisher issues: + +1. Ensure IPFS daemon is running on both compute node and client +2. Check for network connectivity between nodes +3. Verify you have enough disk space for pinning +``` + + +## README + +# What is Bacalhau? + +Bacalhau is an open-source distributed compute orchestration framework designed to bring compute to the data. Instead of moving large datasets around networks, Bacalhau makes it easy to execute jobs close to the data's location, drastically reducing latency and resource overhead. + +### Why It Matters + +* **Highly Distributed Architecture**: Deploy compute networks that span regions, cloud providers, and on-premises datacenters—all working together as a unified system. +* **Resilient Operation**: Compute nodes operate effectively even with intermittent connectivity to orchestrators, maintaining service availability during network partitioning or isolation. +* **Data Sovereignty & Security**: Process sensitive data within security boundaries without requiring it to leave your premises, enabling computation while preserving data control. +* **Cross-Organizational Computation**: Allow specific vetted computations on protected datasets without exposing raw data, breaking data silos between organizations. +* **Resource Efficiency**: By minimizing data transfers, Bacalhau saves bandwidth costs and ensures jobs run faster. +* **High Scalability**: As your data and processing needs grow, simply add more compute nodes on demand—whether on-premises or in the cloud. +* **Ease of Integration**: Bacalhau works with existing container images (Docker, etc.), meaning you can leverage your current workflows without major rewrites. + +### Key Features + +1. **Single Binary Simplicity**: Bacalhau is a single self-contained binary that functions as a client, orchestrator, and compute node—making it incredibly easy to set up and scale your distributed compute network. +2. **Modular Architecture**: Bacalhau's design supports multiple execution engines (Docker, WebAssembly) and storage providers through clean interfaces, allowing for easy extension. +3. **Orchestrator-Compute Model**: A dedicated orchestrator coordinates job scheduling, while compute nodes run tasks—all from the same binary with different runtime modes. +4. **Flexible Storage Integrations**: Bacalhau integrates with S3, HTTP/HTTPS, and other storage systems, letting you pull data from various sources. +5. **Multiple Job Types**: Support for batch, ops, daemon, and service job types to accommodate different workflow requirements. +6. **Declarative & Imperative Submissions**: Define jobs in a YAML spec (declarative) or pass all arguments via CLI (imperative). +7. **Publisher Support**: Output results to local volumes, S3, or other storage backends—so your artifacts are readily accessible. + +### Use Cases + +Bacalhau's distributed compute framework enables a wide range of applications across different industries: + +
+ +Log Processing + +Process logs efficiently at scale by running distributed jobs directly at the source, reducing costs by up to 93% in bandwidth usage while improving real-time insights. Bacalhau supports various job types for log management: + +* **Daemon Jobs**: Continuously run on each node for real-time log aggregation and compression +* **Service Jobs**: Handle ongoing processing tasks like log aggregation and issue detection +* **Batch Jobs**: Execute on-demand in-depth analysis of historical log data +* **Ops Jobs**: Enable real-time querying of live logs for urgent investigations + +
+ +
+ +Distributed Data Warehousing + +Query and analyze data across multiple regions by deploying compute tasks directly where your data resides. This approach reduces latency, enhances performance, and ensures compliance with data sovereignty regulations. Bacalhau integrates with modern data tools like Apache Iceberg and DuckDB to enable: + +* Reduced data movement with local query execution +* Improved query performance through compute-data proximity +* Seamless scalability with dynamic node addition +* Compliance with data regulations through region-specific processing + +
+ +
+ +Fleet Management + +Efficiently manage distributed nodes across multiple environments with capabilities for: + +* Remote execution of commands without requiring SSH access +* Automated software deployment and configuration updates +* Real-time metrics and logs collection +* Targeted job execution based on node attributes +* Rapid incident response and automated recovery + +
+ +
+ +Distributed Machine Learning + +Train and deploy ML models across a distributed compute fleet, optimizing performance while keeping data in place: + +* Distribute training across multiple machines to handle larger models +* Process data locally to minimize network transfers +* Deploy inference jobs near users for low-latency predictions +* Support federated learning for privacy-sensitive applications + +
+ +
+ +Edge Computing + +Run compute tasks closer to the data source for applications requiring low latency and minimal bandwidth usage: + +* Process and analyze sensor, IoT, or video data in real time +* Perform pre-processing and filtering at the edge before sending refined data +* Distribute tasks across available edge resources dynamically +* Ensure data privacy by keeping computations near the source + +
+ +### How It Works + +Bacalhau's architecture enables you to create compute networks that bridge traditional infrastructure boundaries. When you submit a job, Bacalhau intelligently determines which compute nodes are best positioned to process the data based on locality, availability, and your defined constraints—without requiring data movement or constant connectivity. + +This approach is particularly valuable for: + +* Organizations with data that cannot leave certain security boundaries +* Multi-region operations where data transfer is expensive or impractical +* Scenarios where multiple parties need to collaborate on analysis without sharing raw data +* Edge computing environments with intermittent connectivity + +### Community + +Bacalhau has a very friendly community and we are always happy to help you get started: + +* [Join the Slack Community](https://bit.ly/bacalhau-project-slack) Go to **#bacalhau** channel – it is the easiest way to engage with other members in the community and get help. +* [Contributing](/community/ways-to-contribute) – learn how to contribute to the Bacalhau project. + + +## README + +# Other Specifications + + +## README + +# Job Specification + +A `Job` represents a discrete unit of work that can be scheduled and executed. It carries all the necessary information to define the nature of the work, how it should be executed, and the resources it requires. + +```yaml +Type: batch +Count: 1 +Priority: 50 +Meta: + version: "1.2.5" +Labels: + project: "my-project" +Constraints: + - Key: Architecture + Operator: '=' + Values: + - arm64 + - Key: region + Operator: '=' + Values: + - us-west-2 +Tasks: + #... +``` + +## `job` Parameters + +* **Name** `(string : )`: A logical name to refer to the job. Defaults to job ID. +* **Namespace** `(string: "default")`: The namespace in which the job is running. `ClientID` is used as a namespace in the public demo network. +* **Type** `(string: )`: The type of the job, such as `batch`, `ops`, `daemon` or `service`. You can learn more about the supported jobs types in the [Job Types](type.md) guide. +* **Priority** `(int: 0`): Determines the scheduling priority. +* **Count** `(int: )`:: Task associated with the job, which defines a unit of work within the job. Today we are only supporting single task per job, but with future plans to extend this. + +## Server-Generated Parameters + +The following parameters are generated by the server and should not be set directly. + +* **ID** `(string)`: A unique identifier assigned to this job. It's auto-generated by the server and should not be set directly. Used for distinguishing between jobs with similar names. +* **State** `(`[`State`](../other/state.md)`)`: Represents the current state of the job. +* **Version** `(int)`: A monotonically increasing version number incremented on job specification update. +* **Revision** `(int)`: A monotonically increasing revision number incremented on each update to the job's state or specification. +* **CreateTime** `(int)`: Timestamp of job creation. +* **ModifyTime** `(int)`: Timestamp of last job modification. + + +## README + +# Engines + + +## README + +# Sources + + +## README + +# Publishers + + +## auth + +# Authentication & Authorization + +## Introduction + +Robust authentication and authorization mechanisms are essential for maintaining security while enabling seamless collaboration. As of Bacalhau 1.7 release, we introduced a significant overhaul to its authentication and authorization systems, offering more flexibility, improved security, and better integration with enterprise environments. + +## 1. Bacalhau Authentication + +With Bacalhau 1.7, we have introduced three distinct authentication paths, each designed to cater to different use cases and environments. The authentication paths are: + +1. **Basic HTTP Authentication** +2. **API Tokens Auth** +3. **Single Sign-On via OAuth 2.0** + +### 1.1 HTTP Basic Authentication + +The simplest approach leverages the time-tested HTTP Basic Authentication protocol, allowing users to access Bacalhau APIs using traditional username and password credentials. These credentials can be defined in the Node Configuration file, which offers two options for password storage: + +- _Plain text passwords for simplicity and ease of setup_ +- _Bcrypt-hashed passwords for enhanced security_ + +For CLI usage, users simply need to set the environment variables `BACALHAU_API_USERNAME` and `BACALHAU_API_PASSWORD`. For direct API calls, the standard Basic Authorization header with base64-encoded credentials can be used. + +Below is a sample orchestrator config file that defines 3 users that can authenticate through basic auth. + +```yaml +Orchestrator: + Enabled: true +API: + Port: 1234 + Auth: + Users: + # User with plain text password + - Alias: Admin User + Username: admin + Password: secureAdminPassword + # The Capabilities section will be covered + # in the Authorization section below + Capabilities: + - Actions: ['*'] + + # User with limited permissions and plain text password + - Alias: Read Only User + Username: reader + Password: readerPassword + # The Capabilities section will be covered + # in the Authorization section below + Capabilities: + - Actions: ['read:*'] + + # User with bcrypt hashed password + - Alias: Job Manager + Username: jobmanager + # This is a bcrypt password hash for the password "MySecretPassword" + Password: '$2a$10$3ZvxUe5OudgRIQQheomjMO/Ufx1Bb04SH/y0PXnR19oDRXNGps3r2' + # The Capabilities section will be covered + # in the Authorization section below + Capabilities: + - Actions: ['read:job', 'write:job', 'read:node'] +``` + +In the above configuration: + +1. The first two users have plain text passwords, while the third uses a BCRYPT hashed password for added security. +2. We have three users with different permission levels. These capabilities will be covered in detail in the authorization section below. + +To help users and operators generate secure hashed passwords, a convenient CLI command was added that generates a BCRYPT hash of a password of your choosing. This command takes a plain string and converts it into a BCRYPT hash. + +```shell +bacalhau auth hash-password +``` + +To use this configuration with the Bacalhau CLI, you would set the following environment variables: + +```shell +# For admin access +export BACALHAU_API_USERNAME=admin +export BACALHAU_API_PASSWORD=secureAdminPassword + +# For read-only access +export BACALHAU_API_USERNAME=reader +export BACALHAU_API_PASSWORD=readerPassword + +# For job management +# Please note: Set the password env variable to the actual password, +# not the hashed password. +export BACALHAU_API_USERNAME=jobmanager +export BACALHAU_API_PASSWORD=MySecretPassword +``` + +For direct API calls, for example by using curl, you would encode the credentials in base64: + +```shell +# For admin (base64 of "admin:secureAdminPassword") +curl -X GET -H "Authorization: Basic YWRtaW46c2VjdXJlQWRtaW5QYXNzd29yZA==" "http://orchestrator:1234/api/v1/orchestrator/nodes" + +# For reader (base64 of "reader:readerPassword") +curl -X GET -H "Authorization: Basic cmVhZGVyOnJlYWRlclBhc3N3b3Jk" "http://orchestrator:1234/api/v1/orchestrator/nodes" + +# For Job Manager (base64 of "jobmanager:MySecretPassword") +curl -X GET -H "Authorization: Basic am9ibWFuYWdlcjpNeVNlY3JldFBhc3N3b3Jk" "http://orchestrator:1234/api/v1/orchestrator/nodes" +``` + +--- + +### 1.2 Authentication through API Tokens + +For applications and scenarios where password-based authentication isn't ideal, Bacalhau 1.7 introduces API token support. Instead of username and password pairs, users can generate and use API keys as bearer tokens in authorization headers. + +Configuration is straightforward – API keys are defined in the orchestrator config under user profiles. To use them with the Bacalhau CLI, users set the `BACALHAU_API_KEY` environment variable. For direct API access, the token is included in the Authorization header using the Bearer scheme. + +Please note that API Keys are opaque tokens. + +Here's a sample configuration for API tokens in Bacalhau: + +```yaml +Orchestrator: + Enabled: true +API: + Port: 1234 + Auth: + Users: + # Administrator API token with full access + - Alias: Admin API Token + APIKey: 8F42A91D7C6E4B3DA5E9F8C12B76D3A4 + # The Capabilities section will be covered + # in the Authorization section below + Capabilities: + - Actions: ['*'] + + # Read-only API token + - Alias: Monitoring Token + APIKey: C5D8E3F1A7B94026895C1D4E3F2A0B78 + # The Capabilities section will be covered + # in the Authorization section below + Capabilities: + - Actions: ['read:*'] + + # Job management API token + - Alias: CI/CD Pipeline Token + APIKey: 2E8D7F5B3A9C41608D2E6B7F4A5C3D9E + # The Capabilities section will be covered + # in the Authorization section below + Capabilities: + - Actions: ['read:job', 'write:job', 'read:node'] + + # Agent management API token + - Alias: Agent Management Token + APIKey: 1A3B5C7D9E0F2G4H6I8J0K2L4M6N8P0 + # The Capabilities section will be covered + # in the Authorization section below + Capabilities: + - Actions: ['read:agent', 'write:agent'] +``` + +In this configuration: + +1. We have four API tokens with different permission levels: + - An administrator token with full access to all capabilities + - A monitoring token with read-only access to all resources + - A CI/CD pipeline token that can view nodes and has full control over jobs + - An agent management token that has full control over agents +2. Each token has a unique, randomly generated API key. You should generate strong, unique keys for your production environment using a secure random generator. + +Please note that API keys do not support BCRYPT hashing. + +To use these API tokens with the Bacalhau CLI, you would set the following environment variable: + +```shell +export BACALHAU_API_KEY=8F42A91D7C6E4B3DA5E9F8C12B76D3A4 +``` + +For direct API calls, for example by using curl, you would use the Bearer token authentication scheme: + +```shell +curl -X GET -H "Authorization: Bearer 8F42A91D7C6E4B3DA5E9F8C12B76D3A4" "http://orchestrator:1234/api/v1/orchestrator/nodes" +``` + +--- + +### 1.3 Single Sign-On via OAuth 2.0 + +Perhaps the most significant addition since Bacalhau 1.7 is the support for OAuth 2.0 using the Device Code Flow. This enables Bacalhau to integrate seamlessly with enterprise identity providers such as Okta, Auth0, Azure Active Directory, and Google SSO. + +This approach eliminates the need to define users directly in Bacalhau's configuration, instead delegating user management to the identity provider – a considerable advantage in corporate environments with existing identity infrastructure. + +The configuration process involves specifying OAuth 2.0 endpoints, client IDs, and desired scopes. When users need to authenticate, they run `bacalhau auth sso login`, which presents a device code and URL. After completing authentication through their browser, they receive a JWT token that's automatically used for subsequent API calls (_this token exchange will be done seamlessly and the user is not required to perform any extra actions_). + +Here's a sample configuration for OAuth 2.0 SSO in Bacalhau: + +```yaml +Orchestrator: + Enabled: true +API: + Port: 1234 + Auth: + Oauth2: + # Identity provider details, those are names for your own reference only + ProviderId: 'okta' + ProviderName: 'Okta SSO' + + # OAuth 2.0 endpoints - Device Code Endpoint + DeviceAuthorizationEndpoint: 'https://your-domain.okta.com/oauth2/v1/device/authorize' + # The endpoint used to get the JWT token + TokenEndpoint: 'https://your-domain.okta.com/oauth2/v1/token' + # The Expected issuer, should match the issuer in the JWT token + Issuer: 'https://your-domain.okta.com' + # The JWKS URI + JWKSUri: 'https://your-domain.okta.com/.well-known/jwks.json' + + # Client details + DeviceClientId: '0ab2c3d4e5f6g7h8i9j0' + # CLI polling interval to check if the device code was approved + PollingInterval: 5 + + # Audience: Expected "aud" in the JWT token + Audience: 'https://bacalhau.your-company.com/api' + + # Scopes requested in the token exchange + Scopes: + - 'openid' + - 'profile' + - 'email' +``` + +For this to setup work properly: + +1. Register an OAuth 2.0 application in your identity provider (Okta, Auth0, Azure AD, etc.) +2. Configure it to support the **Device Code Flow**. Make sure the provider supports OAuth2 Device code flow. +3. Set up appropriate roles or groups in your identity provider to map to Bacalhau permissions + +The permission mapping would happen in your identity provider. For example, in Okta you might create: + +- A "Bacalhau Admins" group with permissions: `["*"]` +- A "Bacalhau Readers" group with permissions: `["read:*"]` +- A "Bacalhau Job Managers" group with permissions: `["read:job", "write:job", "read:node"]` + +These permissions should be included in the JWT token under the custom claim `permissions`. + +To authenticate using this setup, users would run: + +```shell +# Login +bacalhau auth sso login + +# Logout +bacalhau auth sso logout +``` + +Then the CLI would display something like this: + +```shell +To login, please: + +1. Open this URL in your browser: /usr/local/bin/greet && \ + chmod +x /usr/local/bin/greet +``` + +### Step 2: Build and Test Your Container Locally + +Build your Docker image: + +```bash +docker build -t your-registry/simple-greeter:latest . +``` + +Test it locally: + +```bash +docker run --rm your-registry/simple-greeter:latest sh -c 'greet "Local Test"' +``` + +### Step 3: Push Your Container to a Registry + +```bash +# For multi-architecture support (recommended) +docker buildx build --platform linux/amd64,linux/arm64 --push -t your-registry/simple-greeter:latest . + +# For single architecture +docker push your-registry/simple-greeter:latest +``` + +## Running Your Custom Container on Bacalhau + +Run your container on Bacalhau: + +```bash +bacalhau docker run your-registry/simple-greeter:latest \ + sh -c 'greet "Bacalhau"' +``` + +View your job logs: + +```bash +bacalhau job logs JOB_ID +``` + + +## docker + +# Docker + +Docker Engine is one of the execution engines supported in Bacalhau. It allows users to run tasks inside Docker containers, offering an isolated and consistent environment for execution. Below are the parameters to configure the Docker Engine. + +## `Docker` Engine Parameters + +* **Image** `(string: )`: Specifies the Docker image to use for task execution. It should be an image that can be pulled by Docker. +* **Entrypoint** `(string[]: )`: Allows overriding the default entrypoint set in the Docker image. Each string in the array represents a segment of the entrypoint command. +* **Parameters** `(string[]: )`: Additional command-line arguments to be included in the container’s startup command, appended after the entrypoint. +* **EnvironmentVariables** `(string[]: )`: Sets environment variables within the Docker container during task execution. Each string should be formatted as `KEY=value`. +* **WorkingDirectory** `(string: )`: Sets the path inside the container where the task executes. If not specified, it defaults to the working directory defined in the Docker image. + +### Example + +Here’s an example of configuring the Docker Engine within a job or task using YAML: + +```yaml +Engine: + Type: "Docker" + Params: + Image: "ubuntu:20.04" + Entrypoint: + - "/bin/bash" + - "-c" + Parameters: + - "echo Hello, World!" + EnvironmentVariables: + - "MY_ENV_VAR=myvalue" + WorkingDirectory: "/app" +``` + +In this example, the task will be executed inside an Ubuntu 20.04 Docker container. The entrypoint is overridden to execute a bash shell that runs an echo command. An environment variable MY\_ENV\_VAR is set with the value myvalue, and the working directory inside the container is set to /app. + + +## downloading-results + +# Downloading Results + +After a Bacalhau job completes, you'll need to retrieve the output files generated by your job. This guide explains the basics of downloading job results. + +## What You'll Learn + +- How to specify output paths in your jobs +- How to retrieve job results using the CLI + +## Getting Job Results + +To download the results of a completed job: + +```bash +bacalhau job get +``` + +This command downloads all outputs from the job to your current directory. + +### Specifying an Output Directory + +You can specify where to save the downloaded results: + +```bash +bacalhau job get --output-dir /path/to/save +``` + +### Download Timeout Setting + +For larger downloads, you can adjust the timeout: + +```bash +bacalhau job get --download-timeout-secs 10m +``` + +## Specifying Job Outputs and Publisher + +When submitting a job, you need to define which files or directories should be collected as outputs, and where those outputs should be published. + +### Using Command Line + +For Docker jobs, use the `--output` flag to define outputs and the `--publisher` flag to specify where to publish the results: + +```bash +bacalhau docker run \ + --output results:/outputs \ + --publisher s3://my-bucket/results-folder \ + ubuntu:latest \ + -- echo "Hello, World!" > /outputs/hello.txt +``` + +This tells Bacalhau to: + +1. Collect everything in the `/outputs` directory of the container +2. Publish it to the specified S3 bucket and path +3. Make it available for download with `bacalhau job get` + +### Using Declarative Submission + +You can also define outputs in a job specification file: + +```yaml +Type: batch +Count: 1 +Tasks: + - Name: main + Engine: + Type: docker + Params: + Image: python:3.9 + Entrypoint: + - 'python' + - '-c' + - "import os; os.makedirs('/outputs', exist_ok=True); open('/outputs/result.txt', 'w').write('Analysis complete!')" + Publisher: + Type: s3 + Params: + Bucket: my-bucket + Key: results-folder + ResultPaths: + - Name: results + Path: /outputs +``` + +Submit this job using: + +```bash +bacalhau job run job-spec.yaml +``` + +### Multiple Output Paths + +You can specify multiple output paths in a single job: + +```bash +bacalhau docker run \ + --output logs:/var/log \ + --output results:/outputs \ + --publisher s3://my-bucket/results-folder \ + ubuntu:latest \ + -- +``` + +## Downloaded Results Structure + +After running `bacalhau job get`, the results will be organized in a directory structure like this: + +``` +. +├── job-j-6bafb8d4 +│ ├── exitCode +│ ├── outputs +│ │ └── file1 +│ ├── stderr +│ └── stdout +``` + +The directory structure includes: + +- `exitCode`: Contains the exit code of the job +- `outputs`: Contains all the files from the job's specified output directories +- `stderr`: Captures any error output from the job +- `stdout`: Captures the standard output from the job + + +## faqs + +# FAQs + +## How do I restrict what nodes my jobs run on? + +You can describe each node with labels in a `key=value` format, which can later be used as conditions for choosing nodes to run your jobs on: + +```bash +bacalhau config set Labels=NodeType=WebServer +``` + +For multiple labels, use comma-separated values: + +```bash +bacalhau config set Labels=foo=bar,baz=qaz +``` + +## How do I specify the orchestrator for my compute node? + +The `Compute.Orchestrator` field in the config tells the Bacalhau compute node where to connect: + +```bash +bacalhau config set Compute.Orchestrators=my-great-orchestrator.com +``` + +You can add protocol and port if needed: + +```bash +bacalhau config set Compute.Orchestrators=nats://my-great-orchestrator.com:4222 +``` + +## How do I enable the WebUI? + +By default, the WebUI for Bacalhau is disabled for security reasons. To enable it: + +```bash +bacalhau config set WebUI.Enabled=true +``` + +## Can I run non-Docker jobs? + +Yes! You can run programs using WebAssembly instead. Refer to the [WebAssembly onboarding documentation](../../references/developers/workload-onboarding/wasm.md) for instructions. + +## How do I see a job's progress while it's running? + +Use the job describe command with your job ID: + +```bash +bacalhau job describe b4491a4a-7b55-4fa7-a5af-80f3c99bc379 +``` + +If your job writes to stdout or stderr while running, you can also view the output with the `logs` command. + +## Can I stop a running job? + +Yes. Given a valid `job ID`, you can use the `stop` command to cancel the job and stop it from running: + +```bash +bacalhau job stop +``` + + +## glossary + +# Glossary + +## Compute Node + +A Compute Node in the Bacalhau platform is responsible for executing jobs and producing results. These nodes are part of a private network that allows workload distribution and communication between computers. Compute Nodes handle various types of jobs based on their capabilities and resources. They work in tandem with Requester Nodes, which manage user requests, discover and rank Compute Nodes and monitor job lifecycles. + +## CLI (Command Line Interface) + +A CLI (Command Line Interface) in the Bacalhau platform is a tool that allows users to interact with Bacalhau through text-based commands entered into a terminal or command prompt. The CLI provides a set of commands for managing and executing various tasks on the platform, including submitting jobs, monitoring job status, managing nodes and configuring the environment. + +## Data Source + +A Data Source in Bacalhau refers to the origin of the data used in jobs. This can include various types of storage such as IPFS, S3, local files or URLs. Data sources are specified in the job configuration and are essential for providing the necessary input data for job execution. + +## Docker + +Docker in Bacalhau refers to the use of Docker containers to package and run applications. Docker provides a standardized unit of software, enabling users to create and manage containers efficiently. Bacalhau supports running Docker workloads, allowing users to utilize containerized applications seamlessly on the platform. + +## IPFS + +The InterPlanetary File System (IPFS) is a protocol and peer-to-peer network for storing and sharing data in a distributed file system. In Bacalhau, IPFS is used as a data source and a way to distribute job inputs and outputs, leveraging its decentralized nature for efficient data management. + +## Job + +A Job in the Bacalhau platform is a unit of work that a user submits for execution. Jobs can be simple tasks or complex workflows involving multiple steps. They are defined by specifications that include the job type, resources required and input/output data. Jobs are managed by Requester Nodes, which ensure they are distributed to appropriate Compute Nodes for execution. + +## Job Results + +Job Results are the output generated after a job has been executed on a Compute Node. These results can include processed data, logs and any other relevant output files. Results are often stored in specified locations such as IPFS or S3, allowing users to retrieve and utilize them after job completion. + +## Node + +A Node in the Bacalhau is a fundamental component of the network, responsible for executing and managing jobs. A Node is the Bacalhau entity installed Nodes can be classified into different types based on their roles, such as Compute Nodes and Requester Nodes. Each node operates as part of a decentralized network, allowing distributed processing and resource management. + +## Node Management + +Node Management in Bacalhau involves configuring and maintaining the nodes within the network, including both Compute Nodes and Requester Nodes. This includes tasks like onboarding new nodes, managing node resources, setting access controls and ensuring nodes meet operational standards for job execution. + +## Network + +In the context of the Bacalhau, a Network refers to the interconnected system of nodes that collaborate to execute jobs, manage data and maintain communication. This network is decentralized, meaning it does not rely on a central authority, which enhances its robustness, scalability and efficiency. + +## Network Specification + +The Network Specification in Bacalhau defines the network requirements and settings for job execution. This includes configurations for network access, data transfer protocols and connectivity between nodes. Proper network specification ensures that jobs can communicate effectively and access necessary resources. + +## Workload Onboarding + +Workload Onboarding in Bacalhau is the process of preparing and integrating different types of workloads for execution on the platform. This involves setting up environments for various programming languages, configuring containers and ensuring workloads are optimized for execution across the distributed network of Compute Nodes. + +## WebAssembly (WASM) + +WebAssembly (WASM) in Bacalhau is a binary instruction format for a stack-based virtual machine. WASM is designed for safe and efficient execution, making it a suitable target for compilation from high-level languages. Bacalhau supports running WASM workloads, enabling efficient execution of lightweight and portable code. + +## Requestor Node + +A Requester Node in the Bacalhau platform is responsible for handling user requests, discovering and ranking Compute Nodes, forwarding jobs to these nodes and monitoring the lifecycle of the jobs. Requester Nodes play a crucial role in managing the flow of tasks and ensuring they are executed efficiently by the appropriate Compute Nodes in the network. + +## S3 + +Amazon Simple Storage Service (S3) is a scalable object storage service. Bacalhau supports S3 as a data source, allowing users to store and retrieve input and output data for jobs. S3's integration with Bacalhau provides robust and reliable storage options for large-scale data processing tasks. + + +## gpu + +# GPU Workloads + +## Overview + +Bacalhau supports running jobs on GPUs out of the box. This guide covers how to set up and use GPUs with Bacalhau. + +## Supported GPU Types + +Bacalhau currently supports: + +- NVIDIA GPUs +- AMD GPUs +- Intel GPUs + +These are only available with the Docker executor. + +## Prerequisites + +### Basic Requirements + +1. [Docker](https://get.docker.com/) installed +2. Appropriate GPU drivers for your hardware + +### GPU-Specific Setup + +#### NVIDIA GPUs + +1. Install [NVIDIA GPU Drivers](https://docs.nvidia.com/datacenter/tesla/tesla-installation-notes/index.html) +2. Install [NVIDIA Container Toolkit (nvidia-docker2)](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html) +3. Verify with `nvidia-smi` command + +#### AMD GPUs + +1. Install [AMD GPU drivers](https://www.amd.com/en/support/download/drivers.html) +2. Set up Docker for ROCm following [this guide](https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/docker.html) +3. Verify with `rocm-smi` command + +#### Intel GPUs + +1. Install [Intel GPU drivers](https://www.intel.com/content/www/us/en/download-center/home.html) +2. Set up Docker for Intel GPUs following [this guide](https://github.com/Intel-Media-SDK/MediaSDK/wiki/Running-on-GPU-under-docker) +3. Verify with `xpu-smi` command + +## Running GPU Jobs + +### Command Line + +Use the `--gpu` flag to specify the number of GPUs your job requires: + +```bash +bacalhau docker run --gpu=1 nvidia/cuda:11.0.3-base-ubuntu20.04 nvidia-smi +``` + +### Using YAML + +You can also submit GPU jobs using YAML configuration: + +```yaml +Name: gpu-test-job +Type: batch +Count: 1 +Tasks: + - Engine: + Type: docker + Params: + Image: 'nvidia/cuda:11.6.2-base-ubuntu20.04' + Entrypoint: + - /bin/bash + Parameters: + - -c + - nvidia-smi && echo 'GPU is working!' + Name: TestGPU + ResourcesConfig: + CPU: '1' + Memory: '1GB' + Disk: '10GB' + GPU: '1' +``` + +## Important Notes + +- Your container must include the appropriate CUDA runtime and be compatible with the CUDA version on the node +- GPU access can be controlled using resource limits +- The Bacalhau network must have executor nodes with GPUs exposed + + +## input-source + +[Content truncated - see full docs] + + +## inspecting-jobs + +[Content truncated - see full docs] + + +## ipfs + +[Content truncated - see full docs] + + +## ipfs + +[Content truncated - see full docs] + + +## label + +[Content truncated - see full docs] + + +## labels-and-constraints + +[Content truncated - see full docs] + + +## listing-jobs + +[Content truncated - see full docs] + + +## local + +[Content truncated - see full docs] + + +## local + +[Content truncated - see full docs] + + +## meta + +[Content truncated - see full docs] + + +## mounting-input-data + +[Content truncated - see full docs] + + +## network + +[Content truncated - see full docs] + + +## networking + +[Content truncated - see full docs] + + +## node-management + +[Content truncated - see full docs] + + +## node-tls + +[Content truncated - see full docs] + + +## partitioning + +[Content truncated - see full docs] + + +## queueing + +[Content truncated - see full docs] + + +## resource-requirements + +[Content truncated - see full docs] + + +## resources + +[Content truncated - see full docs] + + +## result-path + +# ResultPath + +A `ResultPath` denotes a specific location within a `Task` that contains meaningful output or results. By specifying a `ResultPath`, you can pinpoint which files or directories are essential and should be retained or published after the task's execution. + +## `ResultPath` Parameters: + +1. **Name**: A descriptive label or identifier for the result, allowing for easier referencing and understanding of the output's nature or significance. +2. **Path**: Specifies the exact location, either a file or a directory, within the task's environment where the result or output is stored. This ensures that after the task completes, the critical data at this path can be accessed, retained, or published as necessary. + + +## retrieving-logs + +[Content truncated - see full docs] + + +## running-locally + +[Content truncated - see full docs] + + +## s3 + +[Content truncated - see full docs] + + +## s3 + +[Content truncated - see full docs] + + +## s3-partitioning + +[Content truncated - see full docs] + + +## s3managed + +[Content truncated - see full docs] + + +## selection-policy + +[Content truncated - see full docs] + + +## specconfig + +[Content truncated - see full docs] + + +## state + +[Content truncated - see full docs] + + +## stopping-a-job + +[Content truncated - see full docs] + + +## submitting-jobs + +[Content truncated - see full docs] + + +## task + +[Content truncated - see full docs] + + +## timeouts + +[Content truncated - see full docs] + + +## timeouts + +# Timeouts + +The `Timeouts` object provides a mechanism to impose timing constraints on specific task operations, particularly execution. By setting these timeouts, users can ensure tasks don't run indefinitely and align them with intended durations. + +## `Timeouts` Parameters: + +* **ExecutionTimeout** `(int: )`: Defines the maximum duration (in seconds) that a task is permitted to run. A value of zero indicates that there's no set timeout. This could be particularly useful for tasks that function as daemons and are designed to run indefinitely. + +Utilizing the `Timeouts` judiciously helps in managing resource utilization and ensures tasks adhere to expected timelines, thereby enhancing the efficiency and predictability of job executions. + + +## troubleshooting + +[Content truncated - see full docs] + + +## type + +[Content truncated - see full docs] + + +## url + +[Content truncated - see full docs] + + +## wasm + +[Content truncated - see full docs] + + +## wasm + +[Content truncated - see full docs] + + +## webui + +[Content truncated - see full docs] + + +## working-with-nodes + +[Content truncated - see full docs] + diff --git a/static/robots.txt b/static/robots.txt new file mode 100644 index 0000000..50350dd --- /dev/null +++ b/static/robots.txt @@ -0,0 +1,11 @@ +# Bacalhau Documentation +# https://docs.bacalhau.org + +User-agent: * +Allow: / + +# LLM-friendly content for AI systems +# See https://llmstxt.org/ for specification +LLMs-Txt: https://docs.bacalhau.org/llms.txt + +Sitemap: https://docs.bacalhau.org/sitemap.xml