diff --git a/.gitignore b/.gitignore
index 7cf569d6ff..86bf942491 100644
--- a/.gitignore
+++ b/.gitignore
@@ -29,3 +29,19 @@ dist/
# Local Netlify folder
.netlify
src/emulator/release/
+
+# ======================================================================
+# vscode
+# ======================================================================
+# vscode configuration
+.vscode/
+
+# https://code.visualstudio.com/docs/languages/jsconfig
+jsconfig.json
+
+# ======================================================================
+# node js
+# ======================================================================
+# the exact tree installed in the node_modules folder
+package-lock.json
+
diff --git a/.prettierignore b/.prettierignore
new file mode 100644
index 0000000000..0e796c3f18
--- /dev/null
+++ b/.prettierignore
@@ -0,0 +1,3 @@
+node_modules
+dist
+build
\ No newline at end of file
diff --git a/.prettierrc b/.prettierrc
new file mode 100644
index 0000000000..74612f8818
--- /dev/null
+++ b/.prettierrc
@@ -0,0 +1,7 @@
+{
+ "singleQuote": true,
+ "semi": false,
+ "tabWidth": 2,
+ "trailingComma": "all",
+ "printWidth": 100
+}
diff --git a/README.md b/README.md
index 63f7f71adc..c0e27510ff 100644
--- a/README.md
+++ b/README.md
@@ -47,7 +47,7 @@ cd puter
npm install
npm start
```
-✨ This should launch Puter at
+**→** This should launch Puter at
http://puter.localhost:4100 (or the next available port).
@@ -62,7 +62,7 @@ troubleshooting steps.
```bash
mkdir puter && cd puter && mkdir -p puter/config puter/data && sudo chown -R 1000:1000 puter && docker run --rm -p 4100:4100 -v `pwd`/puter/config:/etc/puter -v `pwd`/puter/data:/var/puter ghcr.io/heyputer/puter
```
-✨ This should launch Puter at
+**→** This should launch Puter at
http://puter.localhost:4100 (or the next available port).
http://puter.localhost:4100 (or the next available port).
http://puter.localhost:4100 (or the next available port).
El Sistema Operativo de Internet! Gratis, de Código abierto, y Autohospedable.
-
-
« DEMO EN VIVO » Puter.com
·
- SDK
+ App Store
+ ·
+ Developers
+ ·
+ CLI
·
Discord
·
Reddit
·
- X (Twitter)
+ X (Twitter)
O Sistema Operacional da Internet! Gratuito, de Código Aberto e Auto-Hospedável.
-
-
« DEMONSTRAÇÃO AO VIVO » Puter.com
·
- SDK
+ App Store
+ ·
+ Developers
+ ·
+ CLI
·
Discord
·
Reddit
·
- X (Twitter)
+ X (Twitter)
` (e.g., `gpt-4o`)
+ - `/` (e.g., `openai/gpt-4o`)
+ - `:/` (e.g., `azure:openai/gpt-4o`)
+-
+- All models have a vender name. But not all models have a supplier name.
+- When the supplier name is specified, the validation of the model name is done by the supplier instead of puter backend.
+- We should only initialize an AI service if the corresponding config is given.
+
+- Backend will pick a model when multiple candidates match the request. For instance, if a request specifies `gpt-4o`, the system will pick the most affordable model that matches `gpt-4o` from all available models.
+- Backend will return an error on invalid model names and invalid formats.
+- Available models are defined in this [list](https://puter.com/puterai/chat/models).
diff --git a/src/backend/doc/modules/filesystem/API_SPEC.md b/src/backend/doc/modules/filesystem/API_SPEC.md
new file mode 100644
index 0000000000..8ca72cead4
--- /dev/null
+++ b/src/backend/doc/modules/filesystem/API_SPEC.md
@@ -0,0 +1,69 @@
+# Filesystem API
+
+Filesystem endpoints allow operations on files and directories in the Puter filesystem.
+
+## POST `/mkdir` (auth required)
+
+### Description
+
+Creates a new directory in the filesystem. Currently support 2 formats:
+
+- Full path: `{"path": "/foo/bar", args ...}` — this API is used by apitest (`./tools/api-tester/apitest.js`) and aligns more closely with the POSIX spec (https://linux.die.net/man/3/mkdir)
+- Parent + path: `{"parent": "/foo", "path": "bar", args ...}` — this API is used by `puter-js` via `puter.fs.mkdir`
+
+A future work would be use a unified format for all filesystem operations.
+
+### Parameters
+
+- **path** _- required_
+ - **accepts:** `string`
+ - **description:** The path where the directory should be created
+ - **notes:** Cannot be empty, null, or undefined
+
+- **parent** _- optional_
+ - **accepts:** `string | UUID`
+ - **description:** The parent directory path or UUID
+ - **notes:** If not provided, path is treated as full path
+
+- **overwrite** _- optional_
+ - **accepts:** `boolean`
+ - **default:** `false`
+ - **description:** Whether to overwrite existing files/directories
+
+- **dedupe_name** _- optional_
+ - **accepts:** `boolean`
+ - **default:** `false`
+ - **description:** Whether to automatically rename if name exists
+
+- **create_missing_parents** _- optional_
+ - **accepts:** `boolean`
+ - **default:** `false`
+ - **description:** Whether to create parent directories if they don't exist
+ - **aliases:** `create_missing_ancestors`
+
+- **shortcut_to** _- optional_
+ - **accepts:** `string | UUID`
+ - **description:** Creates a shortcut/symlink to the specified target
+
+### Example
+
+```json
+{
+ "path": "/user/Desktop/new-directory"
+}
+```
+
+```json
+{
+ "parent": "/user",
+ "path": "Desktop/new-directory"
+}
+```
+
+### Response
+
+Returns the created directory's metadata including name, path, uid, and any parent directories created.
+
+## Other Filesystem Endpoints
+
+[Additional endpoints would be documented here...]
\ No newline at end of file
diff --git a/src/backend/doc/modules/puterai/api-specification.md b/src/backend/doc/modules/puterai/api-specification.md
new file mode 100644
index 0000000000..2fa7350c68
--- /dev/null
+++ b/src/backend/doc/modules/puterai/api-specification.md
@@ -0,0 +1,689 @@
+# Puter AI API Specification
+
+This document describes the API interfaces for Puter's AI services, including chat completions, image generation, and text-to-speech capabilities.
+
+## Table of Contents
+
+1. [Chat Completions](#chat-completions)
+2. [Image Generation](#image-generation)
+3. [Text-to-Speech](#text-to-speech)
+4. [Model Management](#model-management)
+5. [Common Patterns](#common-patterns)
+
+---
+
+## Chat Completions
+
+### Overview
+
+Chat completions allow you to send messages to AI language models and receive intelligent responses. This interface supports multiple providers including OpenAI, Claude, Gemini, and others.
+
+The `puter-chat-completion` interface provides a standardized way to interact with various Large Language Model (LLM) providers through Puter's driver system. This interface abstracts away provider-specific implementations and provides a consistent API for chat completions, model listing, and model information retrieval.
+
+### Interface: `puter-chat-completion`
+
+#### Method: `complete`
+
+**Summary**: Send a conversation to an AI model and receive a completion response.
+
+**Description**: Get completions for a chat log with support for multiple input formats including text prompts, conversation arrays, and vision capabilities.
+
+**Parameters**:
+- `messages` (type: `json`, required): Array of chat messages or content array for vision
+ - Format: Array of message objects with `role` and `content` properties
+ - For vision: Content array with text and image objects
+ - Example: `[{ role: "user", content: "Hello!" }, { role: "assistant", content: "Hi there!" }]`
+ - Vision Example: `[{ content: ["Describe this image", { image_url: { url: "data:image/..." } }] }]`
+- `model` (type: `string`, optional): Specific model to use for completion
+- `temperature` (type: `number`, optional): Controls randomness (0.0 to 2.0)
+- `max_tokens` (type: `number`, optional): Maximum number of tokens to generate
+- `stream` (type: `boolean`, optional): Whether to stream the response
+- `tools` (type: `json`, optional): Function calling tools to make available
+- `vision` (type: `boolean`, optional): Whether to enable vision capabilities (auto-detected)
+- `response` (type: `json`, optional): Response format specification
+- `driver` (type: `string`, optional): Override automatic driver selection
+
+**Result**: JSON response with completion data
+
+#### puter.js Usage
+
+The `puter.ai.chat()` method supports multiple function signatures for different use cases:
+
+**Basic Text Chat**:
+```javascript
+// Simple text prompt
+const response = await puter.ai.chat("Hello, how are you?");
+
+// With specific model and parameters
+const response = await puter.ai.chat("Explain quantum computing", {
+ model: "gpt-4o",
+ temperature: 0.8,
+ max_tokens: 1000
+});
+```
+
+**Conversation History**:
+```javascript
+// With conversation array
+const response = await puter.ai.chat([
+ { role: "user", content: "What is AI?" },
+ { role: "assistant", content: "AI stands for Artificial Intelligence..." },
+ { role: "user", content: "Can you give me examples?" }
+]);
+
+// Simple message array (auto-converted to proper format)
+const response = await puter.ai.chat(["hi"]);
+const response = await puter.ai.chat(["Hello", "How are you?"]);
+```
+
+**Vision Capabilities**:
+```javascript
+// Single image with prompt
+const response = await puter.ai.chat("Describe this image", imageFile);
+
+// Single image with prompt and test mode
+const response = await puter.ai.chat("Describe this image", imageFile, true);
+
+// Multiple images with prompt
+const response = await puter.ai.chat("Compare these images", [image1, image2, image3]);
+
+// Image URLs
+const response = await puter.ai.chat("Analyze this image", "https://example.com/image.jpg");
+```
+
+**Advanced Parameters**:
+```javascript
+// Function calling with tools
+const response = await puter.ai.chat("What's the weather like?", {
+ tools: [
+ {
+ type: "function",
+ function: {
+ name: "get_weather",
+ description: "Get current weather for a location",
+ parameters: {
+ type: "object",
+ properties: {
+ location: { type: "string" }
+ },
+ required: ["location"]
+ }
+ }
+ }
+ ]
+});
+
+// Streaming response
+const response = await puter.ai.chat("Write a story", {
+ stream: true
+});
+
+// Override driver selection
+const response = await puter.ai.chat("Hello", {
+ driver: "claude"
+});
+```
+
+**Test Mode**:
+```javascript
+// Enable test mode (bypasses actual API calls)
+const response = await puter.ai.chat("Hello", true);
+const response = await puter.ai.chat("Hello", imageFile, true);
+const response = await puter.ai.chat("Hello", { testMode: true });
+```
+
+#### Function Signature Overloads
+
+The `chat` method automatically detects the input format and processes accordingly:
+
+1. **`chat(prompt: string)`** → Basic text completion
+2. **`chat(prompt: string, testMode: boolean)`** → Text with test mode
+3. **`chat(prompt: string, image: File|string)`** → Vision with single image
+4. **`chat(prompt: string, image: File|string, testMode: boolean)`** → Vision with test mode
+5. **`chat(prompt: string, images: File[]|string[])`** → Vision with multiple images
+6. **`chat(messages: Message[])`** → Conversation array with proper format
+7. **`chat(messages: string[])`** → Simple string array (auto-converted to proper format)
+8. **`chat(params: object)`** → Full parameter object
+9. **`chat(prompt: string, params: object)`** → Text with parameters
+
+#### Vision Processing
+
+When images are provided, the method automatically:
+- Converts File objects to data URIs
+- Sets `vision: true` flag
+- Structures content as arrays with text and image objects
+- Supports both single and multiple images
+
+**Image Format**:
+```javascript
+{
+ vision: true,
+ messages: [{
+ content: [
+ "Describe this image",
+ {
+ image_url: {
+ url: "data:image/jpeg;base64,..."
+ }
+ }
+ ]
+ }]
+}
+```
+
+#### Model Mapping and Driver Selection
+
+The system automatically maps models to appropriate drivers:
+
+**OpenAI Models** (driver: `openai-completion`):
+- `gpt-*` models (gpt-4o, gpt-3.5-turbo, etc.)
+- `openai/*` prefix is automatically removed
+
+**Claude Models** (driver: `claude`):
+- `claude-*` models
+- `anthropic/*` prefix is automatically removed
+- Model aliases: `claude` → `claude-3-7-sonnet-latest`
+
+**Gemini Models** (driver: `gemini`):
+- `gemini-1.5-flash`, `gemini-2.0-flash`
+
+**Groq Models** (driver: `groq`):
+- `mistral` → `mistral-large-latest`
+- `groq` → `llama3-8b-8192`
+- Various Llama models
+
+**OpenRouter Models** (driver: `openrouter`):
+- `openrouter:*` prefix
+- Meta Llama, Google, DeepSeek, x-AI models automatically prefixed
+
+**Special Models**:
+- `o1-mini` → `openrouter:openai/o1-mini`
+- `deepseek` → `deepseek-chat`
+
+#### Backend HTTP API
+
+**Endpoint**: `POST /drivers/call`
+
+**Request**:
+```json
+{
+ "interface": "puter-chat-completion",
+ "driver": "openai-completion",
+ "method": "complete",
+ "args": {
+ "messages": [
+ {
+ "role": "user",
+ "content": "Hello, how are you?"
+ }
+ ],
+ "model": "gpt-4o",
+ "temperature": 0.7,
+ "max_tokens": 1000,
+ "stream": false,
+ "vision": false
+ }
+}
+```
+
+**Response**:
+
+*Success Response*:
+```json
+{
+ "success": true,
+ "result": {
+ "message": {
+ "role": "assistant",
+ "content": "Hello! I'm doing well, thank you for asking. How can I help you today?"
+ },
+ "usage": {
+ "prompt_tokens": 10,
+ "completion_tokens": 25,
+ "total_tokens": 35
+ }
+ }
+}
+```
+
+*Error Response*:
+```json
+{
+ "success": false,
+ "error": {
+ "code": "rate_limit_exceeded",
+ "message": "Rate limit exceeded. Please try again later."
+ }
+}
+```
+
+**HTTP Status**: All responses return HTTP 200, even for errors. Check the `success` field in the response body to determine if the operation succeeded.
+
+#### Response Transformation
+
+The `puter.ai.chat()` method automatically transforms responses to provide convenient access to content:
+
+```javascript
+const response = await puter.ai.chat("Hello");
+
+// Direct content access
+console.log(response.toString()); // Returns response.message.content
+console.log(response.valueOf()); // Returns response.message.content
+
+// Standard response structure
+console.log(response.message.content); // The actual response text
+console.log(response.message.role); // Usually "assistant"
+console.log(response.usage); // Token usage information
+```
+
+#### Test Mode
+
+Test mode allows you to bypass actual API calls for development and testing:
+
+**Enabling Test Mode**:
+```javascript
+// Method 1: Boolean parameter
+const response = await puter.ai.chat("Hello", true);
+
+// Method 2: Object parameter
+const response = await puter.ai.chat("Hello", { testMode: true });
+
+// Method 3: With vision
+const response = await puter.ai.chat("Describe this", imageFile, true);
+```
+
+**Test Mode Behavior**:
+- Bypasses actual API calls to external providers
+- Returns mock responses for development/testing
+- Does not count against usage limits
+- Useful for integration testing and development
+- Automatically sets `test_mode: true` in backend requests
+
+#### Function Signature Detection
+
+The `puter.ai.chat()` method automatically detects the input format and processes parameters accordingly:
+
+**Parameter Detection Logic**:
+1. **String + Boolean**: `chat(prompt, true)` → Text with test mode
+2. **String + File/String**: `chat(prompt, image)` → Vision with single image
+3. **String + File/String + Boolean**: `chat(prompt, image, true)` → Vision with test mode
+4. **String + Array**: `chat(prompt, [image1, image2])` → Vision with multiple images
+5. **Array**: `chat([messages])` → Conversation array or simple strings
+6. **Object**: `chat({ messages, model, ... })` → Full parameter object
+
+**Simple Message Conversion**:
+- `chat(["hi"])` → Automatically converted to `[{ content: "hi" }]`
+- `chat(["Hello", "How are you?"])` → Converted to `[{ content: "Hello" }, { content: "How are you?" }]`
+
+**Parameter Merging**:
+- User parameters in object form are merged with detected parameters
+- Vision detection automatically sets `vision: true` when images are present
+- Test mode can be specified in multiple ways and is automatically detected
+- Model mapping and driver selection happen automatically
+
+**Example of Complex Detection**:
+```javascript
+// This automatically detects:
+// - Vision mode (due to image parameter)
+// - Test mode (due to boolean parameter)
+// - Maps to appropriate driver based on model
+const response = await puter.ai.chat(
+ "Describe this image",
+ imageFile,
+ true,
+ { model: "claude-3-opus" }
+);
+```
+
+---
+
+## Image Generation
+
+### Overview
+
+Generate images from text descriptions using AI models like DALL-E. Supports various image qualities and aspect ratios.
+
+### Interface: `puter-image-generation`
+
+#### Method: `generate`
+
+**Summary**: Create an image from a text prompt using AI image generation models.
+
+**Description**: Generate an image from a prompt.
+
+**Parameters**:
+- `prompt` (type: `string`, required): Text description of the desired image
+- `quality` (type: `string`, optional): Image quality setting
+
+**Result**: URL of the generated image.
+
+**Result Choices**:
+- `image`: Stream with content_type: 'image'
+- `url`: String URL with content_type: 'image'
+
+#### puter.js Usage
+
+```javascript
+// Generate image from text
+const image = await puter.ai.txt2img("A beautiful sunset over mountains");
+
+// Access the image
+console.log(image.src); // Image URL
+```
+
+#### Backend HTTP API
+
+**Endpoint**: `POST /drivers/call`
+
+**Request**:
+```json
+{
+ "interface": "puter-image-generation",
+ "method": "generate",
+ "args": {
+ "prompt": "A beautiful sunset over mountains"
+ }
+}
+```
+
+**Response**: Image blob with `Content-Type: image/*`
+
+---
+
+## Text-to-Speech
+
+### Overview
+
+Convert text into natural-sounding speech using various TTS engines and voices.
+
+### Interface: `puter-tts`
+
+#### Method: `synthesize`
+
+**Summary**: Convert text to speech using specified voice and engine.
+
+**Description**: Synthesize speech from text.
+
+**Parameters**:
+- `text` (type: `string`, required): Text to convert to speech
+- `voice` (type: `string`, required): Voice identifier
+- `language` (type: `string`, optional): Language code
+- `ssml` (type: `flag`, optional): Enable SSML markup
+- `engine` (type: `string`, optional): TTS engine to use
+
+**Result Choices**:
+- `audio`: Stream with content_type: 'audio'
+
+#### puter.js Usage
+
+```javascript
+// Convert text to speech
+const audio = await puter.ai.txt2speech("Hello world", "en-US-Standard-A");
+
+// Play the audio
+audio.play();
+```
+
+#### Backend HTTP API
+
+**Endpoint**: `POST /drivers/call`
+
+**Request**:
+```json
+{
+ "interface": "puter-tts",
+ "method": "synthesize",
+ "args": {
+ "text": "Hello world",
+ "voice": "en-US-Standard-A"
+ }
+}
+```
+
+**Response**: Audio stream with `Content-Type: audio/*`
+
+---
+
+## Model Management
+
+### Overview
+
+Discover and manage available AI models across different providers and capabilities.
+
+### Interface: `puter-chat-completion`
+
+#### Method: `list`
+
+**Summary**: Get a list of available AI models.
+
+**Description**: List supported models.
+
+**Parameters**: None
+
+**Result**: Array of model identifiers (strings)
+
+#### puter.js Usage
+
+**Model Management**:
+```javascript
+// List available models
+const models = await puter.ai.listModels();
+
+// List models by provider
+const openaiModels = await puter.ai.listModels("openai");
+const claudeModels = await puter.ai.listModels("claude");
+
+// Get model providers
+const providers = await puter.ai.listModelProviders();
+```
+
+#### Backend HTTP API
+
+**Endpoint**: `POST /drivers/call`
+
+**Request**:
+```json
+{
+ "interface": "puter-chat-completion",
+ "method": "list"
+}
+```
+
+**Response**:
+```json
+{
+ "success": true,
+ "result": [
+ "gpt-4o",
+ "gpt-3.5-turbo",
+ "claude-3-opus",
+ "claude-3-sonnet"
+ ]
+}
+```
+
+#### Method: `models`
+
+**Summary**: Get detailed information about available models including pricing and capabilities.
+
+**Description**: List supported models and their details.
+
+**Parameters**: None
+
+**Result**: JSON array with detailed model information including:
+- `id`: Model identifier
+- `provider`: Service provider name
+- `aliases`: Alternative names for the model
+- `context_length`: Maximum context length
+- `capabilities`: Available features
+- `pricing`: Cost information
+
+#### puter.js Usage
+
+```javascript
+// Get detailed model information
+const modelDetails = await puter.ai.listModels();
+```
+
+#### Backend HTTP API
+
+**Endpoint**: `POST /drivers/call`
+
+**Request**:
+```json
+{
+ "interface": "puter-chat-completion",
+ "method": "models"
+}
+```
+
+**Response**:
+```json
+{
+ "success": true,
+ "result": [
+ {
+ "id": "gpt-4o",
+ "provider": "openai",
+ "aliases": ["gpt4o"],
+ "context_length": 128000,
+ "capabilities": ["chat", "vision"],
+ "pricing": {
+ "input": 0.000005,
+ "output": 0.000015
+ }
+ }
+ ]
+}
+```
+
+---
+
+## Common Patterns
+
+### AI Model Naming and Mapping
+
+**Scope**: This rule applies to `model` field in AI-related API endpoints. Such as:
+- `puter-chat-completion/complete`
+
+**Request Format**:
+- interface (type: string, required) (`"puter-chat-completion"`)
+- method (type: string, required) (`"complete"`)
+- driver (type: string, optional) (auto-detected based on model)
+- args
+ - model (type: string, required) (e.g., `"gpt-4o"`, `"claude-3-opus"`, `"gemini-1.5-flash"`)
+ - messages (type: array, required)
+ - (e.g., `["Hello! How are you?"]`)
+ - (e.g., `[{ content: "Hello! How are you?" }]`)
+
+**Model Format Support**: Multiple formats are supported:
+- `` (e.g., `gpt-4o`, `claude-3-opus`)
+- `/` (e.g., `openai/gpt-4o`, `anthropic/claude-3-opus`)
+- `:/` (e.g., `azure:openai/gpt-4o`)
+
+**Automatic Model Mapping**: The system automatically maps models to appropriate drivers:
+
+**OpenAI Models** → `openai-completion` driver:
+- `gpt-*` models (gpt-4o, gpt-3.5-turbo, etc.)
+- `openai/*` prefix is automatically removed
+- Examples: `gpt-4o`, `openai/gpt-4o` → both use OpenAI driver
+
+**Claude Models** → `claude` driver:
+- `claude-*` models
+- `anthropic/*` prefix is automatically removed
+- Model aliases: `claude` → `claude-3-7-sonnet-latest`
+
+**Gemini Models** → `gemini` driver:
+- `gemini-1.5-flash`, `gemini-2.0-flash`
+
+**Groq Models** → `groq` driver:
+- `mistral` → `mistral-large-latest`
+- `groq` → `llama3-8b-8192`
+- Various Llama models
+
+**OpenRouter Models** → `openrouter` driver:
+- `openrouter:*` prefix
+- Meta Llama, Google, DeepSeek, x-AI models automatically prefixed
+- Examples: `meta-llama/Llama-3.1-8B-Instruct-Turbo` → `openrouter:meta-llama/Llama-3.1-8B-Instruct-Turbo`
+
+**Special Model Mappings**:
+- `o1-mini` → `openrouter:openai/o1-mini`
+- `deepseek` → `deepseek-chat`
+- `mistral` → `mistral-large-latest`
+
+### Driver Implementations
+
+The following services implement the `puter-chat-completion` interface:
+
+- **`ai-chat`**: Main AI chat service with fallback and provider selection
+- **`openai-completion`**: OpenAI API integration (GPT models)
+- **`claude`**: Anthropic Claude API integration
+- **`gemini`**: Google Gemini API integration
+- **`groq`**: Groq API integration (Llama models)
+- **`deepseek`**: DeepSeek API integration
+- **`xai`**: xAI Grok integration
+- **`openrouter`**: OpenRouter API integration (multiple providers)
+
+### Error Handling
+
+All API responses use HTTP 200 status. Check the `success` field in the response body:
+
+**Error Response Format**:
+```json
+{
+ "success": false,
+ "error": {
+ "code": "error_code",
+ "message": "Human-readable description"
+ }
+}
+```
+
+**Common Error Codes**:
+- `permission_denied`: User lacks permission to use the driver
+- `rate_limit_exceeded`: API rate limit exceeded
+- `usage_limit_exceeded`: User usage limit exceeded
+- `invalid_model`: Specified model is not available
+- `invalid_parameters`: Request parameters are invalid
+- `provider_error`: Error from the underlying AI provider
+- `moderation_error`: Content flagged by moderation systems
+
+### Streaming Responses
+
+When `stream: true` is specified, the response is returned as a stream with the following format:
+
+**Response Headers**:
+- `Content-Type: application/ndjson`
+- `Transfer-Encoding: chunked` (if applicable)
+
+**Response Body**: Newline-delimited JSON stream
+```json
+{"role": "assistant", "content": "Hello"}
+{"role": "assistant", "content": "! How"}
+{"role": "assistant", "content": " can I help"}
+{"role": "assistant", "content": " you today?"}
+```
+
+### Usage Tracking
+
+All chat completions are tracked for:
+- Token usage (prompt, completion, total)
+- Cost calculation based on provider pricing
+- Rate limiting and quota management
+- User analytics and billing
+
+### Testing Mode
+
+Set `test_mode: true` in the request to enable testing mode, which:
+- Bypasses actual API calls to external providers
+- Returns mock responses for development/testing
+- Does not count against usage limits
+- Useful for integration testing and development
+
+### Security and Permissions
+
+- **Authentication**: All requests require valid authentication
+- **Permissions**: Users must have permission to use specific driver interfaces
+- **Moderation**: Content is automatically checked against moderation policies
+- **Rate Limiting**: Requests are rate-limited per user and per provider
+- **Usage Limits**: Users are subject to usage limits based on their plan
diff --git a/src/backend/exports.js b/src/backend/exports.js
index d4a836b0cd..ad28359147 100644
--- a/src/backend/exports.js
+++ b/src/backend/exports.js
@@ -20,6 +20,7 @@ const CoreModule = require("./src/CoreModule.js");
const { Kernel } = require("./src/Kernel.js");
const DatabaseModule = require("./src/DatabaseModule.js");
const LocalDiskStorageModule = require("./src/LocalDiskStorageModule.js");
+const MemoryStorageModule = require("./src/MemoryStorageModule.js");
const SelfHostedModule = require("./src/modules/selfhosted/SelfHostedModule.js");
const { testlaunch } = require("./src/index.js");
const BaseService = require("./src/services/BaseService.js");
@@ -39,6 +40,7 @@ const { InternetModule } = require("./src/modules/internet/InternetModule.js");
const { CaptchaModule } = require("./src/modules/captcha/CaptchaModule.js");
const { EntityStoreModule } = require("./src/modules/entitystore/EntityStoreModule.js");
const { KVStoreModule } = require("./src/modules/kvstore/KVStoreModule.js");
+const { AIRouterModule } = require("./src/modules/airouter/AIRouterModule.js");
module.exports = {
helloworld: () => {
@@ -71,8 +73,10 @@ module.exports = {
WebModule,
DatabaseModule,
LocalDiskStorageModule,
+ MemoryStorageModule,
SelfHostedModule,
TestDriversModule,
+ AIRouterModule,
PuterAIModule,
BroadcastModule,
InternetModule,
diff --git a/src/backend/package.json b/src/backend/package.json
index 07d44101a2..bc0bce52f2 100644
--- a/src/backend/package.json
+++ b/src/backend/package.json
@@ -15,6 +15,7 @@
"@heyputer/kv.js": "^0.1.9",
"@heyputer/multest": "^0.0.2",
"@heyputer/putility": "^1.0.0",
+ "@heyputer/airouter": "^0.0.0",
"@mistralai/mistralai": "^1.3.4",
"@opentelemetry/api": "^1.4.1",
"@opentelemetry/auto-instrumentations-node": "^0.43.0",
diff --git a/src/backend/src/ExtensionService.js b/src/backend/src/ExtensionService.js
index 2c9cf1f9df..23409888d6 100644
--- a/src/backend/src/ExtensionService.js
+++ b/src/backend/src/ExtensionService.js
@@ -63,6 +63,7 @@ class ExtensionServiceState extends AdvancedBase {
mw,
route: path,
handler: handler,
+ ...(options.subdomain ? { subdomain: options.subdomain } : {}),
});
this.endpoints_.push(endpoint);
diff --git a/src/backend/src/MemoryStorageModule.js b/src/backend/src/MemoryStorageModule.js
new file mode 100644
index 0000000000..a8985460c4
--- /dev/null
+++ b/src/backend/src/MemoryStorageModule.js
@@ -0,0 +1,27 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see } The created directory.
+ */
+ async _create_dir (dir) {
+ console.log('CREATING DIR', dir.selector.describe());
+
+ if ( await dir.exists() ) {
+ if ( ! dir.entry.is_dir ) {
throw APIError.create('dest_is_not_a_directory');
}
- return top_parent;
+ return dir;
}
const maybe_path_selector =
- top_parent.get_selector_of_type(NodePathSelector);
+ dir.get_selector_of_type(NodePathSelector);
if ( ! maybe_path_selector ) {
throw APIError.create('dest_does_not_exist');
}
- const path = maybe_path_selector.value;
+ let path = maybe_path_selector.value;
const fs = this.context.get('services').get('filesystem');
diff --git a/src/backend/src/filesystem/hl_operations/hl_move.js b/src/backend/src/filesystem/hl_operations/hl_move.js
index 340fe089c4..ecbe659421 100644
--- a/src/backend/src/filesystem/hl_operations/hl_move.js
+++ b/src/backend/src/filesystem/hl_operations/hl_move.js
@@ -147,7 +147,7 @@ class HLMove extends HLFilesystemOperation {
if ( await dest.exists() ) {
if ( ! values.overwrite && ! values.dedupe_name ) {
throw APIError.create('item_with_same_name_exists', null, {
- entry_name: target_name,
+ entry_name: await dest.get('name'),
});
}
diff --git a/src/backend/src/filesystem/hl_operations/hl_stat.js b/src/backend/src/filesystem/hl_operations/hl_stat.js
index 809620c4b5..53419aea3b 100644
--- a/src/backend/src/filesystem/hl_operations/hl_stat.js
+++ b/src/backend/src/filesystem/hl_operations/hl_stat.js
@@ -29,7 +29,8 @@ class HLStat extends HLFilesystemOperation {
const {
subject, user,
return_subdomains,
- return_permissions,
+ return_permissions, // Deprecated: kept for backwards compatiable with `return_shares`
+ return_shares,
return_versions,
return_size,
} = this.values;
@@ -55,7 +56,9 @@ class HLStat extends HLFilesystemOperation {
if (return_size) await subject.fetchSize(user);
if (return_subdomains) await subject.fetchSubdomains(user)
- if (return_permissions) await subject.fetchShares();
+ if (return_shares || return_permissions) {
+ await subject.fetchShares();
+ }
if (return_versions) await subject.fetchVersions();
await subject.fetchIsEmpty();
diff --git a/src/backend/src/filesystem/ll_operations/ll_mkdir.js b/src/backend/src/filesystem/ll_operations/ll_mkdir.js
index a097b2109c..f92f5cba50 100644
--- a/src/backend/src/filesystem/ll_operations/ll_mkdir.js
+++ b/src/backend/src/filesystem/ll_operations/ll_mkdir.js
@@ -31,112 +31,13 @@ class LLMkdir extends LLFilesystemOperation {
}
async _run () {
- const { context } = this;
- const { parent, name } = this.values;
-
- this.checkpoint('lock requested');
- this.log.noticeme('GET FSLOCK');
- const svc_fslock = context.get('services').get('fslock');
- this.log.noticeme('REQUESTING LOCK', {
- parent: await parent.get('path'),
- name,
- });
- const lock_handle = await svc_fslock.lock_child(
- await parent.get('path'),
- name,
- MODE_WRITE,
- );
- this.log.noticeme('GOT LOCK', {
- parent: await parent.get('path'),
+ const { parent, name, immutable } = this.values;
+ return await parent.provider.mkdir({
+ context: this.context,
+ parent,
name,
+ immutable,
});
- this.checkpoint('lock acquired');
-
- try {
- return await this._locked_run();
- } finally {
- await lock_handle.unlock();
- }
- }
- async _locked_run () {
- const { _path, uuidv4 } = this.modules;
- const { context } = this;
- const { parent, name, immutable, actor } = this.values;
-
- const ts = Math.round(Date.now() / 1000);
- const uid = uuidv4();
- const resourceService = context.get('services').get('resourceService');
- const svc_fsEntry = context.get('services').get('fsEntryService');
- const svc_event = context.get('services').get('event');
- const fs = context.get('services').get('filesystem');
-
- this.field('fsentry-uid', uid);
-
- const existing = await fs.node(
- new NodeChildSelector(parent.selector, name)
- );
-
- if ( await existing.exists() ) {
- throw APIError.create('item_with_same_name_exists', null, {
- entry_name: name,
- });
- }
-
- this.checkpoint('before acl');
- const svc_acl = context.get('services').get('acl');
- if ( ! await parent.exists() ) {
- throw APIError.create('subject_does_not_exist');
- }
- if ( ! await svc_acl.check(actor, parent, 'write') ) {
- throw await svc_acl.get_safe_acl_error(actor, parent, 'write');
- }
-
- resourceService.register({
- uid,
- status: RESOURCE_STATUS_PENDING_CREATE,
- });
-
- const raw_fsentry = {
- is_dir: 1,
- uuid: uid,
- parent_uid: await parent.get('uid'),
- path: _path.join(await parent.get('path'), name),
- user_id: actor.type.user.id,
- name,
- created: ts,
- accessed: ts,
- modified: ts,
- immutable: immutable ?? false,
- ...(this.values.thumbnail ? {
- thumbnail: this.values.thumbnail,
- } : {}),
- };
-
- this.log.debug('creating fsentry', { fsentry: raw_fsentry })
-
- this.checkpoint('about to enqueue insert');
- const entryOp = await svc_fsEntry.insert(raw_fsentry);
-
- this.field('fsentry-created', false);
-
- this.checkpoint('enqueued insert');
- // Asynchronous behaviour temporarily disabled
- // (async () => {
- await entryOp.awaitDone();
- this.log.debug('finished creating fsentry', { uid })
- resourceService.free(uid);
- this.field('fsentry-created', true);
- // })();
-
- const node = await fs.node(new NodeUIDSelector(uid));
-
- svc_event.emit('fs.create.directory', {
- node,
- context: Context.get(),
- });
-
- this.checkpoint('returning node');
- return node
}
}
diff --git a/src/backend/src/filesystem/ll_operations/ll_read.js b/src/backend/src/filesystem/ll_operations/ll_read.js
index 91abf5acb3..1173cc44d6 100644
--- a/src/backend/src/filesystem/ll_operations/ll_read.js
+++ b/src/backend/src/filesystem/ll_operations/ll_read.js
@@ -18,6 +18,7 @@
*/
const APIError = require("../../api/APIError");
const { Sequence } = require("../../codex/Sequence");
+const { MemoryFSProvider } = require("../../modules/puterfs/customfs/MemoryFSProvider");
const { DB_WRITE } = require("../../services/database/consts");
const { buffer_to_stream } = require("../../util/streamutil");
@@ -115,10 +116,13 @@ class LLRead extends LLFilesystemOperation {
},
async function create_S3_read_stream (a) {
const context = a.iget('context');
- const storage = context.get('storage');
const { fsNode, version_id, offset, length, has_range } = a.values();
+ const svc_mountpoint = context.get('services').get('mountpoint');
+ const provider = await svc_mountpoint.get_provider(fsNode.selector);
+ const storage = svc_mountpoint.get_storage(provider.constructor);
+
// Empty object here is in the case of local fiesystem,
// where s3:location will return null.
// TODO: storage interface shouldn't have S3-specific properties.
@@ -133,6 +137,8 @@ class LLRead extends LLFilesystemOperation {
...(has_range ? {
range: `bytes=${offset}-${offset+length-1}`
} : {}),
+
+ memory_file: fsNode.entry,
}));
a.set('stream', stream);
@@ -144,8 +150,11 @@ class LLRead extends LLFilesystemOperation {
const { fsNode, stream, has_range } = a.values();
if ( ! has_range ) {
- const res = await svc_fileCache.maybe_store(fsNode, stream);
- if ( res.stream ) a.set('stream', res.stream);
+ // only cache for non-memoryfs providers
+ if ( ! (fsNode.provider instanceof MemoryFSProvider) ) {
+ const res = await svc_fileCache.maybe_store(fsNode, stream);
+ if ( res.stream ) a.set('stream', res.stream);
+ }
}
},
async function return_stream (a) {
diff --git a/src/backend/src/filesystem/ll_operations/ll_rmdir.js b/src/backend/src/filesystem/ll_operations/ll_rmdir.js
index 08add7c7d4..302070745a 100644
--- a/src/backend/src/filesystem/ll_operations/ll_rmdir.js
+++ b/src/backend/src/filesystem/ll_operations/ll_rmdir.js
@@ -17,6 +17,7 @@
* along with this program. If not, see } The file node, or null if the file does not exist.
+ */
+async function get_entry(uid) {
+ const svc_mountpoint = Context.get('services').get('mountpoint');
+ const uid_selector = new NodeUIDSelector(uid);
+ const provider = await svc_mountpoint.get_provider(uid_selector);
+
+ // NB: We cannot import MemoryFSProvider here because it will cause a circular dependency.
+ if ( provider.constructor.name !== 'MemoryFSProvider' ) {
+ return null;
+ }
+
+ return provider.stat({
+ selector: uid_selector,
+ });
+}
+
async function is_ancestor_of(ancestor_uid, descendant_uid){
+ const ancestor = await get_entry(ancestor_uid);
+ const descendant = await get_entry(descendant_uid);
+
+ if ( ancestor && descendant ) {
+ return descendant.path.startsWith(ancestor.path);
+ }
+
+
/** @type BaseDatabaseAccessService */
const db = services.get('database').get(DB_READ, 'filesystem');
diff --git a/src/backend/src/modules/puterai/AIChatService.js b/src/backend/src/modules/airouter/AIChatService.js
similarity index 84%
rename from src/backend/src/modules/puterai/AIChatService.js
rename to src/backend/src/modules/airouter/AIChatService.js
index 6698caeea5..75e73f5f5a 100644
--- a/src/backend/src/modules/puterai/AIChatService.js
+++ b/src/backend/src/modules/airouter/AIChatService.js
@@ -28,13 +28,12 @@ const { TypeSpec } = require("../../services/drivers/meta/Construct");
const { TypedValue } = require("../../services/drivers/meta/Runtime");
const { Context } = require("../../util/context");
const { AsModeration } = require("./lib/AsModeration");
-const FunctionCalling = require("./lib/FunctionCalling");
-const Messages = require("./lib/Messages");
-const Streaming = require("./lib/Streaming");
// Maximum number of fallback attempts when a model fails, including the first attempt
const MAX_FALLBACKS = 3 + 1; // includes first attempt
+// Imported in _construct bleow.
+let CompletionWriter, UniversalPromptNormalizer, NormalizedPromptUtil, UniversalToolsNormalizer;
/**
* AIChatService class extends BaseService to provide AI chat completion functionality.
@@ -50,6 +49,41 @@ class AIChatService extends BaseService {
cuid2: require('@paralleldrive/cuid2').createId,
}
+ async ['__on_driver.register.interfaces'] () {
+ const svc_registry = this.services.get('registry');
+ const col_interfaces = svc_registry.get('interfaces');
+
+ col_interfaces.set('puter-chat-completion', {
+ description: 'Chatbot.',
+ methods: {
+ models: {
+ description: 'List supported models and their details.',
+ result: { type: 'json' },
+ parameters: {},
+ },
+ list: {
+ description: 'List supported models',
+ result: { type: 'json' },
+ parameters: {},
+ },
+ complete: {
+ description: 'Get completions for a chat log.',
+ parameters: {
+ messages: { type: 'json' },
+ tools: { type: 'json' },
+ vision: { type: 'flag' },
+ stream: { type: 'flag' },
+ response: { type: 'json' },
+ model: { type: 'string' },
+ temperature: { type: 'number' },
+ max_tokens: { type: 'number' },
+ },
+ result: { type: 'json' },
+ }
+ }
+ });
+ }
+
/**
* Initializes the service by setting up core properties.
@@ -58,12 +92,15 @@ class AIChatService extends BaseService {
* Called during service instantiation.
* @private
*/
- _construct () {
+ async _construct () {
this.providers = [];
this.simple_model_list = [];
this.detail_model_list = [];
this.detail_model_map = {};
+
+
+ ({ CompletionWriter, UniversalPromptNormalizer, NormalizedPromptUtil, UniversalToolsNormalizer } = await import("@heyputer/airouter.js"));
}
get_model_details (model_name, context) {
@@ -397,7 +434,7 @@ class AIChatService extends BaseService {
if ( parameters.messages ) {
parameters.messages =
- Messages.normalize_messages(parameters.messages);
+ UniversalPromptNormalizer.normalize_messages(parameters.messages);
}
if ( ! test_mode && ! await this.moderate(parameters) ) {
@@ -416,16 +453,30 @@ class AIChatService extends BaseService {
}
if ( parameters.tools ) {
- FunctionCalling.normalize_tools_object(parameters.tools);
+ UniversalToolsNormalizer.normalize_tools_object(parameters.tools);
}
- if ( intended_service === this.service_name ) {
- throw new Error('Calling ai-chat directly is not yet supported');
+ const { target_service, supplier, vendor, model } = this.disentangle_model(parameters.model, intended_service);
+ // Write "target_service" back to "intended_service".
+ //
+ // TODO (xiaochen): remove the redundant "target_service".
+ intended_service = target_service;
+
+ // Hardcode the services that accept / format. This will be
+ // removed in the future.
+ //
+ // TODO (xiaochen): pass the whole details (supplier, vendor, model) to the
+ // delegate service, and let it decide how to handle it.
+ if (intended_service === 'openrouter') {
+ parameters.model = vendor + '/' + model;
+ } else {
+ parameters.model = model;
}
const svc_driver = this.services.get('driver');
let ret, error;
let service_used = intended_service;
+
let model_used = this.get_model_from_request(parameters, {
intended_service
});
@@ -452,7 +503,7 @@ class AIChatService extends BaseService {
const model_input_cost = model_details.cost.input;
const model_output_cost = model_details.cost.output;
const model_max_tokens = model_details.max_tokens;
- const text = Messages.extract_text(parameters.messages);
+ const text = NormalizedPromptUtil.extract_text(parameters.messages);
const approximate_input_cost = text.length / 4 * model_input_cost;
const usageAllowed = await svc_cost.get_funding_allowed({
available,
@@ -668,7 +719,7 @@ class AIChatService extends BaseService {
chunked: true,
}, stream);
- const chatStream = new Streaming.AIChatStream({
+ const chatStream = new CompletionWriter({
stream,
});
@@ -719,7 +770,7 @@ class AIChatService extends BaseService {
if ( parameters.response?.normalize ) {
ret.result.message =
- Messages.normalize_single_message(ret.result.message);
+ UniversalPromptNormalizer.normalize_single_message(ret.result.message);
ret.result = {
message: ret.result.message,
via_ai_chat_service: true,
@@ -731,7 +782,6 @@ class AIChatService extends BaseService {
}
}
}
-
/**
* Checks if the user has permission to use AI services and verifies usage limits
@@ -973,6 +1023,118 @@ class AIChatService extends BaseService {
return model;
}
+
+ /**
+ * Determines the target service to use for AI operations based on intended service and model specification.
+ *
+ * The method follows this priority order:
+ * 1. If "intended_service" is not "ai-chat", use it as the target service.
+ * 2. If "supplier" is specified by model (e.g., "azure" in "azure:openai/gpt-4o"), use it as the target service.
+ * 3. If "vendor" is specified by model (e.g., "openai" in "openai/gpt-4o"), use it as the target service.
+ * 4. Try to infer the service from the model name itself. (e.g., "gpt-4o" -> "openai")
+ * 5. If all else fails, throw an error.
+ *
+ * Note that the "intended_service" corresponds to the "driver" field and "interface" field of the http request.
+ *
+ * This method also inherits the "fallback to the default model" feature from the old "get_model_from_request" api. This logic may be removed in the future.
+ *
+ * @param {string} qualified_model - The model string to parse
+ * @param {string} intended_service - The originally intended service name
+ * @returns {string} returns.target_service - The target service name to use for the AI operation
+ * @returns {string} returns.supplier - The supplier (e.g., "openrouter") or null if not specified
+ * @returns {string} returns.vendor - The vendor (e.g., "openai") or null if not specified
+ * @returns {string} returns.model - The model name (e.g., "gpt-4o")
+ */
+ disentangle_model(qualified_model, intended_service) {
+ if (!qualified_model) {
+ const service = this.services.get(intended_service);
+ if (!service.get_default_model) {
+ throw new Error('could not infer model from service');
+ }
+ qualified_model = service.get_default_model();
+ if (!qualified_model) {
+ throw new Error('could not infer model from service');
+ }
+ }
+
+ const { supplier, vendor, model } = parse_qualified_model(qualified_model);
+
+ let result = {
+ target_service: intended_service,
+ supplier,
+ vendor,
+ model,
+ };
+
+ if (intended_service !== 'ai-chat') {
+ // Already have a valid target service, no need to infer.
+ return result;
+ }
+
+ let service = null;
+ if (supplier) {
+ result.target_service = supplier;
+ } else if (vendor) {
+ result.target_service = vendor;
+ } else if (model) {
+ // TODO (xiaochen): infer the service from the model name
+ result.target_service = model;
+ }
+
+ const service_map = {
+ 'openai': 'openai-completion',
+ }
+ result.target_service = service_map[result.target_service] || result.target_service;
+
+ return result;
+ }
+}
+
+/**
+ * Parses a qualified model name string into its components.
+ *
+ * 3 formats are allowed:
+ * - `` (e.g., `gpt-4o`)
+ * - `/` (e.g., `openai/gpt-4o`)
+ * - `:/` (e.g., `azure:openai/gpt-4o`)
+ *
+ * @param {string} qualified_model - The model string to parse
+ * @returns {Object} Object containing parsed components
+ * @returns {string|null} returns.supplier - The supplier (e.g., "azure") or null if not specified
+ * @returns {string|null} returns.vendor - The vendor (e.g., "openai") or null if not specified
+ * @returns {string} returns.model - The model name (e.g., "gpt-4o")
+ * @throws {Error} If the input is not a string or has an invalid format
+ */
+function parse_qualified_model(qualified_model) {
+ if (typeof qualified_model !== 'string') {
+ throw new Error('Input must be a string');
+ }
+
+ let supplier = null;
+ let vendor = null;
+ let model_name = null;
+
+ // Match patterns
+ const fullPattern = /^([^:]+):([^/]+)\/(.+)$/; // supplier:vendor/model
+ const vendorPattern = /^([^/]+)\/(.+)$/; // vendor/model
+ const simplePattern = /^[^:/]+$/; // model
+
+ if (fullPattern.test(qualified_model)) {
+ const match = qualified_model.match(fullPattern);
+ supplier = match[1];
+ vendor = match[2];
+ model_name = match[3];
+ } else if (vendorPattern.test(qualified_model)) {
+ const match = qualified_model.match(vendorPattern);
+ vendor = match[1];
+ model_name = match[2];
+ } else if (simplePattern.test(qualified_model)) {
+ model_name = qualified_model;
+ } else {
+ throw new Error(`Invalid model format: "${qualified_model}"`);
+ }
+
+ return { supplier, vendor, model: model_name };
}
module.exports = { AIChatService };
diff --git a/src/backend/src/modules/airouter/AIRouterModule.js b/src/backend/src/modules/airouter/AIRouterModule.js
new file mode 100644
index 0000000000..13c7bd541f
--- /dev/null
+++ b/src/backend/src/modules/airouter/AIRouterModule.js
@@ -0,0 +1,102 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see }
*/
async _init () {
- // Temporary solution - we'll develop this incrementally
- this.storage_ = null;
+ // key: provider class (e.g: PuterFSProvider, MemoryFSProvider)
+ // value: storage instance
+ this.storage_ = {};
}
async ['__on_boot.consolidation'] () {
@@ -87,12 +88,37 @@ class MountpointService extends BaseService {
}
async get_provider (selector) {
+ // type check
+ if ( ! (selector instanceof NodeSelector) ) {
+ throw new Error('Invalid selector type');
+ }
+
+ try_infer_attributes(selector);
+
if ( selector instanceof RootNodeSelector ) {
return this.mountpoints_['/'].provider;
}
if ( selector instanceof NodeUIDSelector ) {
- return this.mountpoints_['/'].provider;
+ for ( const [path, { provider }] of Object.entries(this.mountpoints_) ) {
+ const result = await provider.quick_check({
+ selector,
+ });
+ if ( result ) {
+ return provider;
+ }
+ }
+
+ // No provider found, but we shouldn't throw an error here
+ // because it's a valid case for a node that doesn't exist.
+ }
+
+ if ( selector instanceof NodeChildSelector ) {
+ if ( selector.path ) {
+ return this.get_provider(new NodePathSelector(selector.path));
+ } else {
+ return this.get_provider(selector.parent);
+ }
}
const probe = {};
@@ -118,15 +144,16 @@ class MountpointService extends BaseService {
}
// Temporary solution - we'll develop this incrementally
- set_storage (storage) {
- this.storage_ = storage;
+ set_storage (provider, storage) {
+ this.storage_[provider] = storage;
}
+
/**
* Gets the current storage backend instance
* @returns {Object} The storage backend instance
*/
- get_storage () {
- return this.storage_;
+ get_storage (provider) {
+ return this.storage_[provider];
}
}
diff --git a/src/backend/src/modules/puterfs/PuterFSModule.js b/src/backend/src/modules/puterfs/PuterFSModule.js
index 91b88fb52f..234565fdcb 100644
--- a/src/backend/src/modules/puterfs/PuterFSModule.js
+++ b/src/backend/src/modules/puterfs/PuterFSModule.js
@@ -40,6 +40,9 @@ class PuterFSModule extends AdvancedBase {
const DatabaseFSEntryFetcher = require("./DatabaseFSEntryFetcher");
services.registerService('fsEntryFetcher', DatabaseFSEntryFetcher);
+
+ const { MemoryFSService } = require('./customfs/MemoryFSService');
+ services.registerService('memoryfs', MemoryFSService);
}
}
diff --git a/src/backend/src/modules/puterfs/customfs/MemoryFSProvider.js b/src/backend/src/modules/puterfs/customfs/MemoryFSProvider.js
new file mode 100644
index 0000000000..3b1cb6c5e5
--- /dev/null
+++ b/src/backend/src/modules/puterfs/customfs/MemoryFSProvider.js
@@ -0,0 +1,605 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see }
+ */
+ _integrity_check() {
+ if (config.env !== 'dev') {
+ // only check in debug mode since it's expensive
+ return;
+ }
+
+ // check the 2 maps are consistent
+ if (this.entriesByPath.size !== this.entriesByUUID.size) {
+ throw new Error('Path map and UUID map have different sizes');
+ }
+
+ for (const [inner_path, uuid] of this.entriesByPath) {
+ const entry = this.entriesByUUID.get(uuid);
+
+ // entry should exist
+ if (!entry) {
+ throw new Error(`Entry ${uuid} does not exist`);
+ }
+
+ // path should match
+ if (this._inner_path(entry.path) !== inner_path) {
+ throw new Error(`Path ${inner_path} does not match entry ${uuid}`);
+ }
+
+ // uuid should match
+ if (entry.uuid !== uuid) {
+ throw new Error(`UUID ${uuid} does not match entry ${entry.uuid}`);
+ }
+
+ // parent should exist
+ if (entry.parent_uid) {
+ const parent_entry = this.entriesByUUID.get(entry.parent_uid);
+ if (!parent_entry) {
+ throw new Error(`Parent ${entry.parent_uid} does not exist`);
+ }
+ }
+
+ // parent's path should be a prefix of the entry's path
+ if (entry.parent_uid) {
+ const parent_entry = this.entriesByUUID.get(entry.parent_uid);
+ if (!entry.path.startsWith(parent_entry.path)) {
+ throw new Error(
+ `Parent ${entry.parent_uid} path ${parent_entry.path} is not a prefix of entry ${entry.path}`,
+ );
+ }
+ }
+
+ // parent should be a directory
+ if (entry.parent_uid) {
+ const parent_entry = this.entriesByUUID.get(entry.parent_uid);
+ if (!parent_entry.is_dir) {
+ throw new Error(`Parent ${entry.parent_uid} is not a directory`);
+ }
+ }
+ }
+ }
+
+ /**
+ * Check if a given node exists.
+ *
+ * @param {Object} param
+ * @param {NodeSelector} param.selector - The selector used for checking.
+ * @returns {Promise} - True if the node exists, false otherwise.
+ */
+ async quick_check({ selector }) {
+ // memoryfs relies on attributes path/uid
+ try_infer_attributes(selector);
+
+ // shortcut: has full path
+ if (selector?.path) {
+ const inner_path = this._inner_path(selector.path);
+ return this.entriesByPath.has(inner_path);
+ }
+
+ // shortcut: has uid
+ if (selector?.uid) {
+ return this.entriesByUUID.has(selector.uid);
+ }
+
+ if (selector instanceof NodeChildSelector) {
+ let parent_entry = null;
+ if (selector.parent instanceof NodeUIDSelector) {
+ // shortcut: uid parent + name child
+ parent_entry = this.entriesByUUID.get(selector.parent.uid);
+ } else {
+ // composite parent, has to stat it first
+ parent_entry = await this.stat({
+ selector: selector.parent,
+ });
+ }
+
+ if (parent_entry) {
+ const full_path = _path.join(parent_entry.path, selector.name);
+ const path_selector = new NodePathSelector(full_path);
+ return this.quick_check({
+ selector: path_selector,
+ });
+ }
+ }
+
+ return false;
+ }
+
+ /**
+ * Performs a stat operation using the given selector.
+ *
+ * NB: Some returned fields currently contain placeholder values. And the
+ * `path` of the absolute path from the root.
+ *
+ * @param {Object} param
+ * @param {NodeSelector} param.selector - The selector to stat.
+ * @returns {Promise} - The result of the stat operation, or `null` if the node doesn't exist.
+ */
+ async stat({ selector }) {
+ // memoryfs relies on attributes path/uid
+ try_infer_attributes(selector);
+
+ let entry_uuid = null;
+
+ if (selector?.path) {
+ // stat by path
+ const inner_path = this._inner_path(selector.path);
+ entry_uuid = this.entriesByPath.get(inner_path);
+ } else if (selector?.uid) {
+ // stat by uid
+ entry_uuid = selector.uid;
+ } else {
+ // the tricky case: selector has a composite (i.e. non path/uid) parent
+ if (selector instanceof NodeChildSelector) {
+ const parent_entry = await this.stat({
+ selector: selector.parent,
+ });
+ if (parent_entry) {
+ const full_path = _path.join(parent_entry.path, selector.name);
+ const path_selector = new NodePathSelector(full_path);
+ return await this.stat({
+ selector: path_selector,
+ });
+ }
+ }
+
+ throw APIError.create('invalid_node');
+ }
+
+ const entry = this.entriesByUUID.get(entry_uuid);
+ if ( ! entry ) {
+ return null;
+ }
+
+ // Return a copied entry with `full_path`, since external code only cares
+ // about full path.
+ const copied_entry = { ...entry };
+ copied_entry.path = _path.join(this.mountpoint, entry.path);
+ return copied_entry;
+ }
+
+ /**
+ * Read directory contents.
+ *
+ * @param {Object} param
+ * @param {Context} param.context - The context of the operation.
+ * @param {FSNodeContext} param.node - The directory node to read.
+ * @returns {Promise} - Array of child UUIDs.
+ */
+ async readdir({ context, node }) {
+ // prerequistes: get required path via stat
+ const entry = await this.stat({ selector: node.selector });
+ if ( ! entry ) {
+ throw APIError.create('invalid_node');
+ }
+
+ const inner_path = this._inner_path(entry.path);
+ const child_uuids = [];
+
+ // Find all entries that are direct children of this directory
+ for (const [path, uuid] of this.entriesByPath) {
+ if (path === inner_path) {
+ continue; // Skip the directory itself
+ }
+
+ const dirname = _path.dirname(path);
+ if (dirname === inner_path) {
+ child_uuids.push(uuid);
+ }
+ }
+
+ return child_uuids;
+ }
+
+ /**
+ * Create a new directory.
+ *
+ * @param {Object} param
+ * @param {Context} param.context - The context of the operation.
+ * @param {FSNodeContext} param.parent - The parent node to create the directory in. Must exist and be a directory.
+ * @param {string} param.name - The name of the new directory.
+ * @returns {Promise} - The new directory node.
+ */
+ async mkdir({ context, parent, name }) {
+ // prerequistes: get required path via stat
+ const parent_entry = await this.stat({ selector: parent.selector });
+ if ( ! parent_entry ) {
+ throw APIError.create('invalid_node');
+ }
+
+ const full_path = _path.join(parent_entry.path, name);
+ const inner_path = this._inner_path(full_path);
+
+ const entry = new MemoryFile({
+ path: inner_path,
+ is_dir: true,
+ content: null,
+ parent_uid: parent_entry.uuid,
+ });
+ this.entriesByPath.set(inner_path, entry.uuid);
+ this.entriesByUUID.set(entry.uuid, entry);
+
+ // create the node
+ const fs = context.get('services').get('filesystem');
+ const node = await fs.node(entry.uuid);
+ await node.fetchEntry();
+
+ this._integrity_check();
+
+ return node;
+ }
+
+ /**
+ * Remove a directory.
+ *
+ * @param {Object} param
+ * @param {Context} param.context
+ * @param {FSNodeContext} param.node: The directory to remove.
+ * @param {Object} param.options: The options for the operation.
+ * @returns {Promise}
+ */
+ async rmdir({ context, node, options = {} }) {
+ // prerequistes: get required path via stat
+ const entry = await this.stat({ selector: node.selector });
+ if ( ! entry ) {
+ throw APIError.create('invalid_node');
+ }
+
+ const inner_path = this._inner_path(entry.path);
+
+ // for mode: non-recursive
+ if (!options.recursive) {
+ const children = await this.readdir({ context, node });
+ if (children.length > 0) {
+ throw APIError.create('not_empty');
+ }
+ }
+
+ // remove all descendants
+ for (const [other_inner_path, other_entry_uuid] of this.entriesByPath) {
+ if (other_entry_uuid === entry.uuid) {
+ // skip the directory itself
+ continue;
+ }
+
+ if (other_inner_path.startsWith(inner_path)) {
+ this.entriesByPath.delete(other_inner_path);
+ this.entriesByUUID.delete(other_entry_uuid);
+ }
+ }
+
+ // for mode: non-descendants-only
+ if (!options.descendants_only) {
+ // remove the directory itself
+ this.entriesByPath.delete(inner_path);
+ this.entriesByUUID.delete(entry.uuid);
+ }
+
+ this._integrity_check();
+ }
+
+ /**
+ * Remove a file.
+ *
+ * @param {Object} param
+ * @param {Context} param.context
+ * @param {FSNodeContext} param.node: The file to remove.
+ * @returns {Promise}
+ */
+ async unlink({ context, node }) {
+ // prerequistes: get required path via stat
+ const entry = await this.stat({ selector: node.selector });
+ if ( ! entry ) {
+ throw APIError.create('invalid_node');
+ }
+
+ const inner_path = this._inner_path(entry.path);
+ this.entriesByPath.delete(inner_path);
+ this.entriesByUUID.delete(entry.uuid);
+ }
+
+ /**
+ * Move a file.
+ *
+ * @param {Object} param
+ * @param {Context} param.context
+ * @param {FSNodeContext} param.node: The file to move.
+ * @param {FSNodeContext} param.new_parent: The new parent directory of the file.
+ * @param {string} param.new_name: The new name of the file.
+ * @param {Object} param.metadata: The metadata of the file.
+ * @returns {Promise}
+ */
+ async move({ context, node, new_parent, new_name, metadata }) {
+ // prerequistes: get required path via stat
+ const new_parent_entry = await this.stat({ selector: new_parent.selector });
+ if ( ! new_parent_entry ) {
+ throw APIError.create('invalid_node');
+ }
+
+ // create the new entry
+ const new_full_path = _path.join(new_parent_entry.path, new_name);
+ const new_inner_path = this._inner_path(new_full_path);
+ const entry = new MemoryFile({
+ path: new_inner_path,
+ is_dir: node.entry.is_dir,
+ content: node.entry.content,
+ parent_uid: new_parent_entry.uuid,
+ });
+ entry.uuid = node.entry.uuid;
+ this.entriesByPath.set(new_inner_path, entry.uuid);
+ this.entriesByUUID.set(entry.uuid, entry);
+
+ // remove the old entry
+ const inner_path = this._inner_path(node.path);
+ this.entriesByPath.delete(inner_path);
+ // NB: should not delete the entry by uuid because uuid does not change
+ // after the move.
+
+ this._integrity_check();
+
+ return entry;
+ }
+
+ /**
+ * Copy a tree of files and directories.
+ *
+ * @param {Object} param
+ * @param {Context} param.context
+ * @param {FSNodeContext} param.source - The source node to copy.
+ * @param {FSNodeContext} param.parent - The parent directory for the copy.
+ * @param {string} param.target_name - The name for the copied item.
+ * @returns {Promise} - The copied node.
+ */
+ async copy_tree({ context, source, parent, target_name }) {
+ const fs = context.get('services').get('filesystem');
+
+ if (source.entry.is_dir) {
+ // Create the directory
+ const new_dir = await this.mkdir({ context, parent, name: target_name });
+
+ // Copy all children
+ const children = await this.readdir({ context, node: source });
+ for (const child_uuid of children) {
+ const child_node = await fs.node(new NodeUIDSelector(child_uuid));
+ await child_node.fetchEntry();
+ const child_name = child_node.entry.name;
+
+ await this.copy_tree({
+ context,
+ source: child_node,
+ parent: new_dir,
+ target_name: child_name,
+ });
+ }
+
+ return new_dir;
+ } else {
+ // Copy the file
+ const new_file = await this.write_new({
+ context,
+ parent,
+ name: target_name,
+ file: { stream: { read: () => source.entry.content } },
+ });
+ return new_file;
+ }
+ }
+
+ /**
+ * Write a new file to the filesystem. Throws an error if the destination
+ * already exists.
+ *
+ * @param {Object} param
+ * @param {Context} param.context
+ * @param {FSNodeContext} param.parent: The parent directory of the destination directory.
+ * @param {string} param.name: The name of the destination directory.
+ * @param {Object} param.file: The file to write.
+ * @returns {Promise}
+ */
+ async write_new({ context, parent, name, file }) {
+ // prerequistes: get required path via stat
+ const parent_entry = await this.stat({ selector: parent.selector });
+ if ( ! parent_entry ) {
+ throw APIError.create('invalid_node');
+ }
+ const full_path = _path.join(parent_entry.path, name);
+ const inner_path = this._inner_path(full_path);
+
+ const entry = new MemoryFile({
+ path: inner_path,
+ is_dir: false,
+ content: file.stream.read(),
+ parent_uid: parent_entry.uuid,
+ });
+ this.entriesByPath.set(inner_path, entry.uuid);
+ this.entriesByUUID.set(entry.uuid, entry);
+
+ const fs = context.get('services').get('filesystem');
+ const node = await fs.node(entry.uuid);
+ await node.fetchEntry();
+
+ this._integrity_check();
+
+ return node;
+ }
+
+ /**
+ * Overwrite an existing file. Throws an error if the destination does not
+ * exist.
+ *
+ * @param {Object} param
+ * @param {Context} param.context
+ * @param {FSNodeContext} param.node: The node to write to.
+ * @param {Object} param.file: The file to write.
+ * @returns {Promise}
+ */
+ async write_overwrite({ context, node, file }) {
+ const entry = await this.stat({ selector: node.selector });
+ if ( ! entry ) {
+ throw APIError.create('invalid_node');
+ }
+ const inner_path = this._inner_path(entry.path);
+
+ this.entriesByPath.set(inner_path, entry.uuid);
+ let original_entry = this.entriesByUUID.get(entry.uuid);
+ if (!original_entry) {
+ throw new Error(`File ${entry.path} does not exist`);
+ } else {
+ if (original_entry.is_dir) {
+ throw new Error(`Cannot overwrite a directory`);
+ }
+
+ original_entry.content = file.stream.read();
+ original_entry.modified = Math.floor(Date.now() / 1000);
+ original_entry.size = original_entry.content ? original_entry.content.length : 0;
+ this.entriesByUUID.set(entry.uuid, original_entry);
+ }
+
+ const fs = context.get('services').get('filesystem');
+ node = await fs.node(original_entry.uuid);
+ await node.fetchEntry();
+
+ this._integrity_check();
+
+ return node;
+ }
+}
+
+module.exports = {
+ MemoryFSProvider,
+};
diff --git a/src/backend/src/modules/puterfs/customfs/MemoryFSService.js b/src/backend/src/modules/puterfs/customfs/MemoryFSService.js
new file mode 100644
index 0000000000..99397ecfbd
--- /dev/null
+++ b/src/backend/src/modules/puterfs/customfs/MemoryFSService.js
@@ -0,0 +1,41 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see } - True if the node exists, false otherwise.
+ */
+ async quick_check ({
+ selector,
+ }) {
+ // a wrapper that access underlying database directly
+ const fsEntryFetcher = Context.get('services').get('fsEntryFetcher');
+
+ // shortcut: has full path
+ if ( selector?.path ) {
+ const entry = await fsEntryFetcher.findByPath(selector.path);
+ return Boolean(entry);
+ }
+
+ // shortcut: has uid
+ if ( selector?.uid ) {
+ const entry = await fsEntryFetcher.findByUID(selector.uid);
+ return Boolean(entry);
+ }
+
+ // shortcut: parent uid + child name
+ if ( selector instanceof NodeChildSelector && selector.parent instanceof NodeUIDSelector ) {
+ return await fsEntryFetcher.nameExistsUnderParent(
+ selector.parent.uid,
+ selector.name,
+ );
+ }
+
+ // shortcut: parent id + child name
+ if ( selector instanceof NodeChildSelector && selector.parent instanceof NodeInternalIDSelector ) {
+ return await fsEntryFetcher.nameExistsUnderParentID(
+ selector.parent.id,
+ selector.name,
+ );
+ }
+
+ // TODO (xiaochen): we should fallback to stat but we cannot at this moment
+ // since stat requires a valid `FSNodeContext` argument.
+ return false;
+ }
+
async stat ({
selector,
options,
@@ -429,6 +485,416 @@ class PuterFSProvider extends putility.AdvancedBase {
await tasks.awaitAll();
}
+
+ /**
+ * Create a new directory.
+ *
+ * @param {Object} param
+ * @param {Context} param.context
+ * @param {FSNode} param.parent
+ * @param {string} param.name
+ * @param {boolean} param.immutable
+ * @returns {Promise}
+ */
+ async mkdir({ context, parent, name, immutable}) {
+ const { actor, thumbnail } = context.values;
+
+ const svc_fslock = context.get('services').get('fslock');
+ const lock_handle = await svc_fslock.lock_child(
+ await parent.get('path'),
+ name,
+ MODE_WRITE,
+ );
+
+ try {
+ const { _path, uuidv4 } = this.modules;
+
+ const ts = Math.round(Date.now() / 1000);
+ const uid = uuidv4();
+ const resourceService = context.get('services').get('resourceService');
+ const svc_fsEntry = context.get('services').get('fsEntryService');
+ const svc_event = context.get('services').get('event');
+ const fs = context.get('services').get('filesystem');
+
+ const existing = await fs.node(
+ new NodeChildSelector(parent.selector, name)
+ );
+
+ if (await existing.exists()) {
+ throw APIError.create('item_with_same_name_exists', null, {
+ entry_name: name,
+ });
+ }
+
+ const svc_acl = context.get('services').get('acl');
+ if (! await parent.exists()) {
+ throw APIError.create('subject_does_not_exist');
+ }
+ if (! await svc_acl.check(actor, parent, 'write')) {
+ throw await svc_acl.get_safe_acl_error(actor, parent, 'write');
+ }
+
+ resourceService.register({
+ uid,
+ status: RESOURCE_STATUS_PENDING_CREATE,
+ });
+
+ const raw_fsentry = {
+ is_dir: 1,
+ uuid: uid,
+ parent_uid: await parent.get('uid'),
+ path: _path.join(await parent.get('path'), name),
+ user_id: actor.type.user.id,
+ name,
+ created: ts,
+ accessed: ts,
+ modified: ts,
+ immutable: immutable ?? false,
+ ...(thumbnail ? {
+ thumbnail: thumbnail,
+ } : {}),
+ };
+
+ const entryOp = await svc_fsEntry.insert(raw_fsentry);
+
+ await entryOp.awaitDone();
+ resourceService.free(uid);
+
+ const node = await fs.node(new NodeUIDSelector(uid));
+
+ svc_event.emit('fs.create.directory', {
+ node,
+ context: Context.get(),
+ });
+
+ return node
+ } finally {
+ await lock_handle.unlock();
+ }
+ }
+
+ /**
+ * Write a new file to the filesystem. Throws an error if the destination
+ * already exists.
+ *
+ * @param {Object} param
+ * @param {Context} param.context
+ * @param {FSNode} param.parent: The parent directory of the file.
+ * @param {string} param.name: The name of the file.
+ * @param {File} param.file: The file to write.
+ * @returns {Promise}
+ */
+ async write_new({context, parent, name, file}) {
+ const { _path, uuidv4, config } = this.modules;
+
+ const {
+ tmp, fsentry_tmp, message, actor: actor_let, app_id,
+ } = context.values;
+ let actor = actor_let ?? Context.get('actor');
+
+ const svc = Context.get('services');
+ const sizeService = svc.get('sizeService');
+ const resourceService = svc.get('resourceService');
+ const svc_fsEntry = svc.get('fsEntryService');
+ const svc_event = svc.get('event');
+ const fs = svc.get('filesystem');
+
+ // TODO: fs:decouple-versions
+ // add version hook externally so LLCWrite doesn't
+ // need direct database access
+ const db = svc.get('database').get(DB_WRITE, 'filesystem');
+
+ const uid = uuidv4();
+
+ // determine bucket region
+ let bucket_region = config.s3_region ?? config.region;
+ let bucket = config.s3_bucket;
+
+ const svc_acl = context.get('services').get('acl');
+ if ( ! await svc_acl.check(actor, parent, 'write') ) {
+ throw await svc_acl.get_safe_acl_error(actor, parent, 'write');
+ }
+
+ const storage_resp = await this._storage_upload({
+ uuid: uid,
+ bucket, bucket_region, file,
+ tmp: {
+ ...tmp,
+ path: _path.join(await parent.get('path'), name),
+ }
+ });
+
+ fsentry_tmp.thumbnail = await fsentry_tmp.thumbnail_promise;
+ delete fsentry_tmp.thumbnail_promise;
+
+ const ts = Math.round(Date.now() / 1000);
+ const raw_fsentry = {
+ uuid: uid,
+ is_dir: 0,
+ user_id: actor.type.user.id,
+ created: ts,
+ accessed: ts,
+ modified: ts,
+ parent_uid: await parent.get('uid'),
+ name,
+ size: file.size,
+ path: _path.join(await parent.get('path'), name),
+ ...fsentry_tmp,
+
+ bucket_region,
+ bucket,
+
+ associated_app_id: app_id ?? null,
+ };
+
+ svc_event.emit('fs.pending.file', {
+ fsentry: FSNodeContext.sanitize_pending_entry_info(raw_fsentry),
+ context,
+ })
+
+ resourceService.register({
+ uid,
+ status: RESOURCE_STATUS_PENDING_CREATE,
+ });
+
+ const filesize = file.size;
+ sizeService.change_usage(actor.type.user.id, filesize);
+
+ const entryOp = await svc_fsEntry.insert(raw_fsentry);
+
+ (async () => {
+ await entryOp.awaitDone();
+ resourceService.free(uid);
+
+ const new_item_node = await fs.node(new NodeUIDSelector(uid));
+ const new_item = await new_item_node.get('entry');
+ const store_version_id = storage_resp.VersionId;
+ if( store_version_id ){
+ // insert version into db
+ db.write(
+ "INSERT INTO `fsentry_versions` (`user_id`, `fsentry_id`, `fsentry_uuid`, `version_id`, `message`, `ts_epoch`) VALUES (?, ?, ?, ?, ?, ?)",
+ [
+ actor.type.user.id,
+ new_item.id,
+ new_item.uuid,
+ store_version_id,
+ message ?? null,
+ ts,
+ ]
+ );
+ }
+ })();
+
+ const node = await fs.node(new NodeUIDSelector(uid));
+
+ svc_event.emit('fs.create.file', {
+ node,
+ context,
+ });
+
+ return node;
+ }
+
+ /**
+ * Overwrite an existing file. Throws an error if the destination does not
+ * exist.
+ *
+ * @param {Object} param
+ * @param {Context} param.context
+ * @param {FSNodeContext} param.node: The node to write to.
+ * @param {File} param.file: The file to write.
+ * @returns {Promise}
+ */
+ async write_overwrite({ context, node, file }) {
+ const {
+ tmp, fsentry_tmp, message, actor: actor_let
+ } = context.values;
+ let actor = actor_let ?? Context.get('actor');
+
+ const svc = Context.get('services');
+ const sizeService = svc.get('sizeService');
+ const resourceService = svc.get('resourceService');
+ const svc_fsEntry = svc.get('fsEntryService');
+ const svc_event = svc.get('event');
+
+ // TODO: fs:decouple-versions
+ // add version hook externally so LLCWrite doesn't
+ // need direct database access
+ const db = svc.get('database').get(DB_WRITE, 'filesystem');
+
+ const svc_acl = context.get('services').get('acl');
+ if ( ! await svc_acl.check(actor, node, 'write') ) {
+ throw await svc_acl.get_safe_acl_error(actor, node, 'write');
+ }
+
+ const uid = await node.get('uid');
+
+ const bucket_region = node.entry.bucket_region;
+ const bucket = node.entry.bucket;
+
+ const state_upload = await this._storage_upload({
+ uuid: node.entry.uuid,
+ bucket, bucket_region, file,
+ tmp: {
+ ...tmp,
+ path: await node.get('path'),
+ }
+ });
+
+ if ( fsentry_tmp?.thumbnail_promise ) {
+ fsentry_tmp.thumbnail = await fsentry_tmp.thumbnail_promise;
+ delete fsentry_tmp.thumbnail_promise;
+ }
+
+ const ts = Math.round(Date.now() / 1000);
+ const raw_fsentry_delta = {
+ modified: ts,
+ accessed: ts,
+ size: file.size,
+ ...fsentry_tmp,
+ };
+
+ resourceService.register({
+ uid,
+ status: RESOURCE_STATUS_PENDING_CREATE,
+ });
+
+ const filesize = file.size;
+ sizeService.change_usage(actor.type.user.id, filesize);
+
+ const entryOp = await svc_fsEntry.update(uid, raw_fsentry_delta);
+
+ // depends on fsentry, does not depend on S3
+ (async () => {
+ await entryOp.awaitDone();
+ resourceService.free(uid);
+ svc_event.emit('fs.written.file', {
+ node,
+ context: this.context,
+ });
+ })();
+
+ state_upload.post_insert({
+ db, user: actor.type.user, node, uid, message, ts,
+ });
+
+ const svc_fileCache = context.get('services').get('file-cache');
+ await svc_fileCache.invalidate(node);
+
+ svc_event.emit('fs.write.file', {
+ node,
+ context,
+ });
+
+ return node;
+ }
+
+
+ async _storage_upload ({
+ uuid,
+ bucket, bucket_region, file,
+ tmp,
+ }) {
+ const { config } = this.modules;
+
+ const svc = Context.get('services');
+ const log = svc.get('log-service').create('fs._storage_upload');
+ const errors = svc.get('error-service').create(log);
+ const svc_event = svc.get('event');
+
+ const svc_mountpoint = svc.get('mountpoint');
+ const storage = svc_mountpoint.get_storage(this.constructor);
+
+ bucket ??= config.s3_bucket;
+ bucket_region ??= config.s3_region ?? config.region;
+
+ let upload_tracker = new UploadProgressTracker();
+
+ svc_event.emit('fs.storage.upload-progress', {
+ upload_tracker,
+ context: Context.get(),
+ meta: {
+ item_uid: uuid,
+ item_path: tmp.path,
+ }
+ })
+
+ if ( ! file.buffer ) {
+ let stream = file.stream;
+ let alarm_timeout = null;
+ stream = stuck_detector_stream(stream, {
+ timeout: STUCK_STATUS_TIMEOUT,
+ on_stuck: () => {
+ this.frame.status = OperationFrame.FRAME_STATUS_STUCK;
+ log.warn('Upload stream stuck might be stuck', {
+ bucket_region,
+ bucket,
+ uuid,
+ });
+ alarm_timeout = setTimeout(() => {
+ errors.report('fs.write.s3-upload', {
+ message: 'Upload stream stuck for too long',
+ alarm: true,
+ extra: {
+ bucket_region,
+ bucket,
+ uuid,
+ },
+ });
+ }, STUCK_ALARM_TIMEOUT);
+ },
+ on_unstuck: () => {
+ clearTimeout(alarm_timeout);
+ this.frame.status = OperationFrame.FRAME_STATUS_WORKING;
+ }
+ });
+ file = { ...file, stream, };
+ }
+
+ let hashPromise;
+ if ( file.buffer ) {
+ const hash = crypto.createHash('sha256');
+ hash.update(file.buffer);
+ hashPromise = Promise.resolve(hash.digest('hex'));
+ } else {
+ const hs = hashing_stream(file.stream);
+ file.stream = hs.stream;
+ hashPromise = hs.hashPromise;
+ }
+
+ hashPromise.then(hash => {
+ const svc_event = Context.get('services').get('event');
+ console.log('\x1B[36;1m[fs.write]', uuid, hash);
+ svc_event.emit('outer.fs.write-hash', {
+ hash, uuid,
+ });
+ });
+
+ const state_upload = storage.create_upload();
+
+ try {
+ await state_upload.run({
+ uid: uuid,
+ file,
+ storage_meta: { bucket, bucket_region },
+ storage_api: { progress_tracker: upload_tracker },
+ });
+ } catch (e) {
+ errors.report('fs.write.storage-upload', {
+ source: e || new Error('unknown'),
+ trace: true,
+ alarm: true,
+ extra: {
+ bucket_region,
+ bucket,
+ uuid,
+ },
+ });
+ throw APIError.create('upload_failed');
+ }
+
+ return state_upload;
+ }
}
module.exports = {
diff --git a/src/backend/src/modules/selfhosted/DefaultUserService.js b/src/backend/src/modules/selfhosted/DefaultUserService.js
index ad4f641385..b5c633684f 100644
--- a/src/backend/src/modules/selfhosted/DefaultUserService.js
+++ b/src/backend/src/modules/selfhosted/DefaultUserService.js
@@ -89,6 +89,8 @@ class DefaultUserService extends BaseService {
);
if ( ! is_default_password ) return;
+ console.log(`password for admin is: ${tmp_password}`);
+
// show console widget
this.default_user_widget = ({ is_docker }) => {
if ( is_docker ) {
diff --git a/src/backend/src/modules/web/WebServerService.js b/src/backend/src/modules/web/WebServerService.js
index cb319b58ba..4c4fcfa626 100644
--- a/src/backend/src/modules/web/WebServerService.js
+++ b/src/backend/src/modules/web/WebServerService.js
@@ -612,7 +612,7 @@ class WebServerService extends BaseService {
req.co_isolation_enabled
;
- if ( req.path === '/signup' || req.path === '/login' ) {
+ if ( req.path === '/signup' || req.path === '/login' || req.path.startsWith('/extensions/') ) {
res.setHeader('Access-Control-Allow-Origin', origin ?? '*');
}
// Website(s) to allow to connect
diff --git a/src/backend/src/modules/web/lib/eggspress.js b/src/backend/src/modules/web/lib/eggspress.js
index 95aa24e9ab..0567d8fdf4 100644
--- a/src/backend/src/modules/web/lib/eggspress.js
+++ b/src/backend/src/modules/web/lib/eggspress.js
@@ -24,6 +24,7 @@ const api_error_handler = require('./api_error_handler.js');
const APIError = require('../../../api/APIError.js');
const { Context } = require('../../../util/context.js');
const { subdomain } = require('../../../helpers.js');
+const config = require('../../../config.js');
/**
* eggspress() is a factory function for creating express routers.
@@ -169,6 +170,9 @@ module.exports = function eggspress (route, settings, handler) {
return next();
}
}
+ if ( config.env === 'dev' ) {
+ console.log(`request url: ${req.url}, body: ${JSON.stringify(req.body)}`);
+ }
try {
const expected_ctx = res.locals.ctx;
const received_ctx = Context.get(undefined, { allow_fallback: true });
@@ -179,6 +183,13 @@ module.exports = function eggspress (route, settings, handler) {
});
} else await handler(req, res, next);
} catch (e) {
+ if ( config.env === 'dev' ) {
+ if (! (e instanceof APIError)) {
+ // Any non-APIError indicates an unhandled error (i.e. a bug) from the backend.
+ // We add a dedicated branch to facilitate debugging.
+ console.error(e);
+ }
+ }
api_error_handler(e, req, res, next);
}
};
diff --git a/src/backend/src/om/IdentifierUtil.js b/src/backend/src/om/IdentifierUtil.js
index fcecbce6c8..58b8e05d19 100644
--- a/src/backend/src/om/IdentifierUtil.js
+++ b/src/backend/src/om/IdentifierUtil.js
@@ -26,7 +26,7 @@ class IdentifierUtil extends AdvancedBase {
new WeakConstructorFeature(),
]
- async detect_identifier (object) {
+ async detect_identifier (object, allow_mutation = false) {
const redundant_identifiers = this.om.redundant_identifiers ?? [];
let match_found = null;
@@ -60,9 +60,9 @@ class IdentifierUtil extends AdvancedBase {
await object.get(key) : object[key],
}));
if ( object instanceof Entity ) {
- await object.del(key);
+ if ( allow_mutation ) await object.del(key);
} else {
- delete object[key];
+ if ( allow_mutation ) delete object[key];
}
}
let predicate = new And({ children: key_eqs });
diff --git a/src/backend/src/om/entitystorage/SQLES.js b/src/backend/src/om/entitystorage/SQLES.js
index 8af09a66de..32b5cd52d8 100644
--- a/src/backend/src/om/entitystorage/SQLES.js
+++ b/src/backend/src/om/entitystorage/SQLES.js
@@ -22,7 +22,7 @@ const { BaseES } = require("./BaseES");
const APIError = require("../../api/APIError");
const { Entity } = require("./Entity");
const { WeakConstructorFeature } = require("../../traits/WeakConstructorFeature");
-const { And, Or, Eq, Like, Null, Predicate, PredicateUtil, IsNotNull } = require("../query/query");
+const { And, Or, Eq, Like, Null, Predicate, PredicateUtil, IsNotNull, StartsWith } = require("../query/query");
const { DB_WRITE } = require("../../services/database/consts");
class RawCondition extends AdvancedBase {
@@ -400,6 +400,25 @@ class SQLES extends BaseES {
return new RawCondition({ sql, values });
}
+
+ if (om_query instanceof StartsWith) {
+ const key = om_query.key;
+ let value = om_query.value;
+ const prop = this.om.properties[key];
+
+ value = await prop.sql_reference(value);
+
+ const options = prop.descriptor.sql ?? {};
+ const col_name = options.column_name ?? prop.name;
+
+ const sql = `${col_name} LIKE ${this.db.case({
+ sqlite: `? || '%'`,
+ otherwise: `CONCAT(?, '%')`
+ })}`;
+ const values = value === null ? [] : [value];
+
+ return new RawCondition({ sql, values });
+ }
if ( om_query instanceof IsNotNull ) {
const key = om_query.key;
diff --git a/src/backend/src/om/entitystorage/SubdomainES.js b/src/backend/src/om/entitystorage/SubdomainES.js
index 4a063f582d..45cc9473fb 100644
--- a/src/backend/src/om/entitystorage/SubdomainES.js
+++ b/src/backend/src/om/entitystorage/SubdomainES.js
@@ -24,6 +24,8 @@ const { Context } = require("../../util/context");
const { Eq } = require("../query/query");
const { BaseES } = require("./BaseES");
+const PERM_READ_ALL_SUBDOMAINS = 'read-all-subdomains';
+
class SubdomainES extends BaseES {
static METHODS = {
async _on_context_provided () {
@@ -52,12 +54,17 @@ class SubdomainES extends BaseES {
// Note: we don't need to worry about read;
// non-owner users don't have permission to list
// but they still have permission to read.
- options.predicate = options.predicate.and(
- new Eq({
- key: 'owner',
- value: user.id,
- }),
- );
+ const svc_permission = this.context.get('services').get('permission');
+ const has_permission_to_read_all = await svc_permission.check(Context.get("actor"), PERM_READ_ALL_SUBDOMAINS);
+
+ if (!has_permission_to_read_all) {
+ options.predicate = options.predicate.and(
+ new Eq({
+ key: 'owner',
+ value: user.id,
+ }),
+ );
+ }
return await this.upstream.select(options);
},
diff --git a/src/backend/src/om/entitystorage/ValidationES.js b/src/backend/src/om/entitystorage/ValidationES.js
index 5c70d69942..94cee80b13 100644
--- a/src/backend/src/om/entitystorage/ValidationES.js
+++ b/src/backend/src/om/entitystorage/ValidationES.js
@@ -19,6 +19,8 @@
const { BaseES } = require("./BaseES");
const APIError = require("../../api/APIError");
+const { Context } = require("../../util/context");
+const { SKIP_ES_VALIDATION } = require("./consts");
class ValidationES extends BaseES {
async _on_context_provided () {
@@ -61,7 +63,7 @@ class ValidationES extends BaseES {
return await out_entity.get_client_safe();
},
async validate_ (entity, diff) {
- const processed = {};
+ if ( Context.get(SKIP_ES_VALIDATION) ) return;
for ( const prop of Object.values(this.om.properties) ) {
let value = await entity.get(prop.name);
@@ -96,8 +98,6 @@ class ValidationES extends BaseES {
}
throw e;
}
-
- processed[prop.name] = value;
}
},
diff --git a/src/backend/src/om/entitystorage/consts.js b/src/backend/src/om/entitystorage/consts.js
new file mode 100644
index 0000000000..1dade2a72e
--- /dev/null
+++ b/src/backend/src/om/entitystorage/consts.js
@@ -0,0 +1,3 @@
+module.exports = {
+ SKIP_ES_VALIDATION: Symbol('SKIP_ES_VALIDATION'),
+};
diff --git a/src/backend/src/om/query/query.js b/src/backend/src/om/query/query.js
index 4bffcdba67..96d2ea666c 100644
--- a/src/backend/src/om/query/query.js
+++ b/src/backend/src/om/query/query.js
@@ -50,6 +50,12 @@ class Eq extends Predicate {
}
}
+class StartsWith extends Predicate {
+ async check(entity) {
+ return (await entity.get(this.key)).startsWith(this.value);
+ }
+}
+
class IsNotNull extends Predicate {
async check (entity) {
return (await entity.get(this.key)) !== null;
@@ -122,4 +128,5 @@ module.exports = {
Eq,
IsNotNull,
Like,
+ StartsWith
};
diff --git a/src/backend/src/routers/confirm-email.js b/src/backend/src/routers/confirm-email.js
index 465dcb0c21..360ff3a02d 100644
--- a/src/backend/src/routers/confirm-email.js
+++ b/src/backend/src/routers/confirm-email.js
@@ -43,15 +43,22 @@ router.post('/confirm-email', auth, express.json(), async (req, res, next)=>{
const db = req.services.get('database').get(DB_WRITE, 'auth');
// Increment & check rate limit
- if(kv.incr(`confirm-email|${req.ip}|${req.body.email ?? req.body.username}`) > 10)
+ if(kv.incr(`confirm-email|${req.ip}|${req.user.email ?? req.user.username}`) > 10)
return res.status(429).send({error: 'Too many requests.'});
// Set expiry for rate limit
- kv.expire(`confirm-email|${req.ip}|${req.body.email ?? req.body.username}`, 60 * 10, 'NX')
+ kv.expire(`confirm-email|${req.ip}|${req.user.email ?? req.user.username}`, 60 * 10, 'NX')
+
+ console.log('need to check these', typeof req.body.code, typeof req.user.email_confirm_code, req.body.code, req.user.email_confirm_code);
+
+ if(req.body.code !== req.user.email_confirm_code) {
+ res.send({ email_confirmed: false });
+ return;
+ }
// Scenario: email was confirmed on another account already
{
const svc_cleanEmail = req.services.get('clean-email');
- const clean_email = svc_cleanEmail.clean(req.body.email);
+ const clean_email = svc_cleanEmail.clean(req.user.email);
if ( ! await svc_cleanEmail.validate(clean_email) ) {
APIError.create('field_invalid', null, {
@@ -63,7 +70,7 @@ router.post('/confirm-email', auth, express.json(), async (req, res, next)=>{
const rows = await db.read(
`SELECT EXISTS(
SELECT 1 FROM user WHERE (email=? OR clean_email=?) AND email_confirmed=1 AND password IS NOT NULL
- ) AS email_exists`, [req.body.email, clean_email]);
+ ) AS email_exists`, [req.user.email, clean_email]);
if ( rows[0].email_exists ) {
APIError.create('email_already_in_use').write(res);
return;
@@ -76,37 +83,34 @@ router.post('/confirm-email', auth, express.json(), async (req, res, next)=>{
[req.user.email],
);
- if(req.body.code === req.user.email_confirm_code) {
- await db.write(
- "UPDATE `user` SET `email_confirmed` = 1, `requires_email_confirmation` = 0 WHERE id = ? LIMIT 1",
- [req.user.id],
- );
- const svc_getUser = req.services.get('get-user');
- await svc_getUser.get_user({ id: req.user.id, force: true });
-
- const svc_event = req.services.get('event');
- svc_event.emit('user.email-confirmed', {
- user_uid: req.user.uuid,
- email: req.user.email,
- });
- }
+ // Update user record to say email is confirmed
+ await db.write(
+ "UPDATE `user` SET `email_confirmed` = 1, `requires_email_confirmation` = 0 WHERE id = ? LIMIT 1",
+ [req.user.id],
+ );
+
+ // Invalidate user cache
+ const svc_getUser = req.services.get('get-user');
+ await svc_getUser.get_user({ id: req.user.id, force: true });
- // Build response object
- const res_obj = {
- email_confirmed: (req.body.code === req.user.email_confirm_code),
- original_client_socket_id: req.body.original_client_socket_id,
- }
+ // Emit internal event
+ const svc_event = req.services.get('event');
+ svc_event.emit('user.email-confirmed', {
+ user_uid: req.user.uuid,
+ email: req.user.email,
+ });
- // Send realtime success msg to client
- if(req.body.code === req.user.email_confirm_code){
- const svc_socketio = req.services.get('socketio');
- svc_socketio.send({ room: req.user.id }, 'user.email_confirmed', {
- original_client_socket_id: req.body.original_client_socket_id
- });
- }
+ // Emit websocket event (TODO: should come from internal event above)
+ const svc_socketio = req.services.get('socketio');
+ svc_socketio.send({ room: req.user.id }, 'user.email_confirmed', {
+ original_client_socket_id: req.body.original_client_socket_id
+ });
// return results
- return res.send(res_obj)
+ return res.send({
+ email_confirmed: true,
+ original_client_socket_id: req.body.original_client_socket_id,
+ });
})
module.exports = router
diff --git a/src/backend/src/routers/filesystem_api/stat.js b/src/backend/src/routers/filesystem_api/stat.js
index 6680714b78..8c0501b054 100644
--- a/src/backend/src/routers/filesystem_api/stat.js
+++ b/src/backend/src/routers/filesystem_api/stat.js
@@ -43,6 +43,7 @@ module.exports = eggspress('/stat', {
user: req.user,
return_subdomains: req.body.return_subdomains,
return_permissions: req.body.return_permissions,
+ return_shares: req.body.return_shares,
return_versions: req.body.return_versions,
return_size: req.body.return_size,
});
diff --git a/src/backend/src/routers/save_account.js b/src/backend/src/routers/save_account.js
index b54af39915..f7f438a509 100644
--- a/src/backend/src/routers/save_account.js
+++ b/src/backend/src/routers/save_account.js
@@ -147,7 +147,7 @@ router.post('/save_account', auth, express.json(), async (req, res, next)=>{
// password
await bcrypt.hash(req.body.password, 8),
// email_confirm_code
- email_confirm_code,
+ '' + email_confirm_code,
//email_confirm_token
email_confirm_token,
// referred_by
diff --git a/src/backend/src/routers/send-confirm-email.js b/src/backend/src/routers/send-confirm-email.js
index 5b5e26a0f8..76ccbfaf2e 100644
--- a/src/backend/src/routers/send-confirm-email.js
+++ b/src/backend/src/routers/send-confirm-email.js
@@ -46,7 +46,7 @@ router.post('/send-confirm-email', auth, express.json(), async (req, res, next)=
`UPDATE user SET email_confirm_code = ? WHERE id = ?`,
[
// email_confirm_code
- email_confirm_code,
+ '' + email_confirm_code,
// id
req.user.id,
]);
diff --git a/src/backend/src/routers/signup.js b/src/backend/src/routers/signup.js
index ce972cd801..63fdbc2cab 100644
--- a/src/backend/src/routers/signup.js
+++ b/src/backend/src/routers/signup.js
@@ -271,7 +271,7 @@ module.exports = eggspress(['/signup'], {
// referrer
req.body.referrer ?? null,
// email_confirm_code
- email_confirm_code,
+ '' + email_confirm_code,
// email_confirm_token
email_confirm_token,
// free_storage
diff --git a/src/backend/src/services/EntityStoreService.js b/src/backend/src/services/EntityStoreService.js
index 334d09ca8d..2bae2f92b1 100644
--- a/src/backend/src/services/EntityStoreService.js
+++ b/src/backend/src/services/EntityStoreService.js
@@ -194,7 +194,7 @@ class EntityStoreService extends BaseService {
om: this.om,
});
- const predicate = await idu.detect_identifier(id ?? {});
+ const predicate = await idu.detect_identifier(id ?? {}, true);
if ( predicate ) {
const maybe_entity = await this.select({ predicate, limit: 1 });
if ( maybe_entity.length ) {
diff --git a/src/backend/src/services/LocalDiskStorageService.js b/src/backend/src/services/LocalDiskStorageService.js
index 79434bde9b..e093f130fb 100644
--- a/src/backend/src/services/LocalDiskStorageService.js
+++ b/src/backend/src/services/LocalDiskStorageService.js
@@ -18,6 +18,7 @@
* along with this program. If not, see
+
diff --git a/src/dev-center/img/website-white.svg b/src/dev-center/img/website-white.svg
new file mode 100644
index 0000000000..7ef28b1af1
--- /dev/null
+++ b/src/dev-center/img/website-white.svg
@@ -0,0 +1,3 @@
+
+
\ No newline at end of file
diff --git a/src/dev-center/img/website.svg b/src/dev-center/img/website.svg
new file mode 100644
index 0000000000..5dcc7b279e
--- /dev/null
+++ b/src/dev-center/img/website.svg
@@ -0,0 +1,3 @@
+
+
\ No newline at end of file
diff --git a/src/dev-center/img/websites-placeholder.svg b/src/dev-center/img/websites-placeholder.svg
new file mode 100644
index 0000000000..06fb067b85
--- /dev/null
+++ b/src/dev-center/img/websites-placeholder.svg
@@ -0,0 +1,2 @@
+
+
+
+
+
\ No newline at end of file
diff --git a/src/dev-center/img/workers-white.svg b/src/dev-center/img/workers-white.svg
new file mode 100644
index 0000000000..b664600efd
--- /dev/null
+++ b/src/dev-center/img/workers-white.svg
@@ -0,0 +1,3 @@
+
+
\ No newline at end of file
diff --git a/src/dev-center/img/workers.svg b/src/dev-center/img/workers.svg
new file mode 100644
index 0000000000..d2faa0fbfa
--- /dev/null
+++ b/src/dev-center/img/workers.svg
@@ -0,0 +1,3 @@
+
+
\ No newline at end of file
diff --git a/src/dev-center/index.html b/src/dev-center/index.html
index df732a9ba5..7a371e52c1 100644
--- a/src/dev-center/index.html
+++ b/src/dev-center/index.html
@@ -19,6 +19,9 @@
Payout Method
+
+
+
+
+
+ You haven't created any workers yet.
+
+
+
+
+
+ You haven't created any websites yet.
+
@@ -209,26 +234,27 @@ Payout Method
My Apps
-
-
+
-
-
-
+
Delete
+
+
+
+
+
+
+
+
My Workers
+
+
+
+
+
Delete
+
+
+
+
+
+
+
My Websites
+
+
+
+
+
Delete
+
+
diff --git a/src/dev-center/js/apps.js b/src/dev-center/js/apps.js
new file mode 100644
index 0000000000..6b52fbcb86
--- /dev/null
+++ b/src/dev-center/js/apps.js
@@ -0,0 +1,2765 @@
+let source_path
+let apps = [];
+let sortBy = 'created_at';
+let sortDirection = 'desc';
+let currently_editing_app;
+let dropped_items;
+let search_query;
+let originalValues = {};
+
+const APP_CATEGORIES = [
+ { id: 'games', label: 'Games' },
+ { id: 'developer-tools', label: 'Developer Tools' },
+ { id: 'photo-video', label: 'Photo & Video' },
+ { id: 'productivity', label: 'Productivity' },
+ { id: 'utilities', label: 'Utilities' },
+ { id: 'education', label: 'Education' },
+ { id: 'business', label: 'Business' },
+ { id: 'social', label: 'Social' },
+ { id: 'graphics-design', label: 'Graphics & Design' },
+ { id: 'music-audio', label: 'Music & Audio' },
+ { id: 'news', label: 'News' },
+ { id: 'entertainment', label: 'Entertainment' },
+ { id: 'finance', label: 'Finance' },
+ { id: 'health-fitness', label: 'Health & Fitness' },
+ { id: 'lifestyle', label: 'Lifestyle' },
+];
+
+async function init_apps() {
+ setTimeout(async function () {
+ puter.ui.onLaunchedWithItems(async function (items) {
+ source_path = items[0].path;
+ // if source_path is provided, this means that the user is creating a new app/updating an existing app
+ // by deploying an existing Puter folder. So we create the app and deploy it.
+ if (source_path) {
+ // todo if there are no apps, go straight to creating a new app
+ $('.insta-deploy-modal').get(0).showModal();
+ // set item name
+ $('.insta-deploy-item-name').html(html_encode(items[0].name));
+ }
+ })
+
+ // Get dev profile. This is only for puter.com for now as we don't have dev profiles in self-hosted Puter
+ if(domain === 'puter.com'){
+ puter.apps.getDeveloperProfile(async function (dev_profile) {
+ window.developer = dev_profile;
+ if (dev_profile.approved_for_incentive_program && !dev_profile.joined_incentive_program) {
+ $('#join-incentive-program').show();
+ }
+
+ // show earn money c2a only if dev is not approved for incentive program or has already joined
+ if (!dev_profile.approved_for_incentive_program || dev_profile.joined_incentive_program) {
+ puter.kv.get('earn-money-c2a-closed').then((value) => {
+ if (value?.result || value === true || value === "true")
+ return;
+
+ $('#earn-money').get(0).showModal();
+ });
+ }
+
+ // show payout method tab if dev has joined incentive program
+ if (dev_profile.joined_incentive_program) {
+ $('.tab-btn[data-tab="payout-method"]').show();
+ $('#payout-method-email').html(dev_profile.paypal);
+ $('.tab-btn-separator').show();
+ }
+ })
+ }
+ // Get apps
+ puter.apps.list({ icon_size: 64 }).then((resp) => {
+ apps = resp;
+
+ // hide loading
+ puter.ui.hideSpinner();
+
+ // set apps
+ if (apps.length > 0) {
+ if (window.activeTab === 'apps') {
+ $('#no-apps-notice').hide();
+ $('#app-list').show();
+ }
+ $('.app-card').remove();
+ apps.forEach(app => {
+ $('#app-list-table > tbody').append(generate_app_card(app));
+ });
+ count_apps();
+ sort_apps();
+ activate_tippy();
+ } else {
+ $('#no-apps-notice').show();
+ }
+ })
+ }, 1000);
+
+}
+
+
+/**
+ * Refreshes the list of apps in the UI.
+ *
+ * @param {boolean} [show_loading=false] - Whether to show a loading indicator while refreshing.
+ *
+ */
+
+window.refresh_app_list = (show_loading = false) => {
+ if (show_loading)
+ puter.ui.showSpinner();
+ // get apps
+ setTimeout(function () {
+ // uncheck the select all checkbox
+ $('.select-all-apps').prop('checked', false);
+
+ puter.apps.list({ icon_size: 64 }).then((apps_res) => {
+ puter.ui.hideSpinner();
+ apps = apps_res;
+ if (apps.length > 0) {
+ if (window.activeTab === 'apps') {
+ $('#no-apps-notice').hide();
+ $('#app-list').show();
+ }
+ $('.app-card').remove();
+ apps.forEach(app => {
+ $('#app-list-table > tbody').append(generate_app_card(app));
+ });
+ count_apps();
+ sort_apps();
+ } else {
+ $('#no-apps-notice').show();
+ $('#app-list').hide()
+ }
+ activate_tippy();
+ puter.ui.hideSpinner();
+ })
+ }, show_loading ? 1000 : 0);
+}
+
+$(document).on('click', '.create-an-app-btn', async function (e) {
+ let title = await puter.ui.prompt('Please enter a title for your app:', 'My Awesome App');
+
+ if (title.length > 60) {
+ puter.ui.alert(`Title cannot be longer than 60.`, [
+ {
+ label: 'Ok',
+ },
+ ]);
+ // todo go back to create an app prompt and prefill the title input with the title the user entered
+ return;
+ }
+ else if (title) {
+ create_app(title);
+ }
+})
+
+async function create_app(title, source_path = null, items = null) {
+ // name
+ let name = slugify(title, {
+ lower: true,
+ strict: true,
+ });
+
+ // icon
+ let icon = await getBase64ImageFromUrl('./img/app.svg');
+
+ // open the 'Creting new app...' modal
+ let start_ts = Date.now();
+
+ puter.ui.showSpinner();
+
+ //----------------------------------------------------
+ // Create app
+ //----------------------------------------------------
+ puter.apps.create({
+ title: title,
+ name: name,
+ indexURL: 'https://dev-center.puter.com/coming-soon.html',
+ icon: icon,
+ description: ' ',
+ maximizeOnStart: false,
+ background: false,
+ dedupeName: true,
+ metadata: {
+ window_resizable: true,
+ fullpage_on_landing: true,
+ },
+ })
+ .then(async (app) => {
+ let app_dir;
+ // ----------------------------------------------------
+ // Create app directory in AppData
+ // ----------------------------------------------------
+ app_dir = await puter.fs.mkdir(
+ `/${auth_username}/AppData/${dev_center_uid}/${app.uid}`,
+ { overwrite: true, recursive: true, rename: false }
+ );
+ // ----------------------------------------------------
+ // Create a router for the app with a fresh hostname
+ // ----------------------------------------------------
+ let subdomain = name + '-' + Math.random().toString(36).substring(2)
+ await puter.hosting.create(subdomain, app_dir.path);
+
+ // ----------------------------------------------------
+ // Update the app with the new hostname
+ // ----------------------------------------------------
+ puter.apps.update(app.name, {
+ title: title,
+ indexURL: source_path ? protocol + `://${subdomain}.` + static_hosting_domain : 'https://dev-center.puter.com/coming-soon.html',
+ icon: icon,
+ description: ' ',
+ maximizeOnStart: false,
+ background: false,
+ metadata: {
+ category: null, // default category on creation
+ window_resizable: true,
+ fullpage_on_landing: true,
+ }
+ }).then(async (app) => {
+ // refresh app list
+ puter.apps.list({ icon_size: 64 }).then(async (resp) => {
+ apps = resp;
+ // Close the 'Creating new app...' modal
+ // but make sure it was shown for at least 2 seconds
+ setTimeout(() => {
+ // open edit app section
+ edit_app_section(app.name);
+
+ // set drop area if source_path was provided or items were dropped
+ if (source_path || items) {
+ $('.drop-area').removeClass('drop-area-hover');
+ $('.drop-area').addClass('drop-area-ready-to-deploy');
+ }
+ puter.ui.hideSpinner();
+ // deploy app if source_path was provided
+ if (source_path) {
+ deploy(app, source_path);
+ } else if (items) {
+ deploy(app, items);
+ }
+ activate_tippy();
+ }, (Date.now() - start_ts) > 2000 ? 1 : 2000 - (Date.now() - start_ts));
+ })
+ }).catch(async (err) => {
+ console.log(err);
+ })
+ // ----------------------------------------------------
+ // Create a "shortcut" on the desktop
+ // ----------------------------------------------------
+ puter.fs.upload(new File([], app.title),
+ `/${auth_username}/Desktop`,
+ {
+ name: app.title,
+ dedupeName: true,
+ overwrite: false,
+ appUID: app.uid,
+ }
+ )
+ //----------------------------------------------------
+ // Increment app count
+ //----------------------------------------------------
+ $('.app-count').html(parseInt($('.app-count').html() ?? 0) + 1);
+
+ }).catch(async (err) => {
+ $('#create-app-error').show();
+ $('#create-app-error').html(err.message);
+ // scroll to top so that user sees error message
+ document.body.scrollTop = document.documentElement.scrollTop = 0;
+ })
+}
+
+$(document).on('click', '.deploy-btn', function (e) {
+ deploy(currently_editing_app, dropped_items);
+})
+
+$(document).on('click', '.edit-app, .go-to-edit-app', function (e) {
+ const cur_app_name = $(this).attr('data-app-name')
+ edit_app_section(cur_app_name);
+})
+
+$(document).on('click', '.delete-app', async function (e) {
+})
+
+// generate app link
+function applink(app) {
+ return protocol + `://${domain}${ port ? ':' + port : '' }/app/${app.name}`;
+}
+
+/**
+ * Generates the HTML for the app editing section.
+ *
+ * @param {Object} app - The app object containing details of the app to be edited.
+ * *
+ * @returns {string} HTML string for the app editing section.
+ *
+ * @description
+ * This function creates the HTML for the app editing interface, including:
+ * - App icon and title display
+ * - Options to open, add to desktop, or delete the app
+ * - Tabs for deployment and settings
+ * - Form fields for editing various app properties
+ * - Display of app statistics
+ *
+ * The generated HTML includes interactive elements and placeholders for
+ * dynamic content to be filled or updated by other functions.
+ *
+ * @example
+ * const appEditHTML = generate_edit_app_section(myAppObject);
+ * $('#edit-app').html(appEditHTML);
+ */
+
+function generate_edit_app_section(app) {
+ if(app.result)
+ app = app.result;
+
+ let maximize_on_start = app.maximize_on_start ? 'checked' : '';
+
+ let h = ``;
+ h += `
+ 
-
+ 
+ 
+