diff --git a/.vuepress/config.js b/.vuepress/config.js index 06ed7fac..7b100ad3 100644 --- a/.vuepress/config.js +++ b/.vuepress/config.js @@ -223,7 +223,20 @@ module.exports = { { title: 'Python SDK', collapsable: false, - children: ['python', 'py-examples', 'py-sample'], + children: [ + 'py-getting-started', + 'py-core-concepts', + 'py-authentication', + 'py-howto-base-objects', + 'py-howto-work-with-transports', + 'py-howto-collaboration', + 'py-howto-work-with-large-data', + 'py-howto-realtime', + 'py-howto-geometry', + 'py-howto-instances', + // 'py-examples', + // 'py-sample', + ], }, { title: 'Javascript SDK', @@ -260,17 +273,19 @@ module.exports = { { title: 'Core Functionalities', collapsable: true, - children: [ - 'for-automate-users', - 'for-function-authors', - ], + children: ['for-automate-users', 'for-function-authors'], }, ], }, { title: 'Automations', collapsable: false, - children: ['create-automation', 'update-automation', 'viewing-results', 'troubleshooting'], + children: [ + 'create-automation', + 'update-automation', + 'viewing-results', + 'troubleshooting', + ], }, { title: 'Functions', @@ -339,8 +354,8 @@ module.exports = { 'rendering-pipeline-api/progressive-pipeline-api', 'rendering-pipeline-api/g-pass-api', 'rendering-pipeline-api/base-g-pass-api', - 'rendering-pipeline-api/progressive-g-pass-api' - ] + 'rendering-pipeline-api/progressive-g-pass-api', + ], }, 'speckle-material-api', 'speckle-renderer-api', @@ -387,7 +402,12 @@ module.exports = { sidebarDepth: 0, title: 'Workspace Docs 👩‍🏭', collapsable: false, - children: ['', 'welcome-to-workspaces', 'getting-started', 'advanced-features'], + children: [ + '', + 'welcome-to-workspaces', + 'getting-started', + 'advanced-features', + ], }, { title: 'Refererences', diff --git a/dev/py-authentication.md b/dev/py-authentication.md new file mode 100644 index 00000000..bb5420bd --- /dev/null +++ b/dev/py-authentication.md @@ -0,0 +1,196 @@ +# Authentication in specklepy + +Authentication is the first step in connecting to a Speckle server and managing your AEC data programmatically. This guide covers the available authentication methods and provides practical examples to get you started. + +## Authentication Methods + +specklepy offers two main authentication methods: + +1. **Account-based Authentication** (Recommended) + - Uses Speckle Manager credentials + - Handles token refresh automatically + - Provides access to all authorized servers + - Best for desktop applications and local development + +2. **Personal Access Token (PAT)** + - Server-specific tokens + - Fixed expiration + - Best for automation, CI/CD, or headless scripts + +## Using Speckle Manager Account + +Speckle Manager simplifies authentication by securely managing your account credentials. + +Example: + +```python +from specklepy.api.client import SpeckleClient +from specklepy.api.credentials import get_default_account, get_local_accounts + +# Get all local accounts +accounts = get_local_accounts() + +# Get default account (recommended) +account = get_default_account() + +# Initialize and authenticate client +client = SpeckleClient(host="speckle.xyz") +client.authenticate_with_account(account) +``` + +## Using Personal Access Tokens + +Personal Access Tokens provide a flexible, server-specific authentication option. + +Example: + +```python +# Direct token authentication +client = SpeckleClient(host="speckle.xyz") +client.authenticate_with_token("your-token-here") + +# Or create account from token +from specklepy.api.credentials import get_account_from_token +account = get_account_from_token("your-token", "https://speckle.xyz") +client.authenticate_with_account(account) +``` + +## When to Use Each Method + +### Use Account Authentication When + +- Working locally with Speckle Manager installed +- Needing access to multiple servers +- Developing desktop applications +- Want automatic token refresh + +### Use Personal Access Tokens When + +- Running automated scripts +- Setting up CI/CD pipelines +- Working in environments without Speckle Manager +- Need to limit access scope +- Working with specific servers + +## Setting Up Personal Access Tokens + +1. Log in to your Speckle server. +2. Navigate to Profile Settings > Tokens. +3. Create a new token with the required scopes: + - `profile:read` for user info. + - `streams:read` and `streams:write` for stream access. + - `users:read` for searching users. + - etc. + +Example: + +```python +# Create token programmatically +token = client.server.create_token( + name="API Access", + scopes=["streams:read", "streams:write"], + lifespan=30 # days +) +``` + +## Security Best Practices + +1. Store tokens securely: Use environment variables to keep tokens safe: + + ```python + import os + token = os.environ.get("SPECKLE_TOKEN") + ``` + +2. Use minimal scopes: Request only the permissions you need: + + ```python + # For read-only access + client.server.create_token( + name="Reader", + scopes=["streams:read"] + ) + ``` + +3. Revoke unused tokens: Revoke tokens no longer in use to minimize security risks: + + ```python + client.server.revoke_token(token) + ``` + +4. Keep Tokens Out of Source Code: Use a `.env` file and add it to `.gitignore`. + +## Error Handling + +Robust error handling ensures smooth authentication workflows: + +```python +try: + client.authenticate_with_token(token) +except SpeckleException as ex: + if "401" in str(ex): + print("Invalid or expired token") + elif "403" in str(ex): + print("Insufficient permissions") +``` + +## Checking Authentication Status + +```python +# Verify authentication +user = client.active_user.get() +if user: + print(f"Authenticated as {user.name}") +else: + print("Not authenticated") + +# Check server connection +server_info = client.server.get() +print(f"Connected to {server_info.name}") +``` + +## Environment Configuration + +Store configuration in `.env` files for portability: + +```bash +# .env +SPECKLE_TOKEN="your-token" +SPECKLE_SERVER="https://speckle.xyz" +PROJECT_ID="your-project-id" +MODEL_ID="your-model-id" +``` + +Load environment variables using python-dotenv: + +```python +from dotenv import load_dotenv +import os + +# Load from .env file +load_dotenv() + +# Initialize client with environment config +client = SpeckleClient( + host=os.getenv("SPECKLE_SERVER", "speckle.xyz") +) +client.authenticate_with_token( + os.getenv("SPECKLE_TOKEN") +) + +# Use in project configuration +project_id = os.getenv("PROJECT_ID") +model_id = os.getenv("MODEL_ID") + +# Different environments can use different .env files: +# .env.development +# .env.production +# .env.staging +load_dotenv(f".env.{os.getenv('ENVIRONMENT', 'development')}") + +# Add .env files to .gitignore +# .env +# .env.* +``` + +This pattern keeps credentials secure and makes scripts portable across different environments and team members. diff --git a/dev/py-core-concepts.md b/dev/py-core-concepts.md new file mode 100644 index 00000000..c15c68f7 --- /dev/null +++ b/dev/py-core-concepts.md @@ -0,0 +1,136 @@ +# Core Concepts + +Understanding the core concepts in specklepy is essential for working effectively with the platform. +This guide introduces the fundamental ideas and their practical applications in managing AEC data. + +## Base Objects + +Base objects are the fundamental building blocks in Speckle. Every piece of data you work with inherits from the Base class. +This allows for a flexible and extensible approach to creating, managing, and sharing AEC data. + +Why Use Base Objects? + +1. Dynamic Properties: Add properties dynamically at runtime to adapt to changing requirements. +2. Typed Objects: Define specific object types with custom properties. +3. Custom Type Names: Assign domain-specific type names for better organization and clarity. + +```python +from specklepy.objects import Base + +# Basic object with dynamic properties +obj = Base() +obj.dynamic_property = "value" # Add properties dynamically +print(obj.dynamic_property) # Outputs: value + +# Create a typed object +class Wall(Base): + height: float = 0.0 + width: float = 0.0 + +wall = Wall() +wall.height = 3.0 +wall.width = 0.3 +print(wall.height, wall.width) # Outputs: 3.0, 0.3 + +# Define a custom type name +class Beam(Base, speckle_type="Elements.Structural.Beam"): + length: float = 0.0 + +beam = Beam() +beam.length = 5.0 +print(beam.speckle_type) # Outputs: Elements.Structural.Beam +``` + +Best Practices: + +- Use dynamic properties for flexibility but prefer typed objects for predictable structures. +- Follow naming conventions for custom types (Namespace.Category.Type) to ensure clarity and consistency. +- Document expected properties and their units. +Real-World Application: +In an AEC workflow, Base objects could represent components like walls, beams, or furniture, allowing teams to iterate and customize data as project requirements evolve. + +## Transport System + +The transport system in Speckle moves data between applications, local storage, and cloud servers. It is designed for flexibility and scalability, enabling both local workflows and global collaboration. + +Why Use Transports? + +1. Local Storage: Cache frequently used data for faster access. +2. Server Communication: Share data with your team or store it long-term. +3. Hybrid Workflows: Combine multiple transport types for robust workflows. + +Example: Sending Data with Transports + +```python +from specklepy.api import operations +from specklepy.transports.server import ServerTransport +from specklepy.transports.sqlite import SQLiteTransport + +# Local cache (default) +sqlite = SQLiteTransport() + +# Server transport +server = ServerTransport( + stream_id="your-stream", + token="your-token", + url="https://speckle.xyz" +) + +# Send using multiple transports +obj_id = operations.send( + base=wall, + transports=[sqlite, server] +) +``` + +Best Practices: + +- Always include a local transport for caching when using a server transport. +- Use meaningful transport scopes and names for better organization. +- Avoid sending large, monolithic models. Use chunking and detaching where appropriate. + +Real-World Application: +Architects can use transports to sync data between local workstations and cloud servers, +ensuring seamless collaboration across dispersed teams. + +## Units & Geometry + +Speckle handles units explicitly to ensure consistency across teams and applications. Geometry in Speckle is unit-aware, enabling automatic conversions during data exchange. + +Why Specify Units? + +1. Consistency: Ensure measurements are interpreted correctly across different applications. +2. Flexibility: Convert units dynamically without losing data integrity. +3. Interoperability: Work seamlessly with collaborators using different unit systems. + +Example: Working with Units and Geometry: + +```python +from specklepy.objects.geometry import Point +from specklepy.objects.units import Units + +# Create geometry with units +pt = Point(x=1, y=2, z=3) +pt.units = Units.mm + +# Check units +print(pt.units) # "mm" + +# Unit conversion +pt.units = Units.m # Values automatically converted +``` + +Supported Units: + +- Metric: mm, cm, m, km +- Imperial: in, ft, yd, mi +- Default: Meters (m) + +Best Practices: + +- Always define units explicitly when creating geometry. +- Verify units after receiving geometry to avoid misinterpretations. +- Use consistent units within teams to reduce conversion errors. + +Real-World Application: +When importing a building model created in meters into a project using millimeters, Speckle ensures automatic conversion, saving time and reducing errors. diff --git a/dev/py-examples.md b/dev/py-examples.md index 3e3d424f..9df606bb 100644 --- a/dev/py-examples.md +++ b/dev/py-examples.md @@ -169,6 +169,7 @@ branch_id = client.branch.create("stream_id", "branch name", "a description of t branch = client.branch.get("stream_id", "branch name") ``` + ### Operations and Transports The `operations` include four main methods: @@ -248,7 +249,7 @@ base_obj["@nested"] = detached_base The `Base` class has a few handy instance methods for identifying your object's typed and dynamic attributes: - `get_typed_member_names()` gets all of the names of the defined (typed) attributes of the object -- `get_dynamic_member_names()` gets all of the names of the dynamic attributes of the object +- `get_dynamic_member_names()` gets all of the names of the dynamic attributes of the object - `get_member_names()` gets a list of all the attributes on the object, dynamic or not Each `Base` object has an `id` (a unique hash) as it does in the other SDKs. This field is only populated if the `Base` has been previously serialized. If you *really* need the hash, you can get it using the `get_id()` method. Be aware that this call will fully serialize the object to create the `id` if the `id` is not populated! By default, the hash will be generated without decomposing the object. However, you can pass `decompose=True` as an argument if you want the decomposed `id`. diff --git a/dev/py-getting-started.md b/dev/py-getting-started.md new file mode 100644 index 00000000..186f0f29 --- /dev/null +++ b/dev/py-getting-started.md @@ -0,0 +1,202 @@ +# Getting Started with specklepy + +specklepy is the Python SDK for Speckle, enabling you to interact with the Speckle platform programmatically. It's compatible with Python `>=3.9.0` (`<4.0`) and provides tools for working with AEC data, including geometry creation, data serialization, and server interactions. + +## Installation + +specklepy can be installed via pip: + +```bash +pip install specklepy +``` + +## Key Components + +SpecklePy consists of three main parts: + +1. `SpeckleClient`: Interact with the server API (authentication, projects, models) +2. `operations` and `transports`: Send and receive large objects +3. `Base` object and serializer: Create and customize Speckle objects + +## Local Data Paths + +SpecklePy stores local accounts and object cache databases in the following locations: + +- Windows: `%APPDATA%\Speckle` or `\AppData\Roaming\Speckle` +- Linux: `$XDG_DATA_HOME/Speckle` or `~/.local/share/Speckle` +- Mac: `~/.config/Speckle` + +## Authentication + +Before interacting with a Speckle server, you need to authenticate. Specklepy offers two authentication methods: + +```python +from specklepy.api.client import SpeckleClient +from specklepy.api.credentials import get_default_account + +# Initialize client - use your server's URL +client = SpeckleClient(host="speckle.xyz") + +# Method 1: Use Speckle Manager account +# Recommended if you've installed Speckle Manager +account = get_default_account() +client.authenticate_with_account(account) + +# Method 2: Use personal access token +# Get this from your Speckle server profile +client.authenticate_with_token("your-token") +``` + +## Creating Your First Project + +Projects in Speckle organize your data and collaboration. Each project can contain multiple models and versions. + +```python +from specklepy.api.models import Project +from specklepy.core.api.inputs.project_inputs import ProjectCreateInput + +# Create a new project +project = client.project.create( + input=ProjectCreateInput( + name="First Project", + description="My first Speckle project" + ) +) + +project_id = project.id + +# Projects can be public or private +# You can set visibility and other properties during creation +project = client.project.create( + input=ProjectCreateInput( + name="Public Project", + description="A collaborative project", + visibility="public" + ) +) +``` + +## Working with Models + +Models represent different aspects or iterations of your project. They can contain various types of data, from 3D geometry to analytical results. + +```python +from specklepy.core.api.inputs.model_inputs import CreateModelInput + +# Create a model within your project +model = client.model.create( + input=CreateModelInput( + name="First Model", + projectId=project_id + ) +) + +model_id = model.id + +# Models can be organized however you like +# For example, by discipline or phase +architecture_model = client.model.create( + input=CreateModelInput( + name="Architecture", + projectId=project_id, + description="Architectural design model" + ) +) +``` + +## Sending Data + +specklepy uses a transport system to move data between your application and Speckle servers. The `StreamWrapper` simplifies this process. + +```python +from specklepy.api import operations +from specklepy.api.wrapper import StreamWrapper +from specklepy.objects import Base + +# Create a simple building object +# Base objects can have any properties you need +building = Base() +building.name = "Simple Building" +building.levels = 5 +building.height = 15.0 +building.function = "office" + +# Setup transport using StreamWrapper +# This handles authentication and server connection +wrapper = StreamWrapper(f"https://speckle.xyz/projects/{project_id}") +transport = wrapper.get_transport() + +# Send object to Speckle +object_id = operations.send( + base=building, + transports=[transport] +) +``` + +## Creating a Version + +Versions capture the state of your model at a point in time, similar to commits in version control. + +```python +from specklepy.core.api.inputs.version_inputs import CreateVersionInput + +# Create a version with your data +version = client.version.create( + input=CreateVersionInput( + objectId=object_id, + modelId=model_id, + projectId=project_id, + message="Initial version", + sourceApplication="python" # Optional: identify the source + ) +) + +# You can create multiple versions to track changes +version2 = client.version.create( + input=CreateVersionInput( + objectId=object_id, + modelId=model_id, + projectId=project_id, + message="Updated floor heights" + ) +) +``` + +## Retrieving Data + +Data can be retrieved from any version using the transport system: + +```python +# Receive object using transport +received = operations.receive( + obj_id=object_id, + remote_transport=transport +) + +print(received.name) # "Simple Building" +print(received.levels) # 5 + +# Access any custom properties you've added +print(received.function) # "office" +``` + +## Next Steps + +This guide covered the basics of specklepy. There's much more you can do: + +1. **Geometry Creation**: specklepy includes a comprehensive geometry system with support for points, curves, meshes, and more. See the Geometry guide for details. + +2. **Custom Objects**: Create your own object types by subclassing `Base` to represent your domain-specific data. + +3. **Advanced Features**: + - Work with multiple transports simultaneously + - Handle large datasets using chunking + - Create custom geometric operations + - Set up real-time subscriptions + +4. **Team Collaboration**: + - Manage project permissions + - Share models with teammates + - Track changes through versions + +Check out the other guides for detailed information on these topics. diff --git a/dev/py-howto-base-objects.md b/dev/py-howto-base-objects.md new file mode 100644 index 00000000..694c13cb --- /dev/null +++ b/dev/py-howto-base-objects.md @@ -0,0 +1,156 @@ +# How-to: Work with Base Objects + +## Creating Custom Types + +Create typed objects by inheriting from `Base`: + +```python +from specklepy.objects.base import Base + +class Wall(Base): + height: float = 0.0 + width: float = 0.0 + material: str = "concrete" + +# Create instance +wall = Wall(height=3.0, width=0.3) +``` + +Specify custom type names using `speckle_type`: + +```python +class Beam(Base, speckle_type="Elements.Structural.Beam"): + length: float = 0.0 +``` + +## Dynamic Properties + +Add properties dynamically at runtime: + +```python +obj = Base() +obj.custom_prop = "value" +obj.nested = Base() +obj.nested.prop = 42 +``` + +Access properties using dot notation or dictionary style: + +```python +value = obj.custom_prop +value = obj["custom_prop"] +``` + +## Property Validation + +Properties are type-checked at runtime: + +```python +class Column(Base): + height: float + +column = Column() +column.height = "invalid" # Raises SpeckleException - expected float +``` + +Supported types: + +- Basic: int, float, str, bool +- Collections: List, Dict, Optional +- Other Base objects +- Enums + +## Detaching + +Use detaching for properties that should be stored as separate objects: + +- Referenced geometry +- Shared components +- Large sub-objects +- Collections of Base objects + +```python +class Model(Base, detachable={"geometry"}): + geometry: List[Base] # Stored as separate objects +``` + +## Chunking + +Use chunking specifically for large lists of primitive values: + +- Arrays of numbers (vertices, indices) +- Lists of coordinates +- Other numeric data collections +- NOT for lists of Base objects (use detaching instead) + +```python +class Mesh(Base, chunkable={"vertices": 1000}): + vertices: List[float] # Split into chunks of 1000 +``` + +## Data Access & Traversal + +Get property names: + +```python +obj.get_member_names() # All properties +obj.get_typed_member_names() # Defined properties +obj.get_dynamic_member_names() # Runtime properties +``` + +Count children: + +```python +child_count = obj.get_children_count() +``` + +Get object ID (hash): + +```python +obj_id = obj.get_id() +``` + +## Serialization + +Convert to dictionary: + +```python +obj_dict = obj.to_dict() +``` + +Use the operations module for full serialization: + +```python +from specklepy.api import operations + +# Serialize +serialized = operations.serialize(obj) + +# Deserialize +obj = operations.deserialize(serialized) +``` + +## Best Practices + +1. Use type hints for all properties in custom classes + +2. Set meaningful default values + +3. Use descriptive speckle_type names following convention: + - Namespace.Category.Type + - e.g. "Elements.Structural.Beam" + +4. Choose between chunking and detaching: + - Use detaching for Base object collections and referenced data + - Use chunking for large lists of primitive values + - Never chunk lists of Base objects + +5. Document custom types: + - Property descriptions + - Units expectations + - Usage examples + +6. Validate data: + - Check required properties + - Verify value ranges + - Handle edge cases diff --git a/dev/py-howto-collaboration.md b/dev/py-howto-collaboration.md new file mode 100644 index 00000000..c5ba29d0 --- /dev/null +++ b/dev/py-howto-collaboration.md @@ -0,0 +1,431 @@ +# How-to: Project Collaboration + +## Understanding Collaboration in Speckle + +When working on building and infrastructure projects, effective collaboration is crucial. Multiple team members, various disciplines, and different organizations all need to work together seamlessly. Speckle's collaboration system is designed to make this complex process manageable. + +### The Project Concept + +Think of a Speckle Project like a digital job site. Just as a physical construction site needs: + +- Different teams working in designated areas +- A way to track progress +- Clear communication channels +- Security and access control + +A Speckle Project provides digital versions of all these necessities: + +- Models organize different aspects of work (like architecture, structure, MEP) +- Versions track progress and changes over time +- Comments and notifications enable communication +- Role-based permissions control who can do what + +## Setting Up Your Digital Workspace + +Let's start by creating a project - your team's digital headquarters: + +```python +from specklepy.api.client import SpeckleClient +from specklepy.core.api.inputs.project_inputs import ProjectCreateInput +from specklepy.core.api.enums import ProjectVisibility + +# Connect to Speckle +client = SpeckleClient(host="speckle.xyz") +client.authenticate_with_token("your-token") + +# Create the project workspace +project = client.project.create( + input=ProjectCreateInput( + name="Office Tower Design", + description="Collaborative design for the new office tower", + visibility=ProjectVisibility.PRIVATE + ) +) + +project_id = project.id +``` + +The visibility setting is important: + +- **PRIVATE**: Only invited team members can access (default and recommended for most projects) +- **PUBLIC**: Anyone can view, but only team members can edit (good for public proposals or educational projects) +- **UNLISTED**: Like private, but accessible via direct link (useful for client presentations) + +## Building Your Team + +Just like a real project needs the right people with the right responsibilities, your Speckle project needs a well-organized team. + +### Understanding Roles + +Speckle has three main project roles: + +- **Owner**: Like a project manager - full control over the project +- **Contributor**: Like team members - can add and modify content +- **Reviewer**: Like consultants or clients - can view and comment + +Here's how to invite team members: + +```python +from specklepy.core.api.inputs.project_inputs import ProjectInviteCreateInput + +# Invite a team member +invite = client.project_invite.create( + project_id=project_id, + input=ProjectInviteCreateInput( + email="architect@company.com", + role="project.contributor", # Their permission level + ) +) +``` + +Think carefully about roles: + +- ✅ Owners should be project leads or BIM managers +- ✅ Contributors are your active team members +- ✅ Reviewers are stakeholders who need to view and comment +- ❌ Don't make everyone an owner - it complicates project management +- ❌ Don't use reviewer role for active team members - they'll be too limited + +### Managing Team Access + +As projects evolve, you might need to adjust team members' roles: + +```python +from specklepy.core.api.inputs.project_inputs import ProjectUpdateRoleInput + +# Update someone's role - maybe they're now leading the project +client.project.update_role( + input=ProjectUpdateRoleInput( + userId="user-id", + projectId=project_id, + role="project.owner" # Promoting to owner + ) +) +``` + +## Organizing Work with Models + +Models in Speckle are like different drawings or files in traditional workflows, but more powerful. They help organize different aspects of your project: + +```python +from specklepy.core.api.inputs.model_inputs import CreateModelInput + +# Create models for different disciplines +architecture = client.model.create( + input=CreateModelInput( + name="100 - Architecture", # Using a numbering system helps organize + projectId=project_id, + description="Main architectural design including layouts and facades" + ) +) + +structure = client.model.create( + input=CreateModelInput( + name="200 - Structure", + projectId=project_id, + description="Primary and secondary structural systems" + ) +) +``` + +Best practices for models: + +- ✅ Use a consistent naming convention (like numbers for disciplines) +- ✅ Give clear, detailed descriptions +- ✅ Create separate models for major disciplines +- ❌ Don't put everything in one model +- ❌ Don't create too many small models + +## Tracking Changes with Versions + +Versions are like save points in your project's history. They help you: + +- Track what changed and why +- Know who made changes +- Roll back if needed +- Understand the project's evolution + +```python +from specklepy.core.api.inputs.version_inputs import CreateVersionInput + +# Create a version when you've made significant changes +version = client.version.create( + input=CreateVersionInput( + objectId=object_id, # The ID of your updated data + modelId=architecture.id, + projectId=project_id, + message="Updated floor layouts based on client feedback from 2024-01-05 meeting:\n" + "- Enlarged lobby area\n" + "- Relocated reception desk\n" + "- Added security office", + sourceApplication="python" + ) +) +``` + +Version best practices: + +- ✅ Write detailed commit messages +- ✅ Include why changes were made +- ✅ Reference meetings or decisions +- ✅ Create versions at logical break points +- ❌ Don't create versions for tiny changes +- ❌ Don't write vague messages like "updates" + +## Real-time Collaboration + +Speckle can notify team members about changes as they happen: + +```python +import asyncio +from typing import Callable + +async def monitor_project_changes(): + def on_model_update(message): + # When someone updates a model: + update_type = message.type # CREATED, UPDATED, or DELETED + model = message.model + + print(f"Update to {model.name}") + print(f"Made by: {model.author.name}") + print(f"Type: {update_type}") + + # Start watching for changes + await client.subscription.project_models_updated( + callback=on_model_update, + id=project_id + ) + +# Run this to stay informed +asyncio.run(monitor_project_changes()) +``` + +This is useful for: + +- ✅ Staying informed about team progress +- ✅ Coordinating work in real-time +- ✅ Catching potential conflicts early +- ❌ Don't use it for critical notifications (use proper project management tools) + +## Sharing Data + +### Within Your Team + +When sharing data with team members, use Speckle's transport system: + +```python +from specklepy.api import operations +from specklepy.api.wrapper import StreamWrapper +from specklepy.objects.base import Base + +# Setup sharing +wrapper = StreamWrapper(f"https://speckle.xyz/projects/{project_id}") +transport = wrapper.get_transport() + +# Prepare your data +design_update = Base() +design_update.name = "Level 3 Layout" +design_update.data = { + "floors": 5, + "area": 1000, + "updated_areas": ["lobby", "offices"] +} + +# Share it +object_id = operations.send(base=design_update, transports=[transport]) + +# Make it official with a version +version = client.version.create( + input=CreateVersionInput( + objectId=object_id, + modelId=architecture.id, + projectId=project_id, + message="Level 3 layout updates for open office configuration" + ) +) +``` + +### With External Stakeholders + +Sometimes you need to share work with people outside your team: + +```python +# Option 1: Make the project public +client.project.update( + input=ProjectUpdateInput( + id=project_id, + visibility=ProjectVisibility.PUBLIC + ) +) + +# Option 2: Invite as reviewers +client.project_invite.create( + project_id=project_id, + input=ProjectInviteCreateInput( + email="client@company.com", + role="project.reviewer" + ) +) + +# Share the URL +project_url = f"https://speckle.xyz/projects/{project_id}" +``` + +Choose the right sharing method: + +- ✅ Use reviewer invites for clients and consultants +- ✅ Use public visibility for showcases or portfolios +- ❌ Don't make sensitive projects public +- ❌ Don't give external stakeholders contributor access + +## Common Collaboration Scenarios + +### Design Review Workflow + +Here's how to set up a design review process: + +```python +# 1. Create a review model +review_model = client.model.create( + input=CreateModelInput( + name="Design Review", + projectId=project_id, + description="Consolidated model for client review meeting 2024-01-15" + ) +) + +# 2. Invite reviewers +client.project_invite.create( + project_id=project_id, + input=ProjectInviteCreateInput( + email="reviewer@client.com", + role="project.reviewer" + ) +) + +# 3. Create a review version with clear documentation +version = client.version.create( + input=CreateVersionInput( + objectId=object_id, + modelId=review_model.id, + projectId=project_id, + message="Design Review Package - January 2024\n" + "For review:\n" + "- Ground floor layout\n" + "- Facade treatment\n" + "- MEP coordination" + ) +) +``` + +### Multi-discipline Coordination + +When different teams need to work together: + +```python +def track_coordination(model_id, object_id, dependencies): + """ + Keep track of which versions of different models work together. + Dependencies is a list of {name, version} dictionaries. + """ + message = "Coordinated Model Update\n\n" + message += "Referenced Models:\n" + for dep in dependencies: + message += f"- {dep['name']}: Version {dep['version']}\n" + + return client.version.create( + input=CreateVersionInput( + objectId=object_id, + modelId=model_id, + projectId=project_id, + message=message + ) + ) + +# Usage example +track_coordination( + model_id=coordination_model.id, + object_id=merged_model_id, + dependencies=[ + {"name": "Architecture", "version": "A1.2"}, + {"name": "Structure", "version": "S1.1"}, + {"name": "MEP", "version": "M1.3"} + ] +) +``` + +## Troubleshooting Guide + +### "Cannot Access" Errors + +If team members can't access what they need: + +```python +def diagnose_access(project_id, user_id): + """Help figure out why someone can't access something""" + + # Check their current role + project = client.project.get_with_team(project_id) + user_role = None + + for member in project.team: + if member.user.id == user_id: + user_role = member.role + break + + if not user_role: + print("User is not a team member!") + print("They need to be invited to the project") + else: + print(f"User has role: {user_role}") + if user_role == "project.reviewer": + print("Note: Reviewers cannot modify content") +``` + +### Version Conflicts + +When multiple people are working simultaneously: + +```python +from datetime import datetime, timedelta + +def check_for_conflicts(model_id, project_id): + """Help identify if there might be conflicting changes""" + + versions = client.version.get_versions( + model_id=model_id, + project_id=project_id, + limit=10 + ) + + # Look for very close timestamps + recent_changes = [] + for version in versions.items: + recent_changes.append({ + 'time': version.createdAt, + 'author': version.authorUser.name, + 'message': version.message + }) + + # Sort by time + recent_changes.sort(key=lambda x: x['time']) + + # Check for changes close together + for i in range(len(recent_changes)-1): + time_diff = recent_changes[i+1]['time'] - recent_changes[i]['time'] + if time_diff < timedelta(minutes=5): + print("Potential conflict detected!") + print(f"Changes by {recent_changes[i]['author']} and " + f"{recent_changes[i+1]['author']} within 5 minutes") +``` + +Remember: Good collaboration is about communication and organization. Speckle provides the tools, but it's up to your team to use them effectively: + +- Keep your project structure clear +- Write helpful version messages +- Use appropriate roles +- Monitor project activity +- Coordinate between disciplines +- Document important decisions + +The more organized your digital workspace is, the smoother your team's collaboration will be. diff --git a/dev/py-howto-geometry.md b/dev/py-howto-geometry.md new file mode 100644 index 00000000..01cd432c --- /dev/null +++ b/dev/py-howto-geometry.md @@ -0,0 +1,393 @@ +# How-to: Work with Geometry + +## Important Note About Geometric Operations + +specklepy provides objects and schemas for representing geometry, but it does not include functionality for geometric computations, analysis, or validation. For these tasks, you will need to integrate with specialized libraries such as: + +- **Trimesh** or **PyMesh** for mesh processing +- **PythonOCC** for BREP operations +- **Numpy** for vector math + +### What specklepy CAN do + +- Programmatically define geometry in text-based formats. +- Serialize and transport geometry efficiently. +- Ensure schema conformance for interoperability. + +### What specklepy CANNOT do + +- Perform geometric operations like measuring distances or intersections. +- Validate geometric correctness, such as checking for watertight meshes or proper face orientations. + +## Available Geometry Objects + +specklepy provides base classes for common geometric entities: + +```python +from specklepy.objects.geometry import ( + Point, + Vector, + Line, + Polyline, + Curve, + Mesh, + Brep, + Surface +) + +# Basic point +pt = Point(x=1.0, y=2.0, z=3.0) + +# Vector with direction +vec = Vector(x=0.0, y=0.0, z=1.0) + +# Line between points +line = Line( + start=Point(x=0, y=0, z=0), + end=Point(x=10, y=0, z=0) +) +``` + +## Units + +Always specify units when creating geometric objects to ensure proper interpretation: + +```python +from specklepy.objects.units import Units + +# Create point with explicit units +point = Point(x=1000, y=2000, z=0) +point.units = Units.mm # Specify as millimeters + +# The same point in different units +point.units = Units.m # Converting to meters +print(point.x) # Will print 1.0 +``` + +## Creating Simple Geometry + +Here's how to create basic geometric objects: + +```python +# Create a polyline +polyline = Polyline() +polyline.value = [0,0,0, 1,0,0, 1,1,0, 0,1,0] # Flattened coordinates +polyline.closed = True +polyline.units = Units.m + +# Create a mesh +mesh = Mesh() +mesh.vertices = [0,0,0, 1,0,0, 1,1,0, 0,1,0] # Vertex coordinates +mesh.faces = [4, 0,1,2,3] # First number is vertex count for n-gon support +mesh.units = Units.m + +# Note: Speckle meshes support n-gon faces (faces with any number of vertices) +# Many mesh libraries including trimesh only support triangles +``` + +## Working with BREPs + +While specklepy can represent BREP geometry, it doesn't perform BREP operations: + +```python +# Create a BREP container +brep = Brep() +brep.displayValue = [mesh] # Display mesh representation +brep.units = Units.m + +# Note: Actual BREP operations would need to be performed +# using an external geometric kernel +``` + +## Surfaces + +Surface representation without operations: + +```python +# Create a surface definition +surface = Surface() +surface.degreeU = 3 +surface.degreeV = 3 +surface.pointData = [/* control points */] +surface.units = Units.m + +# Note: Surface evaluation and operations would need +# to be performed by external libraries +``` + +## Handling Large Geometry + +For large geometric objects, use chunking to manage data efficiently: + +```python +from specklepy.objects.base import Base + +class LargeGeometry(Base, chunkable={ + "vertices": 1000, # Split vertices into chunks of 1000 + "faces": 1000, # Split faces into chunks of 1000 +}): + vertices: List[float] = None + faces: List[int] = None + +# Create large mesh +large_mesh = LargeGeometry() +large_mesh.vertices = [/* many vertices */] +large_mesh.faces = [/* many faces */] +``` + +## Geometric Collections + +Group related geometry using Base objects: + +```python +class GeometryCollection(Base, detachable={"elements"}): + elements: List[Base] = None + +# Create a collection +collection = GeometryCollection() +collection.elements = [ + Point(x=0, y=0, z=0), + Line( + start=Point(x=0, y=0, z=0), + end=Point(x=1, y=0, z=0) + ), + mesh +] +``` + +## Validation and Checking + +Remember that specklepy only validates schema conformance, not geometric validity: + +```python +# This is schema-valid but possibly geometrically invalid +invalid_mesh = Mesh() +invalid_mesh.vertices = [0,0,0, 1,1,1] # Only two vertices +invalid_mesh.faces = [3, 0,1,1] # Invalid face +# specklepy will allow this - geometric validation +# needs to be done by your application + +# Schema validation will catch this: +try: + point = Point(x="not a number", y=0, z=0) +except SpeckleException as e: + print("Schema validation failed:", e) +``` + +## Working with trimesh + +trimesh is a powerful Python library for working with triangle meshes. However, Speckle meshes need conversion as they use a different format and support n-gon faces which trimesh does not. Here's how to convert between the formats: + +```python +import numpy as np +import trimesh +from typing import Tuple + +def speckle_to_trimesh(speckle_mesh: Mesh) -> trimesh.Trimesh: + """ + Convert a Speckle mesh to a trimesh mesh. + Handles n-gon triangulation implicitly through trimesh. + """ + # Convert Speckle's flat vertex array to numpy + vertices = np.array(speckle_mesh.vertices).reshape(-1, 3) + + # Convert Speckle's faces format to triangles + faces = [] + i = 0 + while i < len(speckle_mesh.faces): + # Get number of vertices in this face + n = speckle_mesh.faces[i] + # Get face vertices + face = speckle_mesh.faces[i + 1:i + 1 + n] + + if n == 3: + # Already a triangle + faces.append(face) + else: + # For n-gons, create a fan triangulation + # Note: This is a simple triangulation that may not work well for all cases + for j in range(1, n - 1): + faces.append([face[0], face[j], face[j + 1]]) + + i += n + 1 + + faces = np.array(faces) + + # Create trimesh + return trimesh.Trimesh(vertices=vertices, faces=faces) + +def trimesh_to_speckle(trim_mesh: trimesh.Trimesh) -> Mesh: + """ + Convert a trimesh mesh to a Speckle mesh. + Note that trimesh only supports triangles. + """ + # Create Speckle mesh + mesh = Mesh() + + # Convert vertices to flat array + mesh.vertices = trim_mesh.vertices.flatten().tolist() + + # Convert faces to Speckle format (with count prefix for each face) + faces = [] + for face in trim_mesh.faces: + faces.extend([3, *face]) # 3 vertices per face (triangles) + mesh.faces = faces + + return mesh + +# Example usage: +speckle_mesh = Mesh() +speckle_mesh.vertices = [0,0,0, 1,0,0, 1,1,0, 0,1,0, 0.5,0.5,1] +speckle_mesh.faces = [5, 0,1,2,3,4] # Pentagon (n-gon) face + +# Convert to trimesh for operations +trim_mesh = speckle_to_trimesh(speckle_mesh) + +# Now you can use trimesh's powerful features +# For example, calculate volume: +volume = trim_mesh.volume + +# Or check watertightness: +is_watertight = trim_mesh.is_watertight + +# Or compute normals: +face_normals = trim_mesh.face_normals + +# Convert back to Speckle when done +result_mesh = trimesh_to_speckle(trim_mesh) +``` + +Note that when converting n-gon faces to triangles, the triangulation method can significantly impact the result. The simple fan triangulation shown above works for convex polygons but might not be suitable for all cases. For more complex needs, consider using more sophisticated triangulation algorithms. + +## Integration with Other Libraries + +Example of using specklepy with numpy for geometric calculations: + +```python +import numpy as np + +def calculate_centroid(points: List[Point]) -> Point: + # Convert to numpy for computation + coords = np.array([[p.x, p.y, p.z] for p in points]) + centroid = coords.mean(axis=0) + + # Create speckle point with result + return Point( + x=float(centroid[0]), + y=float(centroid[1]), + z=float(centroid[2]) + ) + +# Example usage +points = [ + Point(x=0, y=0, z=0), + Point(x=1, y=0, z=0), + Point(x=1, y=1, z=0) +] +center = calculate_centroid(points) +``` + +## Best Practices + +1. **Always Specify Units** + + ```python + geometry.units = Units.m # Be explicit about units + ``` + +2. **Use Chunking for Large Data** + + ```python + class BigMesh(Base, chunkable={"vertices": 1000}): + vertices: List[float] = None + ``` + +3. **Group Related Geometry** + + ```python + class Assembly(Base, detachable={"parts"}): + parts: List[Base] = None + ``` + +4. **Handle Geometric Operations Externally** + + ```python + # Use specialized libraries for geometric operations + import numpy as np + + def transform_point(point: Point, matrix: np.ndarray) -> Point: + coords = np.array([point.x, point.y, point.z, 1]) + transformed = matrix @ coords + return Point( + x=float(transformed[0]), + y=float(transformed[1]), + z=float(transformed[2]) + ) + ``` + +5. **Validate Geometry As Needed** + + ```python + def validate_mesh(mesh: Mesh) -> bool: + # Implement your validation using appropriate + # geometric libraries - specklepy won't do this + pass + ``` + +## Common Patterns + +### 1. Geometry Conversion + +When working with other geometric libraries: + +```python +def to_numpy_points(speckle_points: List[Point]) -> np.ndarray: + """Convert Speckle points to numpy array""" + return np.array([ + [p.x, p.y, p.z] for p in speckle_points + ]) + +def from_numpy_points(np_points: np.ndarray) -> List[Point]: + """Convert numpy points to Speckle points""" + return [ + Point(x=float(p[0]), y=float(p[1]), z=float(p[2])) + for p in np_points + ] +``` + +### 2. Geometry Collections + +Organizing geometry hierarchically: + +```python +class Level(Base, detachable={"geometry"}): + """A building level with geometry""" + elevation: float = 0.0 + height: float = 0.0 + geometry: List[Base] = None + +class Building(Base, detachable={"levels"}): + """A building with multiple levels""" + levels: List[Level] = None +``` + +### 3. Display Meshes + +Providing visualization geometry: + +```python +class ComplexGeometry(Base): + """Geometry with visualization mesh""" + # Your complex geometry definition + displayValue: Optional[List[Mesh]] = None + + def create_display_mesh(self): + """ + Create a simplified mesh for display + Note: You'd need to implement this using + appropriate geometric libraries + """ + pass +``` + +By understanding specklepy's role and limitations, you can effectively integrate it into workflows for defining, sharing, and transporting geometric data. diff --git a/dev/py-howto-instances.md b/dev/py-howto-instances.md new file mode 100644 index 00000000..310ab685 --- /dev/null +++ b/dev/py-howto-instances.md @@ -0,0 +1,223 @@ +# How-to: Instances, Definitions and Transforms + +## Block Definitions and Instances + +Block definitions are reusable object templates that can be instantiated multiple times in different locations with different transforms. This pattern is useful for repeating elements like furniture, fixtures, or structural components. + +### Creating Block Definitions + +```python +from specklepy.objects.other import BlockDefinition +from specklepy.objects.geometry import Point + +# Create a block definition for a table +table_def = BlockDefinition( + name="Office Table", + basePoint=Point(x=0, y=0, z=0), # Origin point for the definition + geometry=[ + # Add geometry objects that make up your table + # e.g., surfaces, curves, meshes, etc. + ] +) +``` + +### Creating Instances + +```python +from specklepy.objects.other import BlockInstance +from specklepy.objects.other import Transform + +# Create an instance of the table +table_instance = BlockInstance( + transform=Transform(), # Default identity transform + definition=table_def # Reference to the block definition +) +``` + +## Working with Transforms + +Transforms represent 4x4 transformation matrices that can be applied to instances or geometry. They combine rotation, translation, and scaling operations. + +### Creating Transforms + +```python +from specklepy.objects.other import Transform + +# Identity transform (no change) +identity = Transform() +identity.matrix = [ + 1.0, 0.0, 0.0, 0.0, # First row + 0.0, 1.0, 0.0, 0.0, # Second row + 0.0, 0.0, 1.0, 0.0, # Third row + 0.0, 0.0, 0.0, 1.0 # Fourth row +] + +# Translation transform (move 10 units in x) +translation = Transform() +translation.matrix = [ + 1.0, 0.0, 0.0, 10.0, + 0.0, 1.0, 0.0, 0.0, + 0.0, 0.0, 1.0, 0.0, + 0.0, 0.0, 0.0, 1.0 +] +``` + +### Applying Transforms to Points + +```python +from specklepy.objects.geometry import Point + +# Create a point +point = Point(x=1, y=2, z=3) + +# Create and apply transform +transform = Transform() +transform.matrix = [ + 1.0, 0.0, 0.0, 5.0, # Translation of 5 units in x + 0.0, 1.0, 0.0, 0.0, + 0.0, 0.0, 1.0, 0.0, + 0.0, 0.0, 0.0, 1.0 +] + +# Transform the point +transformed_point = transform.apply_to_point(point) +print(transformed_point.x) # Will be 6.0 (original 1 + translation 5) +``` + +### Multiple Instances with Different Transforms + +```python +# Create multiple instances of the same definition with different transforms +def create_grid_of_tables(rows: int, cols: int, spacing: float) -> List[BlockInstance]: + instances = [] + + for row in range(rows): + for col in range(cols): + # Create transform for this position + transform = Transform() + transform.matrix = [ + 1.0, 0.0, 0.0, col * spacing, + 0.0, 1.0, 0.0, row * spacing, + 0.0, 0.0, 1.0, 0.0, + 0.0, 0.0, 0.0, 1.0 + ] + + # Create instance + instance = BlockInstance( + transform=transform, + definition=table_def + ) + instances.append(instance) + + return instances + +# Create a 3x3 grid of tables spaced 2 units apart +table_grid = create_grid_of_tables(rows=3, cols=3, spacing=2.0) +``` + +## Real-World Example: Office Layout + +Here's a complete example showing how to create a reusable office furniture layout: + +```python +from specklepy.objects.base import Base +from specklepy.objects.other import BlockDefinition, BlockInstance, Transform +from specklepy.objects.geometry import Point +from typing import List + +class OfficeLayout(Base): + """Container for office furniture layout""" + furniture_definitions: List[BlockDefinition] = [] + furniture_instances: List[BlockInstance] = [] + + def add_definition(self, definition: BlockDefinition): + """Add a new furniture definition""" + self.furniture_definitions.append(definition) + + def place_furniture( + self, + definition: BlockDefinition, + x: float, + y: float, + rotation: float = 0.0 + ) -> BlockInstance: + """Place a piece of furniture at the specified location""" + # Create transform for position and rotation + transform = Transform() + # Note: This is a simplified transform. In practice, + # you'd want to build a proper rotation matrix + transform.matrix = [ + 1.0, 0.0, 0.0, x, + 0.0, 1.0, 0.0, y, + 0.0, 0.0, 1.0, 0.0, + 0.0, 0.0, 0.0, 1.0 + ] + + # Create and store instance + instance = BlockInstance( + transform=transform, + definition=definition + ) + self.furniture_instances.append(instance) + return instance + +# Usage example: +if __name__ == "__main__": + # Create office layout + office = OfficeLayout() + + # Create furniture definitions + desk = BlockDefinition( + name="Standard Desk", + basePoint=Point(x=0, y=0, z=0), + geometry=[] # Add actual desk geometry here + ) + + chair = BlockDefinition( + name="Office Chair", + basePoint=Point(x=0, y=0, z=0), + geometry=[] # Add actual chair geometry here + ) + + # Add definitions to layout + office.add_definition(desk) + office.add_definition(chair) + + # Place furniture + for i in range(3): # Create three workstations + x = i * 2.0 # Space desks 2 units apart + + # Place desk + office.place_furniture(desk, x, 0.0) + + # Place chair + office.place_furniture(chair, x, 0.5) # Chair slightly behind desk +``` + +## Best Practices + +1. **Block Definition Tips** + - Keep definitions simple and reusable + - Use meaningful names + - Position geometry relative to a logical base point + - Document any assumptions about orientation + +2. **Transform Tips** + - Initialize with identity matrix when unsure + - Test transforms with simple points first + - Remember matrix multiplication order matters + - Keep track of your coordinate system conventions + +3. **Instance Tips** + - Use instances for repeating elements + - Group related instances logically + - Consider using container classes for organization + - Track relationships between instances if needed + +4. **Performance Considerations** + - Reuse definitions when possible + - Batch create instances for better performance + - Use detaching for large collections of instances + - Consider chunking for large transform arrays + +Remember: The instance/definition pattern is powerful for managing repeating elements while keeping file sizes manageable. Each definition is stored once, with instances just referencing the definition and specifying a transform. diff --git a/dev/py-howto-realtime.md b/dev/py-howto-realtime.md new file mode 100644 index 00000000..a63f3e59 --- /dev/null +++ b/dev/py-howto-realtime.md @@ -0,0 +1,373 @@ +# How-to: Implement Real-time Subscriptions + +Real-time subscriptions in specklepy allow you to monitor changes in your projects, models, and versions as they happen. This feature is particularly useful for creating automated workflows, keeping team members informed, and maintaining coordination between different aspects of your project. + +## Understanding Subscription Types + +specklepy supports several types of subscriptions: + +1. Project Updates - Track when projects are modified or deleted +2. Model Updates - Monitor when models are created, changed, or removed +3. Version Updates - Get notified about new versions being created or modified +4. User Project Updates - Follow changes to all projects a user has access to + +## Basic Setup + +First, let's set up your client and authentication. This is the foundation for all subscription types: + +```python +from specklepy.api.client import SpeckleClient +import asyncio + +# Initialize client +client = SpeckleClient(host="speckle.xyz") +client.authenticate_with_token("your-token") + +# All subscriptions use async/await because they maintain +# a persistent connection to listen for updates +async def main(): + # Subscription code will go here + pass + +# Run your subscription +asyncio.run(main()) +``` + +This setup creates an authenticated connection to your Speckle server. The async/await pattern is used because subscriptions maintain a persistent WebSocket connection to receive real-time updates. + +## Monitoring Project Changes + +This example shows how to track changes to a specific project. You'll be notified when the project is updated or deleted: + +```python +async def monitor_project(): + async def on_update(message): + # The message object contains three key pieces of information: + # - id: The project ID + # - type: The type of change (DELETED or UPDATED) + # - project: The updated project data (None if deleted) + update_type = message.type + + if message.project: # Check if project exists (will be None if deleted) + print(f"Project '{message.project.name}' was {update_type}") + print(f"Updated at: {message.project.updatedAt}") + # You can access other project properties like: + # - message.project.description + # - message.project.visibility + # - message.project.role + + await client.subscription.project_updated( + callback=on_update, + id="your-project-id" # The ID of the project you want to monitor + ) +``` + +This subscription is useful for keeping track of project-level changes such as name updates, description changes, or project deletion. + +## Tracking Model Changes + +This subscription monitors changes to models within a project. It's particularly useful for tracking when new models are added or existing ones are modified: + +```python +async def monitor_models(): + async def on_model_update(message): + # The message contains: + # - id: The project ID + # - type: CREATED, UPDATED, or DELETED + # - model: The model data (None if deleted) + update_type = message.type + + if message.model: # Check if model exists (will be None if deleted) + # Access detailed information about the model + print(f"Model '{message.model.name}' was {update_type}") + print(f"By user: {message.model.author.name}") + print(f"At: {message.model.updatedAt}") + + # You can also access: + # - message.model.description + # - message.model.createdAt + # - message.model.id + + await client.subscription.project_models_updated( + callback=on_model_update, + id="your-project-id", + # Optionally specify which models to monitor + # If not provided, monitors all models in the project + model_ids=["model-1-id", "model-2-id"] + ) +``` + +This type of subscription is essential for teams working on multiple models within a project, helping to maintain awareness of changes across different disciplines or aspects of the project. + +## Version Tracking + +Version tracking lets you monitor when new versions are created or existing versions are modified. This is crucial for tracking the evolution of your models: + +```python +async def monitor_versions(): + async def on_version_update(message): + # The message contains: + # - id: The project ID + # - modelId: The specific model being versioned + # - type: CREATED, UPDATED, or DELETED + # - version: The version data (None if deleted) + update_type = message.type + + if message.version: # Check if version exists (will be None if deleted) + # Get detailed information about the new version + print(f"Version created for model {message.modelId}") + print(f"Message: {message.version.message}") + print(f"Author: {message.version.authorUser.name}") + + # Additional version information available: + # - message.version.referencedObject (the data object ID) + # - message.version.createdAt + # - message.version.sourceApplication + # - message.version.previewUrl + + await client.subscription.project_versions_updated( + callback=on_version_update, + id="your-project-id" + ) +``` + +Version tracking is particularly useful for maintaining a history of changes and understanding who made what changes and when. + +## User Project Updates + +This subscription type allows you to monitor all projects a user has access to, notifying you when projects are added or removed from their access: + +```python +async def monitor_user_projects(): + async def on_project_update(message): + # The message contains: + # - id: The project ID + # - type: ADDED or REMOVED + # - project: The project data (None if removed) + update_type = message.type + + if message.project: # Check if project exists (will be None if removed) + print(f"Project '{message.project.name}' was {update_type}") + print(f"Role: {message.project.role}") + + # Additional project information: + # - message.project.description + # - message.project.visibility + # - message.project.createdAt + # - message.project.updatedAt + + await client.subscription.user_projects_updated( + callback=on_project_update + ) +``` + +This subscription is valuable for maintaining an up-to-date list of accessible projects and monitoring changes in project access permissions. + +## Combining Multiple Subscriptions + +In real-world applications, you often need to monitor multiple aspects simultaneously. Here's how to combine different subscriptions: + +```python +async def monitor_everything(project_id: str): + # Create all monitoring tasks + # asyncio.gather allows us to run multiple coroutines concurrently + tasks = [ + monitor_project(), + monitor_models(), + monitor_versions(), + monitor_user_projects() + ] + + # Run all monitoring tasks concurrently + await asyncio.gather(*tasks) + +# Run the complete monitoring system +if __name__ == "__main__": + asyncio.run(monitor_everything("your-project-id")) +``` + +This pattern allows you to maintain multiple subscriptions efficiently without blocking each other. + +## Practical Applications + +### 1. Automated Build System + +This example shows how to create an automated build system that triggers when specific versions are created: + +```python +async def build_monitor(): + async def on_version_created(message): + # Check if this is a version we should build + # Looking for versions tagged with "build-ready" + if message.version and "build-ready" in message.version.message.lower(): + # Get the ID of the actual data object + obj_id = message.version.referencedObject + + # Trigger the build process + await trigger_build(obj_id) + + await client.subscription.project_versions_updated( + callback=on_version_created, + id="your-project-id" + ) + +async def trigger_build(obj_id: str): + # Here you would implement your specific build process + # For example: + # 1. Download the object data + # 2. Process it according to your needs + # 3. Generate build outputs + print(f"Starting build for object: {obj_id}") +``` + +This pattern is useful for automating processes like generating documentation, running analyses, or creating deliverables whenever specific changes occur. + +### 2. Team Notifications + +Keep your team informed of changes by integrating with notification systems: + +```python +async def team_notifications(): + async def notify_team(message): + if not message.model: + return + + # Create a structured notification object + notification = { + "title": f"Model Update: {message.model.name}", + "author": message.model.author.name, + "time": message.model.updatedAt, + "type": message.type + } + + # Send to your notification system + await send_notification(notification) + + await client.subscription.project_models_updated( + callback=notify_team, + id="your-project-id" + ) + +async def send_notification(notification: dict): + # Implement your notification logic here + # This could integrate with: + # - Slack + # - Email + # - MS Teams + # - Custom notification systems + print(f"Notification: {notification}") +``` + +This pattern helps maintain team awareness and coordination by automatically notifying relevant team members of important changes. + +### 3. Coordination Checker + +This example monitors for potential coordination issues by detecting when related models are updated close together in time: + +```python +async def coordination_monitor(): + # Keep track of recent updates to detect potential conflicts + recent_updates = {} + + async def check_coordination(message): + if not message.model: + return + + model_id = message.model.id + # Store update information + recent_updates[model_id] = { + "time": message.model.updatedAt, + "author": message.model.author.name + } + + # Check for related models updated recently + for other_id, other_update in recent_updates.items(): + if other_id != model_id: + # Calculate time between updates + time_diff = message.model.updatedAt - other_update["time"] + # Alert if updates are within an hour of each other + if time_diff.total_seconds() < 3600: # One hour + print(f"Coordination Alert: Models updated within 1 hour") + print(f"Model 1: {message.model.name} by {message.model.author.name}") + print(f"Model 2: {other_id} by {other_update['author']}") + # You might want to: + # - Send notifications to team leads + # - Flag for review + # - Create coordination tasks + + await client.subscription.project_models_updated( + callback=check_coordination, + id="your-project-id" + ) +``` + +This pattern helps prevent coordination issues by alerting team members when multiple related models are being modified in close succession. + +## Error Handling + +Implement robust error handling to ensure your subscriptions remain active even when problems occur: + +```python +async def resilient_monitor(): + while True: + try: + # Attempt to maintain the subscription + await client.subscription.project_models_updated( + callback=handle_update, + id="your-project-id" + ) + except Exception as e: + # Handle any errors that occur + print(f"Subscription error: {e}") + print("Reconnecting in 5 seconds...") + # Wait before attempting to reconnect + await asyncio.sleep(5) + # The loop will automatically retry the subscription + +async def handle_update(message): + try: + # Your update handling code + pass + except Exception as e: + # Handle errors in the update processing + print(f"Error handling update: {e}") + # Log error but don't crash the subscription +``` + +This error handling pattern ensures your subscriptions remain robust and self-healing. + +## Best Practices + +1. **Error Handling** + - Always implement error recovery mechanisms + - Keep subscriptions running despite temporary failures + - Log issues for debugging purposes + - Use try/except blocks around callback processing + +2. **Resource Management** + - Limit the number of simultaneous subscriptions + - Clean up subscriptions when they're no longer needed + - Monitor memory usage in long-running subscriptions + - Consider implementing reconnection strategies + +3. **Update Processing** + - Keep callback processing quick and efficient + - Move heavy processing work to separate tasks + - Don't block the subscription loop with long operations + - Consider using queues for processing updates + +4. **Security** + - Validate all incoming data before processing + - Check permissions where appropriate + - Don't expose sensitive information in logs or notifications + - Be cautious with automated actions based on updates + +Remember that subscriptions maintain persistent connections that need to be managed carefully. Plan for: + +- Network interruptions +- Server disconnects +- Reconnection strategies +- Error recovery +- Proper resource cleanup + +This will help you create robust and reliable real-time monitoring systems. diff --git a/dev/py-howto-work-with-large-data.md b/dev/py-howto-work-with-large-data.md new file mode 100644 index 00000000..5eab9353 --- /dev/null +++ b/dev/py-howto-work-with-large-data.md @@ -0,0 +1,291 @@ +# How-to: Work with Large Datasets in specklepy + +When working with building data, the key to success is organizing your data effectively. Rather than creating monolithic models, you'll want to: + +- Split data into logical, discipline-specific models +- Enable independent loading and editing of components +- Support efficient collaboration +- Manage memory and network resources + +This guide will show you how to structure and handle large datasets effectively in specklepy. + +## Model Organization and Federation + +### Why Split Models? + +Instead of creating one massive model containing everything, break your project into logical, discipline-specific models: + +```python +from specklepy.core.api.inputs.model_inputs import CreateModelInput + +# Create separate models for different disciplines +architecture = client.model.create( + input=CreateModelInput( + name="Architectural Design", + projectId=project_id, + description="Primary architectural elements and spaces" + ) +) + +structure = client.model.create( + input=CreateModelInput( + name="Structural Design", + projectId=project_id, + description="Primary and secondary structural systems" + ) +) + +mep = client.model.create( + input=CreateModelInput( + name="MEP Systems", + projectId=project_id, + description="Mechanical, electrical, and plumbing" + ) +) +``` + +Benefits of this approach: + +- Teams can work independently +- Faster loading and processing +- Easier version control +- More efficient updates +- Federation handled by viewers or host applications + +## Detaching: Essential for Large Models + +Detaching is your primary tool for managing large datasets. It lets you: + +- Load components independently +- Share common elements +- Manage memory efficiently +- Support collaborative workflows + +### When to Use Detaching + +Use detaching for: + +- Collections of Base objects +- Shared or reusable components +- Large sub-assemblies +- Data that might be accessed independently + +```python +from specklepy.objects.base import Base + +class ArchitecturalModel(Base, detachable={"levels", "curtainSystems", "furniture"}): + """Building model split into logical parts""" + levels: List[Base] = None # Each floor as separate object + curtainSystems: List[Base] = None # Facade systems + furniture: List[Base] = None # Furniture types (shared/repeated) + +class StructuralModel(Base, detachable={"foundation", "framingSystem", "connections"}): + """Structural model with detachable components""" + foundation: List[Base] = None # Foundation elements + framingSystem: List[Base] = None # Primary structure + connections: List[Base] = None # Connection details +``` + +### Using Detaching Effectively + +Consider access patterns when deciding what to detach: + +```python +class Level(Base, detachable={"spaces", "walls", "fixtures"}): + """A building level with independently loadable parts""" + spaces: List[Base] = None # Rooms and areas + walls: List[Base] = None # Wall elements + fixtures: List[Base] = None # Fixed equipment + + # Keep small, frequently accessed data attached + levelNumber: int = None + elevation: float = None + name: str = None +``` + +### Shared Components + +Detaching is particularly useful for repeated elements: + +```python +class FurnitureLibrary(Base, detachable={"types"}): + """Library of furniture types used across the project""" + types: List[Base] = None # Each furniture type stored once + +class Space(Base): + """A space that references furniture types""" + furnitureInstances: List[str] = None # References to furniture types +``` + +## Working with Large Lists (Chunking) + +Chunking is a specialized tool specifically for large lists of primitive values (numbers, strings). It's not for lists of Base objects - use detaching for those instead. + +### When to Use Chunking + +Only use chunking for: + +- Point cloud coordinates +- Mesh vertices/faces +- Analysis results (numeric arrays) +- Other large arrays of primitive values + +```python +class PointCloud(Base, chunkable={"points": 1000, "colors": 1000}): + """Point cloud data with chunked primitive arrays""" + points: List[float] = None # XYZ coordinates + colors: List[int] = None # RGB values + + # Don't chunk these - they're not primitive lists + metadata: dict = None + sections: List[Base] = None # Use detaching instead +``` + +### What Not to Chunk + +❌ Don't use chunking for: + +```python +# WRONG: Don't chunk lists of Base objects +class Building(Base, chunkable={"floors": 10}): # No! Use detaching instead + floors: List[Base] = None + +# RIGHT: Use detaching for Base objects +class Building(Base, detachable={"floors"}): + floors: List[Base] = None +``` + +## Transport Strategies for Large Models + +When working with multiple models and detached components: + +```python +from specklepy.api import operations +from specklepy.transports.server import ServerTransport +from specklepy.transports.sqlite import SQLiteTransport + +# Set up transports +local = SQLiteTransport( + scope="ProjectA", + name="StructuralDesign" # Organize by discipline +) + +server = ServerTransport(stream_id="your-stream") + +# Send structural model +operations.send( + base=structural_model, + transports=[local, server] +) + +# Send architectural model separately +operations.send( + base=architectural_model, + transports=[local, server] +) +``` + +## Best Practices for Large Projects + +1. **Model Organization** + - Split by discipline + - Create logical sub-models + - Use federation rather than massive single models + - Consider access patterns + +2. **Detaching Strategy** + - Detach large collections of Base objects + - Detach shared/reused components + - Keep related data together + - Consider collaborative workflows + +3. **Data Access Patterns** + - Load only what's needed + - Use references for shared data + - Cache frequently accessed components + - Plan for incremental loading + +4. **Memory Management** + + ```python + def process_large_model(model_id, project_id): + """Process a large model in parts""" + # Get model data + model = client.model.get( + model_id=model_id, + project_id=project_id + ) + + # Process each level independently + for level in model.levels: + # Load level data + level_data = operations.receive( + obj_id=level.id, + remote_transport=transport + ) + + # Process level + process_level(level_data) + + # Clear memory + del level_data + ``` + +5. **Collaboration Workflow** + - Use separate models per discipline + - Coordinate through versions + - Share references not copies + - Plan update strategies + +## Example: Large Project Structure + +Here's a typical large project organization: + +```python +# Create models for each discipline +models = { + "arch": client.model.create( + input=CreateModelInput( + name="100-Architectural", + projectId=project_id + ) + ), + "struct": client.model.create( + input=CreateModelInput( + name="200-Structural", + projectId=project_id + ) + ), + "mep": client.model.create( + input=CreateModelInput( + name="300-MEP", + projectId=project_id + ) + ) +} + +# Track relationships between models +def update_model_version(model_id, object_id, dependencies): + """Create new version with tracked dependencies""" + message = "Model Update\n\nDependent Models:\n" + + for dep in dependencies: + message += f"- {dep['name']}: Version {dep['version']}\n" + + client.version.create( + input=CreateVersionInput( + objectId=object_id, + modelId=model_id, + projectId=project_id, + message=message + ) + ) +``` + +Remember: + +- Split large projects into discipline models +- Use detaching for Base object collections +- Only use chunking for primitive value lists +- Plan for federation in viewers/applications +- Consider team workflows and access patterns diff --git a/dev/py-howto-work-with-transports.md b/dev/py-howto-work-with-transports.md new file mode 100644 index 00000000..15b1faab --- /dev/null +++ b/dev/py-howto-work-with-transports.md @@ -0,0 +1,289 @@ +# How-to: Work with Transports + +## What are Transports? + +Transports are Speckle's way of moving data between different locations. Think of them as specialized delivery services - each type of transport is optimized for a specific kind of "journey" your data might need to take: + +- From your application to your local computer +- From your computer to a Speckle server +- Between different applications +- Into temporary memory for testing + +## Why Do We Need Different Transports? + +Different scenarios require different approaches to handling data: + +- **Local Storage**: When you need fast access to data you use frequently +- **Server Communication**: When sharing data with others or storing it long-term +- **Memory Storage**: When running tests or doing temporary operations + +## Available Transport Types + +### 1. SQLiteTransport + +This is your local storage system. It saves data on your computer in a SQLite database, making it fast to access and reliable. + +```python +from specklepy.transports.sqlite import SQLiteTransport + +# Basic setup - stores data in default location +transport = SQLiteTransport() + +# Custom setup - store data where you want it +transport = SQLiteTransport( + base_path="C:/MySpeckleData", # Where to store the data + scope="ProjectA", # Organize data by project + name="Design Phase 1" # Give it a meaningful name +) +``` + +When to use SQLiteTransport: + +- ✅ Caching frequently used data +- ✅ Working offline +- ✅ Need fast access to data +- ❌ Sharing data with others +- ❌ Long-term cloud storage + +### 2. ServerTransport + +This transport handles communication with Speckle servers. It's like a courier service for your data - packaging it up, sending it over the internet, and making sure it arrives safely. + +```python +from specklepy.api.wrapper import StreamWrapper +from specklepy.transports.server import ServerTransport + +# Recommended way - StreamWrapper handles authentication automatically +wrapper = StreamWrapper("https://speckle.xyz/projects/3073b96e86") +transport = wrapper.get_transport() + +# Alternative manual setup +transport = ServerTransport( + stream_id="3073b96e86", + token="your-token", + url="https://speckle.xyz" +) +``` + +When to use ServerTransport: + +- ✅ Sharing data with team members +- ✅ Long-term storage of projects +- ✅ Collaboration across different locations +- ❌ Need super fast access +- ❌ Working completely offline + +### 3. MemoryTransport + +This transport keeps data in your computer's memory (RAM). It's fast but temporary - everything is lost when your program ends. + +```python +from specklepy.transports.memory import MemoryTransport + +transport = MemoryTransport() +``` + +When to use MemoryTransport: + +- ✅ Running tests +- ✅ Temporary data processing +- ✅ Need extremely fast access +- ❌ Need to store data permanently +- ❌ Working with large datasets + +## Sending Data: How It Works + +When you send data using transports, several things happen behind the scenes: + +1. Your data is serialized (converted into a format that can be stored/transmitted) +2. The transport checks if it needs to break large data into chunks +3. The data is stored or transmitted according to the transport type + +Here's how to send data: + +```python +from specklepy.api import operations +from specklepy.objects.base import Base + +# Create some data +building = Base() +building.height = 100 +building.floors = 25 + +# Send using multiple transports (recommended) +obj_id = operations.send( + base=building, + transports=[ + SQLiteTransport(), # Local cache for fast access + server_transport # Server storage for sharing + ] +) + +# The returned obj_id is like a tracking number - +# you can use it to retrieve your data later +``` + +## Receiving Data: The Process + +When receiving data, Speckle tries to be smart about it: + +1. First checks local storage (faster) +2. If not found locally, fetches from server +3. Automatically caches server data locally for future use + +```python +# Basic receive operation +received_building = operations.receive( + obj_id="abc123", + remote_transport=server_transport +) + +# Speckle will cache this data locally so next time it's faster +print(received_building.height) # 100 +``` + +## Common Scenarios and Solutions + +### Scenario 1: Working with Large Models + +When dealing with large architectural or engineering models: + +```python +# Increase batch size for large models +transport = ServerTransport( + stream_id="your-stream", + max_batch_size_mb=20 # Default is 10MB +) + +# Use local cache to avoid repeated downloads +sqlite_transport = SQLiteTransport() + +obj = operations.receive( + obj_id="large-model-id", + remote_transport=transport, + local_transport=sqlite_transport +) +``` + +### Scenario 2: Collaborative Project + +When multiple team members are working on the same project: + +```python +# Set up project access +wrapper = StreamWrapper("https://speckle.xyz/projects/team-project") +transport = wrapper.get_transport() + +# Each team member can send updates +obj_id = operations.send( + base=my_design_update, + transports=[transport] +) + +# Team members can receive updates +latest_design = operations.receive( + obj_id=obj_id, + remote_transport=transport +) +``` + +### Scenario 3: Offline Work + +When you need to work without internet access: + +```python +# Set up local storage +local = SQLiteTransport( + scope="OfflineWork", + name="ProjectBackup" +) + +# Store everything locally +operations.send(base=my_work, transports=[local]) + +# Later, when online, sync with server +server_transport = wrapper.get_transport() +operations.send( + base=my_work, + transports=[local, server_transport] +) +``` + +## Best Practices + +1. **Always Use Local Cache** + - Faster access to data + - Works offline + - Reduces server load + + ```python + operations.send( + base=obj, + transports=[SQLiteTransport(), server_transport] + ) + ``` + +2. **Handle Errors Gracefully** + + ```python + try: + result = operations.send(base=obj, transports=[transport]) + except SpeckleException as ex: + print(f"Failed to send: {ex}") + # Implement retry logic or user notification + ``` + +3. **Clean Up Resources** + + ```python + # Always close transports when done + transport.close() + ``` + +4. **Use Meaningful Names** + + ```python + # Good naming helps with organization + transport = SQLiteTransport( + scope="ProjectPhase2", + name="StructuralAnalysis" + ) + ``` + +## Common Issues and Solutions + +### "Object Not Found" Errors + +```python +# Check if objects exist before trying to receive +has_objects = transport.has_objects([obj_id]) +if not all(has_objects.values()): + print("Some objects are missing!") +``` + +### Performance Issues + +```python +# Use batch operations for multiple objects +transport.begin_write() +for obj in many_objects: + transport.save_object(obj.id, obj_data) +transport.end_write() +``` + +### Connection Problems + +```python +# Implement retry logic +max_retries = 3 +for attempt in range(max_retries): + try: + obj = operations.receive(obj_id, transport) + break + except SpeckleException: + if attempt == max_retries - 1: + raise + time.sleep(1 * (attempt + 1)) +``` + +Remember: Transports are a fundamental part of how Speckle moves data around. Understanding how they work and when to use each type will help you build more efficient and reliable applications.