From eba6194c3fdf21c0be73951a029f2a793671489d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 6 Sep 2024 20:48:44 +0000 Subject: [PATCH 01/14] [Changelog CI] Add Changelog for Version 0.1.2 --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..7772c8d --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,3 @@ +# Version: 0.1.2 + +* [#1](https://github.com/ConductionNL/openregister/pull/1): Create openregister.csr From e8aab6cfe84555d8753bf2c4432d2002e448ed88 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Mon, 27 Jan 2025 15:40:03 +0000 Subject: [PATCH 02/14] Bump version to 0.1.41 [skip ci] --- appinfo/info.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appinfo/info.xml b/appinfo/info.xml index 6d7e7de..b8f0208 100644 --- a/appinfo/info.xml +++ b/appinfo/info.xml @@ -22,7 +22,7 @@ Create a [bug report](https://github.com/OpenRegister/.github/issues/new/choose) Create a [feature request](https://github.com/OpenRegister/.github/issues/new/choose) ]]> - 0.1.40 + 0.1.41 agpl Conduction OpenRegister From 35c1c7722756ae4419a3871351c65fb2e9087380 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Thu, 30 Jan 2025 14:25:09 +0000 Subject: [PATCH 03/14] Bump version to 0.1.42 [skip ci] --- appinfo/info.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appinfo/info.xml b/appinfo/info.xml index b8f0208..c4f099c 100644 --- a/appinfo/info.xml +++ b/appinfo/info.xml @@ -22,7 +22,7 @@ Create a [bug report](https://github.com/OpenRegister/.github/issues/new/choose) Create a [feature request](https://github.com/OpenRegister/.github/issues/new/choose) ]]> - 0.1.41 + 0.1.42 agpl Conduction OpenRegister From 93d8007ff3d5a676b166e8b2c95839d4314eb0d5 Mon Sep 17 00:00:00 2001 From: Ruben van der Linde Date: Tue, 4 Feb 2025 20:00:52 +0100 Subject: [PATCH 04/14] Lets document the shizle out of this --- README.md | 62 ++++++++++++++++++++++++++ docs/access-control.md | 33 ++++++++++++++ docs/audit-trails.md | 46 ++++++++++++++++++++ docs/automatic-facets.md | 33 ++++++++++++++ docs/content-search.md | 33 ++++++++++++++ docs/elasticsearch.md | 33 ++++++++++++++ docs/file-attachments.md | 33 ++++++++++++++ docs/metadata.md | 34 +++++++++++++++ docs/object-locking.md | 33 ++++++++++++++ docs/object-relations.md | 33 ++++++++++++++ docs/object-storage.md | 86 +++++++++++++++++++++++++++++++++++++ docs/register-management.md | 33 ++++++++++++++ docs/schema-import.md | 74 +++++++++++++++++++++++++++++++ docs/schema-validation.md | 33 ++++++++++++++ docs/soft-deletes.md | 33 ++++++++++++++ docs/time-travel.md | 34 +++++++++++++++ 16 files changed, 666 insertions(+) create mode 100644 docs/access-control.md create mode 100644 docs/audit-trails.md create mode 100644 docs/automatic-facets.md create mode 100644 docs/content-search.md create mode 100644 docs/elasticsearch.md create mode 100644 docs/file-attachments.md create mode 100644 docs/metadata.md create mode 100644 docs/object-locking.md create mode 100644 docs/object-relations.md create mode 100644 docs/object-storage.md create mode 100644 docs/register-management.md create mode 100644 docs/schema-import.md create mode 100644 docs/schema-validation.md create mode 100644 docs/soft-deletes.md create mode 100644 docs/time-travel.md diff --git a/README.md b/README.md index 4283f36..e7ae46b 100644 --- a/README.md +++ b/README.md @@ -34,3 +34,65 @@ For more detailed information, please refer to the documentation files in the `d - **lib**: Contains all the PHP code for the application. - **src**: Contains all the Vue.js code for the application. - **docs**: Contains documentation files. + +# Open Register + +Open Register is a powerful object management system for Nextcloud that helps organizations store, track, and manage objects with their associated metadata, files, and relationships. Born from the Dutch Common Ground initiative, it addresses the need for quickly deploying standardized registers based on centralized definitions from standardization organizations. + +## Background + +Open Register emerged from the Dutch Common Ground movement, which aims to modernize municipal data management. The project specifically addresses the challenge many organizations face: implementing standardized registers quickly and cost-effectively while maintaining compliance with central definitions. + +### Common Ground Principles +- Decentralized data storage +- Component-based architecture +- Standardized definitions +- API-first approach + +Open Register makes these principles accessible to any organization by providing: +- Quick register deployment based on standard schemas +- Flexible storage options +- Built-in compliance features +- Cost-effective implementation + +## Key Features + +| Feature | Description | Benefits | +|---------|-------------|-----------| +| 💾 [Object Storage](docs/object-storage.md) | Flexible storage backend selection per register | Storage flexibility, system integration, scalability | +| 📝 [Audit Trails](docs/audit-trails.md) | Complete history of all object changes | Compliance, accountability, change tracking | +| ⏰ [Time Travel](docs/time-travel.md) | View and restore previous object states | Data recovery, historical analysis, version control | +| 🔒 [Object Locking](docs/object-locking.md) | Prevent concurrent modifications | Data integrity, process management, conflict prevention | +| 🗑️ [Soft Deletes](docs/soft-deletes.md) | Safely remove objects with recovery options | Data safety, compliance, mistake recovery | +| 🔗 [Object Relations](docs/object-relations.md) | Create and manage connections between objects | Complex data structures, linked information, dependencies | +| 📎 [File Attachments](docs/file-attachments.md) | Manage files associated with objects | Document management, version control, previews | +| 🔍 [Content Search](docs/content-search.md) | Full-text search across objects and files | Quick discovery, unified search, advanced filtering | +| 🏷️ [Automatic Facets](docs/automatic-facets.md) | Dynamic filtering based on object properties | Intuitive navigation, pattern discovery, smart filtering | +| ✅ [Schema Validation](docs/schema-validation.md) | Validate objects against JSON schemas | Data quality, consistency, structure enforcement | +| 📚 [Register Management](docs/register-management.md) | Organize collections of related objects | Logical grouping, access control, process automation | +| 🔐 [Access Control](docs/access-control.md) | Fine-grained permissions management | Security, role management, granular control | +| ⚡ [Elasticsearch](docs/elasticsearch.md) | Advanced search and analytics capabilities | Performance, insights, complex queries | +| 📋 [Schema Import & Sharing](docs/schema-import.md) | Import schemas from Schema.org, OAS, GGM, and share via Open Catalogi | Standards compliance, reuse, collaboration | +| 🔔 [Events & Webhooks](docs/events.md) | React to object changes with events and webhooks | Integration, automation, real-time updates | + +## Documentation + +For detailed information about each feature, please visit our [documentation](docs/). + +## Requirements + +- Nextcloud 25 or higher +- PHP 8.1 or higher +- Database: MySQL/MariaDB + +## Installation + +[Installation instructions] + +## Support + +[Support information] + +## License + +This project is licensed under the AGPL-3.0 License - see the [LICENSE](LICENSE) file for details. diff --git a/docs/access-control.md b/docs/access-control.md new file mode 100644 index 0000000..876c841 --- /dev/null +++ b/docs/access-control.md @@ -0,0 +1,33 @@ +# Access Control + +Access Control provides fine-grained permissions management for objects and registers. + +## Overview + +The access control system provides: +- Role-based access control +- Object-level permissions +- Register-level permissions +- Dynamic permission rules + +## Key Benefits + +1. **Security** + - Granular control + - Role management + - Permission inheritance + +2. **Management** + - Easy administration + - Clear overview + - Flexible configuration + +3. **Integration** + - User management + - Group support + - Process automation + +## Related Features + +- [Register Management](register-management.md) - Manage register access +- [Object Locking](object-locking.md) - Control modifications \ No newline at end of file diff --git a/docs/audit-trails.md b/docs/audit-trails.md new file mode 100644 index 0000000..cac1d86 --- /dev/null +++ b/docs/audit-trails.md @@ -0,0 +1,46 @@ +# Audit Trails + +Audit trails provide a complete history of all changes made to objects in Open Register. This feature ensures transparency and accountability by tracking who made what changes and when. + +## Overview + +The audit trail system automatically records: +- All modifications to objects +- Who made the changes +- When changes occurred +- What specific data was changed +- The reason for changes (when provided) + +## Key Benefits + +1. **Compliance & Accountability** + - Meet regulatory requirements + - Track responsibility for changes + - Maintain data integrity records + +2. **Change Management** + - Review modification history + - Understand data evolution + - Investigate data issues + +3. **Security** + - Detect unauthorized changes + - Monitor sensitive data access + - Support security audits + +## Integration + +Audit trails are automatically enabled for all objects in Open Register. No additional configuration is required to start tracking changes. + +## Access + +Audit trails can be accessed: +- Through the object detail view +- Via the API +- Through database queries (for system administrators) + +## Related Features + +- [Time Travel](time-travel.md) - Use audit trails to restore previous states +- [Object Locking](object-locking.md) - Prevent unauthorized changes +- [Access Control](access-control.md) - Manage who can view audit trails \ No newline at end of file diff --git a/docs/automatic-facets.md b/docs/automatic-facets.md new file mode 100644 index 0000000..6d3b4a7 --- /dev/null +++ b/docs/automatic-facets.md @@ -0,0 +1,33 @@ +# Automatic Facets + +Automatic Facets provide dynamic filtering options based on object properties and metadata. + +## Overview + +The faceting system: +- Generates filters automatically +- Updates in real-time +- Supports multiple filter types +- Provides count information + +## Key Benefits + +1. **User Experience** + - Intuitive filtering + - Dynamic updates + - Clear navigation + +2. **Data Discovery** + - Quick filtering + - Pattern recognition + - Data exploration + +3. **Integration** + - Automatic generation + - Custom configuration + - Search integration + +## Related Features + +- [Content Search](content-search.md) - Enhanced search capabilities +- [Elasticsearch Integration](elasticsearch.md) - Advanced faceting \ No newline at end of file diff --git a/docs/content-search.md b/docs/content-search.md new file mode 100644 index 0000000..ea0272c --- /dev/null +++ b/docs/content-search.md @@ -0,0 +1,33 @@ +# Content Search + +Content Search provides powerful search capabilities across objects and their associated files. + +## Overview + +The search system enables: +- Full-text search +- Metadata search +- File content search +- Advanced query options + +## Key Benefits + +1. **Discovery** + - Find relevant content + - Explore relationships + - Discover patterns + +2. **Integration** + - Combined search results + - Unified interface + - Rich filtering + +3. **Performance** + - Optimized indexing + - Fast results + - Scalable search + +## Related Features + +- [Automatic Facets](automatic-facets.md) - Enhanced search filtering +- [Elasticsearch Integration](elasticsearch.md) - Advanced search capabilities \ No newline at end of file diff --git a/docs/elasticsearch.md b/docs/elasticsearch.md new file mode 100644 index 0000000..0b8e6f6 --- /dev/null +++ b/docs/elasticsearch.md @@ -0,0 +1,33 @@ +# Elasticsearch Integration + +Elasticsearch Integration provides advanced search and analytics capabilities for objects and their content. + +## Overview + +The Elasticsearch integration provides: +- Advanced full-text search +- Real-time indexing +- Complex query support +- Analytics capabilities + +## Key Benefits + +1. **Search Performance** + - Fast results + - Scalable search + - Complex queries + +2. **Analytics** + - Data insights + - Pattern recognition + - Trend analysis + +3. **Integration** + - Real-time updates + - Custom mappings + - Advanced features + +## Related Features + +- [Content Search](content-search.md) - Basic search capabilities +- [Automatic Facets](automatic-facets.md) - Enhanced filtering \ No newline at end of file diff --git a/docs/file-attachments.md b/docs/file-attachments.md new file mode 100644 index 0000000..d6d8a58 --- /dev/null +++ b/docs/file-attachments.md @@ -0,0 +1,33 @@ +# File Attachments + +File Attachments allow objects to include and manage associated files and documents. + +## Overview + +The file attachment system provides: +- Secure file storage +- Version control +- File metadata tracking +- Preview capabilities + +## Key Benefits + +1. **Document Management** + - Centralized storage + - Version tracking + - Access control + +2. **Integration** + - Preview support + - Search capabilities + - Workflow integration + +3. **Organization** + - Structured storage + - Metadata tracking + - Relationship management + +## Related Features + +- [Content Search](content-search.md) - Search file contents +- [Object Relations](object-relations.md) - Link files to objects \ No newline at end of file diff --git a/docs/metadata.md b/docs/metadata.md new file mode 100644 index 0000000..0cdb0bb --- /dev/null +++ b/docs/metadata.md @@ -0,0 +1,34 @@ +# Object Metadata + +Object Metadata provides a structured way to store and manage additional information about objects beyond their core data. + +## Overview + +The metadata system supports: +- Custom metadata fields +- Automatic system metadata +- Searchable attributes +- Version tracking + +## Key Benefits + +1. **Enhanced Organization** + - Improved searchability + - Better categorization + - Flexible classification + +2. **Process Support** + - Workflow status tracking + - Process automation + - Integration support + +3. **Data Management** + - Rich context storage + - Extended object information + - Custom attributes + +## Related Features + +- [Schema Validation](schema-validation.md) - Validate metadata +- [Content Search](content-search.md) - Search metadata +- [Automatic Facets](automatic-facets.md) - Generate facets from metadata \ No newline at end of file diff --git a/docs/object-locking.md b/docs/object-locking.md new file mode 100644 index 0000000..97ff4f4 --- /dev/null +++ b/docs/object-locking.md @@ -0,0 +1,33 @@ +# Object Locking + +Object Locking provides a mechanism to prevent concurrent modifications to objects, ensuring data integrity in multi-user environments. + +## Overview + +The locking system provides: +- Temporary exclusive access to objects +- Lock duration management +- Lock ownership tracking +- Automatic lock expiration + +## Key Benefits + +1. **Data Integrity** + - Prevent concurrent modifications + - Avoid data conflicts + - Maintain consistency + +2. **Process Management** + - Support long-running operations + - Coordinate multi-step updates + - Manage workflow dependencies + +3. **User Coordination** + - Clear ownership indication + - Transparent lock status + - Managed access control + +## Related Features + +- [Audit Trails](audit-trails.md) - Track lock operations +- [Access Control](access-control.md) - Manage lock permissions \ No newline at end of file diff --git a/docs/object-relations.md b/docs/object-relations.md new file mode 100644 index 0000000..66c04db --- /dev/null +++ b/docs/object-relations.md @@ -0,0 +1,33 @@ +# Object Relations + +Object Relations enable the creation and management of connections between objects, supporting complex data structures and relationships. + +## Overview + +The relations system provides: +- Multiple relationship types +- Bi-directional relationships +- Relationship metadata +- Integrity management + +## Key Benefits + +1. **Data Organization** + - Model complex relationships + - Maintain data connections + - Support hierarchical structures + +2. **Data Integration** + - Link related information + - Create data networks + - Support cross-referencing + +3. **Process Management** + - Track dependencies + - Manage workflows + - Support business processes + +## Related Features + +- [Schema Validation](schema-validation.md) - Validate relationships +- [Content Search](content-search.md) - Search through relations \ No newline at end of file diff --git a/docs/object-storage.md b/docs/object-storage.md new file mode 100644 index 0000000..e2b9dda --- /dev/null +++ b/docs/object-storage.md @@ -0,0 +1,86 @@ +# Object Storage + +Open Register provides flexible storage options for objects through configurable data sources per register. This allows organizations to store their data where it makes the most sense while maintaining a unified interface. + +## Overview + +The storage system supports: +- Default blob storage in Nextcloud +- External database connections +- Object storage systems +- Elasticsearch backends +- External API integrations +- Multiple sources per installation + +## Storage Options + +### Default Storage +When no source is defined, objects are stored as JSON blobs in the Nextcloud database, providing: +- Simple setup +- Built-in backup +- Direct integration +- No external dependencies + +### External Databases +Connect registers to external databases: +- MySQL/MariaDB +- PostgreSQL +- MongoDB +- Custom database adapters + +### Object Stores +Store objects in dedicated object storage: +- S3-compatible storage +- MinIO +- Azure Blob Storage +- Custom object store adapters + +### Search Engines +Use search engines as primary storage: +- Elasticsearch +- OpenSearch +- Custom search engine adapters + +### External APIs +Connect to external systems: +- REST APIs +- GraphQL endpoints +- Custom API adapters +- Other Open Register instances + +## Key Benefits + +1. **Flexibility** + - Choose optimal storage per register + - Mix different storage types + - Adapt to existing infrastructure + - Scale storage independently + +2. **Integration** + - Connect to existing systems + - Maintain data where it belongs + - Unified access interface + - Transparent to users + +3. **Performance** + - Optimize for specific use cases + - Scale according to needs + - Use specialized systems + - Maintain efficiency + +4. **Migration** + - Easy data migration + - Change sources without impact + - Test different storage options + - Gradual transitions + +## Configuration + +Sources are configured at the register level, allowing different registers to use different storage backends while maintaining a consistent interface for users and applications. + +## Related Features + +- [Register Management](register-management.md) - Configure register storage +- [Schema Validation](schema-validation.md) - Validate across storage types +- [Content Search](content-search.md) - Search across storage types +- [Access Control](access-control.md) - Unified permissions across sources \ No newline at end of file diff --git a/docs/register-management.md b/docs/register-management.md new file mode 100644 index 0000000..2384049 --- /dev/null +++ b/docs/register-management.md @@ -0,0 +1,33 @@ +# Register Management + +Register Management provides tools to organize and manage collections of related objects. + +## Overview + +The register system enables: +- Object organization +- Access control +- Process management +- Schema association + +## Key Benefits + +1. **Organization** + - Logical grouping + - Clear structure + - Process support + +2. **Management** + - Access control + - Schema enforcement + - Process automation + +3. **Integration** + - API support + - Custom workflows + - External integration + +## Related Features + +- [Access Control](access-control.md) - Manage permissions +- [Schema Validation](schema-validation.md) - Validate objects \ No newline at end of file diff --git a/docs/schema-import.md b/docs/schema-import.md new file mode 100644 index 0000000..ddc9ffd --- /dev/null +++ b/docs/schema-import.md @@ -0,0 +1,74 @@ +# Schema Import & Sharing + +Open Register provides powerful schema import capabilities, allowing organizations to leverage existing standards and share their own schemas through Open Catalogi. + +## Overview + +The schema system supports importing from: +- Schema.org definitions +- OpenAPI Specification (OAS) files +- Gemeentelijk Gegevensmodel (GGM) +- Open Catalogi +- Custom JSON Schema files + +## Import Sources + +### Schema.org +- Import standard web vocabularies +- Use established data structures +- Benefit from widespread adoption +- Maintain semantic compatibility + +### OpenAPI Specification +- Import API definitions +- Reuse existing data models +- Maintain API compatibility +- Leverage API documentation + +### Gemeentelijk Gegevensmodel (GGM) +- Import Dutch municipal data models +- Comply with government standards +- Ensure data compatibility +- Support Common Ground principles + +### Open Catalogi +- Share schemas between organizations +- Import from central repositories +- Collaborate on definitions +- Version control schemas + +## Schema Sharing + +Organizations can share their schemas through Open Catalogi: +- Publish schemas publicly +- Version control +- Collaborative development +- Change management +- Documentation +- Usage statistics + +## Key Benefits + +1. **Standardization** + - Reuse existing standards + - Ensure compatibility + - Reduce development time + - Share best practices + +2. **Collaboration** + - Share schemas + - Collaborate on definitions + - Build on existing work + - Community involvement + +3. **Maintenance** + - Central updates + - Version management + - Change tracking + - Documentation + +## Related Features + +- [Schema Validation](schema-validation.md) - Validate against imported schemas +- [Register Management](register-management.md) - Use schemas in registers +- [Object Storage](object-storage.md) - Store objects according to schemas \ No newline at end of file diff --git a/docs/schema-validation.md b/docs/schema-validation.md new file mode 100644 index 0000000..b4a08df --- /dev/null +++ b/docs/schema-validation.md @@ -0,0 +1,33 @@ +# Schema Validation + +Schema Validation ensures data integrity by validating objects against predefined JSON schemas. + +## Overview + +The validation system provides: +- JSON Schema support +- Custom validation rules +- Error reporting +- Schema management + +## Key Benefits + +1. **Data Quality** + - Enforce data structure + - Validate content + - Maintain consistency + +2. **Integration** + - Standard compliance + - API validation + - Process automation + +3. **Management** + - Schema versioning + - Custom rules + - Error handling + +## Related Features + +- [Object Relations](object-relations.md) - Validate relationships +- [Metadata](metadata.md) - Validate metadata \ No newline at end of file diff --git a/docs/soft-deletes.md b/docs/soft-deletes.md new file mode 100644 index 0000000..6c8e614 --- /dev/null +++ b/docs/soft-deletes.md @@ -0,0 +1,33 @@ +# Soft Deletes + +Soft Deletes provide a safe way to remove objects while maintaining the ability to recover them if needed. + +## Overview + +The soft delete system: +- Marks objects as deleted instead of removing them +- Maintains referential integrity +- Provides recovery options +- Supports permanent deletion when needed + +## Key Benefits + +1. **Data Safety** + - Prevent accidental data loss + - Maintain data relationships + - Support data recovery + +2. **Compliance** + - Meet retention requirements + - Support audit processes + - Manage deletion policies + +3. **Business Continuity** + - Recover from mistakes + - Maintain historical context + - Support business processes + +## Related Features + +- [Time Travel](time-travel.md) - Review deleted objects +- [Audit Trails](audit-trails.md) - Track deletion operations \ No newline at end of file diff --git a/docs/time-travel.md b/docs/time-travel.md new file mode 100644 index 0000000..6568126 --- /dev/null +++ b/docs/time-travel.md @@ -0,0 +1,34 @@ +# Time Travel + +Time Travel in Open Register allows you to view and restore objects to any previous state in their history. This powerful feature enables data recovery, audit compliance, and historical analysis. + +## Overview + +Time Travel functionality provides: +- Point-in-time recovery of object states +- Historical views of data changes +- Version comparison capabilities +- Safe restoration processes + +## Key Benefits + +1. **Data Recovery** + - Recover from accidental changes + - Restore deleted data + - Fix incorrect updates + +2. **Historical Analysis** + - Review data evolution + - Track business changes + - Analyze decision points + +3. **Compliance** + - Meet regulatory requirements + - Support audit processes + - Maintain data lineage + +## Related Features + +- [Audit Trails](audit-trails.md) - Track all changes +- [Object Locking](object-locking.md) - Prevent concurrent modifications +- [Soft Deletes](soft-deletes.md) - Safely handle deletions \ No newline at end of file From 86a6eb22ec720991f236fa7a8153fa3512288623 Mon Sep 17 00:00:00 2001 From: Ruben van der Linde Date: Tue, 4 Feb 2025 22:43:44 +0100 Subject: [PATCH 05/14] The more the marier --- README.md | 3 + docs/access-control.md | 64 +++++++---- docs/advanced-search.md | 168 +++++++++++++++++++++++++++ docs/audit-trails.md | 80 ++++++++++++- docs/data-extension.md | 243 ++++++++++++++++++++++++++++++++++++++++ docs/data-filtering.md | 86 ++++++++++++++ docs/metadata.md | 66 +++++++++-- 7 files changed, 679 insertions(+), 31 deletions(-) create mode 100644 docs/advanced-search.md create mode 100644 docs/data-extension.md create mode 100644 docs/data-filtering.md diff --git a/README.md b/README.md index e7ae46b..437f3c2 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,9 @@ Open Register makes these principles accessible to any organization by providing | ⚡ [Elasticsearch](docs/elasticsearch.md) | Advanced search and analytics capabilities | Performance, insights, complex queries | | 📋 [Schema Import & Sharing](docs/schema-import.md) | Import schemas from Schema.org, OAS, GGM, and share via Open Catalogi | Standards compliance, reuse, collaboration | | 🔔 [Events & Webhooks](docs/events.md) | React to object changes with events and webhooks | Integration, automation, real-time updates | +| 🔄 [Data Extension](docs/data-extension.md) | Automatically include related entities in responses | Efficient data retrieval, reduced API calls, complete context | +| ✂️ [Data Filtering](docs/data-filtering.md) | Select specific properties to return | Data minimalization, GDPR compliance, efficient responses | +| 🔍 [Advanced Search](docs/advanced-search.md) | Filter objects using flexible property-based queries | Precise filtering, complex conditions, efficient results | ## Documentation diff --git a/docs/access-control.md b/docs/access-control.md index 876c841..3abb72c 100644 --- a/docs/access-control.md +++ b/docs/access-control.md @@ -1,33 +1,57 @@ # Access Control -Access Control provides fine-grained permissions management for objects and registers. +Access Control provides enterprise-grade permissions management through integration with Nextcloud RBAC (Role-Based Access Control) and Keycloak. ## Overview -The access control system provides: -- Role-based access control -- Object-level permissions -- Register-level permissions -- Dynamic permission rules +The access control system integrates with: +- ADFS (Active Directory Federation Services) for user and group management via Keycloak +- Nextcloud RBAC for role-based permissions +- FCS (Federal Cloud Services) compliance requirements +- Verwerkingen registers for process tracking -## Key Benefits +## Permission Levels -1. **Security** - - Granular control - - Role management - - Permission inheritance +Access can be controlled at multiple levels: +- Register level - Control access to entire registers +- Schema level - Manage permissions for specific register/schema combinations +- Object level - Set permissions on individual objects +- Property level - Fine-grained control over specific object properties -2. **Management** - - Easy administration - - Clear overview - - Flexible configuration +## Permission Types -3. **Integration** - - User management - - Group support - - Process automation +Permissions are granted through: +1. **User Rights** + - CRUD (Create, Read, Update, Delete) operations + - Inherited from ADFS groups via Keycloak + - Role-based access control through Nextcloud + +2. **Contract Rights** + - Application-level permissions + - Process-specific authorizations + - Compliance with FCS requirements + - Integration with verwerkingen registers + +## Implementation + +Access control is implemented through: + +1. **User Authentication** + - Direct integration with Keycloak for identity management + - ADFS synchronization for user and group information + - Single Sign-On (SSO) capabilities + +2. **Permission Management** + - CRUD-level permissions for all system entities + - Hierarchical permission inheritance + - Fine-grained access control at multiple levels + +3. **Process Integration** + - Compliance with FCS guidelines + - Integration with verwerkingen registers for process tracking + - Application-specific permission contracts ## Related Features - [Register Management](register-management.md) - Manage register access -- [Object Locking](object-locking.md) - Control modifications \ No newline at end of file +- [Object Locking](object-locking.md) - Control modifications \ No newline at end of file diff --git a/docs/advanced-search.md b/docs/advanced-search.md new file mode 100644 index 0000000..ab108a5 --- /dev/null +++ b/docs/advanced-search.md @@ -0,0 +1,168 @@ +# Advanced Search + +Open Register provides powerful search capabilities that allow filtering objects based on their properties using a flexible query syntax. + +## Overview + +The search system enables you to filter objects using query parameters. However, it's important to note that this approach is limited by the maximum URL length supported by browsers and servers (typically 2,048 characters for most browsers). + +For more complex queries that exceed URL length limitations, we are planning to implement GraphQL support in the future. This would enable: + +- Deeply nested queries +- Complex filtering logic +- Precise field selection +- Batch operations +- Real-time subscriptions + +> Note: GraphQL implementation is currently pending funding. If you're interested in supporting this feature, please contact us. + +In the meantime, here are the available search capabilities using URL parameters: + +| Operator | Description | Example | +|----------|-------------|---------| +| = | Equals (case insensitive) | `name=nemo` | +| != | Not equals (case insensitive) | `name[!=]=nemo` | +| > | Greater than | `age[>]=5` | +| < | Less than | `weight[<]=10` | +| >= | Greater than or equal | `age[>=]=2` | +| <= | Less than or equal | `age[<=]=10` | +| ~ | Contains (case insensitive) | `name[~]=ne` | +| ^ | Starts with (case insensitive) | `name[^]=ne` | +| $ | Ends with (case insensitive) | `name[$]=mo` | +| === | Equals (case sensitive) | `name[===]=Nemo` | +| exists | Property exists check | `microchip[exists]=true` | +| empty | Empty value check | `notes[empty]=true` | +| null | Null value check | `owner[null]=true` | + +## Full Text Search + +The `_search` parameter allows searching across all text properties of objects in a case-insensitive way: + +``GET /api/pets?_search=nemo`` + +This searches for "nemo" in all text fields like name, description, notes etc. + +### Wildcard Search +You can use wildcards in the search term: + +- `*` matches zero or more characters +``GET /api/pets?_search=ne*o`` +Matches "nemo", "negro", "neuro" etc. + +- `?` matches exactly one character +``GET /api/pets?_search=ne?o`` +Matches "nemo", "nero" but not "neuro" + +### Pattern Matching +- `^` matches start of text +``GET /api/pets?_search=^ne`` +Matches text starting with "ne" + +- `$` matches end of text +``GET /api/pets?_search=mo$`` +Matches text ending with "mo" + +### Phrase Search +Use quotes for exact phrase matching: +``GET /api/pets?_search="orange fish"`` +Matches the exact phrase "orange fish" + +## Basic Search + +Simple equals search (case insensitive): +``GET /api/pets?name=nemo`` + +This returns all pets named "nemo", "Nemo", "NEMO", etc. + +Case sensitive search: +``GET /api/pets?name[===]=Nemo`` + +This returns only pets named exactly "Nemo". + +## Comparison Operators + +### Not Equals (!=) +``GET /api/pets?name[!=]=nemo`` +Returns all pets NOT named "nemo" (case insensitive) + +### Greater Than (>) +``GET /api/pets?age[>]=5`` +Returns pets older than 5 years + +### Less Than (<) +``GET /api/pets?weight[<]=10`` +Returns pets weighing less than 10kg + +### Greater Than or Equal (>=) +``GET /api/pets?age[>=]=2`` +Returns pets 2 years or older + +### Less Than or Equal (<=) +``GET /api/pets?age[<=]=10`` +Returns pets 10 years or younger + +### Contains (~) +``GET /api/pets?name[~]=ne`` +Returns pets with "ne" in their name (like "nemo", "nero", "Nemo", etc) - case insensitive + +### Starts With (^) +``GET /api/pets?name[^]=ne`` +Returns pets whose names start with "ne" (case insensitive) + +### Ends With ($) +``GET /api/pets?name[$]=mo`` +Returns pets whose names end with "mo" (case insensitive) + +## Combining Multiple Conditions + +### AND Operations +``GET /api/pets?name=nemo&type=fish`` +Returns pets named "nemo" (case insensitive) AND of type "fish" + +### OR Operations +``GET /api/pets?name[]=nemo&name[]=dory`` +Returns pets named either "nemo" OR "dory" (case insensitive) + +## Special Filters + +### Exists Check +``GET /api/pets?microchip[exists]=true`` +Returns pets that have a microchip property + +### Empty Check +``GET /api/pets?notes[empty]=true`` +Returns pets with empty notes + +### Null Check +``GET /api/pets?owner[null]=true`` +Returns pets with no owner + +### Between Range +``GET /api/pets?age[>=]=2&age[<=]=5`` +Returns pets between 2 and 5 years old (inclusive) + +``GET /api/pets?age[>]=2&age[<]=5`` +Returns pets between 2 and 5 years old (exclusive) + +## Searching Nested Properties + +``GET /api/pets?owner.city=Amsterdam`` +Returns pets whose owners live in Amsterdam (case insensitive) + +``GET /api/pets?vaccinations.date[>]=2023-01-01`` +Returns pets with vaccinations after January 1st, 2023 + +## Best Practices + +1. Use URL encoding for special characters +2. Keep queries focused and specific +3. Use pagination for large result sets +4. Consider URL length limitations +5. Break complex queries into multiple requests if needed + +## Related Features + +- [Automatic Facets](automatic-facets.md) - Combine with faceted search +- [Elasticsearch](elasticsearch.md) - Advanced search capabilities +- [Content Search](content-search.md) - Full-text search integration +- [Access Control](access-control.md) - Security for search results \ No newline at end of file diff --git a/docs/audit-trails.md b/docs/audit-trails.md index cac1d86..73614ed 100644 --- a/docs/audit-trails.md +++ b/docs/audit-trails.md @@ -1,15 +1,93 @@ # Audit Trails -Audit trails provide a complete history of all changes made to objects in Open Register. This feature ensures transparency and accountability by tracking who made what changes and when. +Audit trails provide a complete history of all changes made to objects in Open Register. This feature ensures transparency and accountability by tracking who made what changes and when. The implementation follows the [GEMMA Processing Logging standard](https://vng-realisatie.github.io/gemma-verwerkingenlogging/gegevensmodel/basisterminologie.html). ## Overview The audit trail system automatically records: - All modifications to objects +- Individual object reads (but not collection reads) - Who made the changes - When changes occurred - What specific data was changed - The reason for changes (when provided) +- Processing activities + +## Read Action Logging + +The system only logs read actions when accessing individual objects (e.g., GET /api/objects/123). Collection reads and search operations (e.g., GET /api/objects?name=test) are intentionally not logged for several important reasons: + +1. **Performance Impact** + - Collection reads can return hundreds or thousands of objects + - Logging each object view would create massive amounts of audit data + - Database performance would degrade significantly + +2. **Storage Concerns** + - The audit log would grow exponentially + - Storage costs would increase dramatically + - Valuable audit data would be diluted with less meaningful entries + +3. **Meaningful Tracking** + - Individual object reads indicate specific interest in that object + - Collection reads are often exploratory or part of routine operations + - Focus on logging deliberate access to specific objects + +## Logged Information + +| Field | Description | Example | How Determined | +|-------|-------------|---------|----------------| +| uuid | Unique identifier for the audit entry | 550e8400-e29b-41d4-a716-446655440000 | Generated using UUID v4 | +| schema | Schema ID of the modified object | 42 | From object's schema reference | +| register | Register ID of the modified object | 123 | From object's register reference | +| object | Object ID that was modified | 456 | From modified object's ID | +| action | Type of change that occurred | create, read, update, delete | Determined by system operation | +| changed | Array of modified fields with old/new values | {"name": {"old": "John", "new": "Jane"}} | Diff between object states | +| user | ID of the user who made the change | admin | From authenticated user session | +| userName | Display name of the user | Administrator | From user profile | +| session | Session ID when change occurred | sess_89d7h2 | From current session | +| request | Request ID for tracing | req_7d8h3j | Generated per request | +| ipAddress | IP address of the request | 192.168.1.1 | From request headers | +| version | Object version after change | 1.0.0 | Incremented on each change | +| created | Timestamp of the change | 2024-03-15T14:30:00Z | Server timestamp | + +### Additional Required Fields + +To enhance audit logging, we should add: + +| Field | Description | How Determined | +|-------|-------------|----------------| +| processingActivity | The processing activity from the registry | From process registry lookup | +| processing | The specific task being performed | From application context | +| operation | The step in the processing task | From system operation | +| legalBasis | Legal basis for the processing | From process configuration | +| retentionPeriod | Retention period for the data | From schema configuration | +| executor | The system or person executing the action | From authenticated context | +| system | The system where the action occurred | From application config | +| dataSource | The source of the data | From data origin tracking | + +## Processing Logging Structure + +The audit trail system follows this hierarchical structure: + +1. **Processing Activity** + - High-level category of data processing + - Determined by: Process registry lookup + - Example: "Citizen Registration" + +2. **Processing** + - Specific task being executed + - Determined by: Application context + - Example: "Address Change Request" + +3. **Operation** + - Individual step in processing + - Determined by: System operation + - Example: "Validate Address" + +4. **Action** + - Actual system operation + - Determined by: Database operation + - Example: "Update Address Field" ## Key Benefits diff --git a/docs/data-extension.md b/docs/data-extension.md new file mode 100644 index 0000000..e79a9d8 --- /dev/null +++ b/docs/data-extension.md @@ -0,0 +1,243 @@ +# Data Extension (_extend) + +Data Extension allows you to automatically include related entities in API responses, reducing the need for multiple API calls and providing complete context in a single request. + +## Overview + +The _extend parameter provides: +- Automatic inclusion of related entities +- Support for nested relations +- Resolution of both ID and URL references +- Configurable depth limits +- Circular reference handling + +## Extension Types + +### Direct Relations +- Single object relations +- Collection relations +- External references +- URI resolutions + +### Nested Relations +- Multi-level extensions +- Depth control +- Circular detection +- Performance optimization + +### Reference Types +- Internal IDs +- External URLs +- URNs +- Custom identifiers + +## Key Benefits + +1. **Performance** + - Reduce API calls + - Optimize data retrieval + - Efficient response handling + - Bandwidth optimization + +2. **Data Completeness** + - Get full context + - Include related data + - Resolve references + - Complete object graphs + +3. **Flexibility** + - Client-driven inclusion + - Dynamic data loading + - Customizable depth + - Selective extension + +## Integration with Privacy + +- Respects access controls on related objects +- Honors data minimalization principles +- Supports GDPR compliance +- Provides audit trail integration + +## Using the objects api + +### Single Object Extension + +Consider a Dog object with a reference to its Breed: + +Request: +```` +GET /api/dogs/123 +```` + +Response: +````json +{ + "id": "550e8400-e29b-41d4-a716-446655440000", + "name": "Max", + "breed": "https://api.petstore.com/breeds/german-shepherd", + "age": 5 +} +```` + +With data extension enabled: + +Request: +```` +GET /api/dogs/123?extend=breed +```` + +Response: +````json +{ + "id": "550e8400-e29b-41d4-a716-446655440000", + "name": "Max", + "breed": { + "id": "german-shepherd", + "name": "German Shepherd", + "origin": "Germany", + "size": "Large" + }, + "age": 5 +} +```` + +### Collection Extension + +The same principle works when retrieving collections: + +Request: +```` +GET /api/dogs?extend=breed +```` + +Response: +````json +{ + "results": [ + { + "id": "550e8400-e29b-41d4-a716-446655440000", + "name": "Max", + "breed": { + "id": "german-shepherd", + "name": "German Shepherd", + "origin": "Germany", + "size": "Large" + }, + "age": 5 + }, + { + "id": "550e8400-e29b-41d4-a716-446655440001", + "name": "Bella", + "breed": { + "id": "golden-retriever", + "name": "Golden Retriever", + "origin": "Scotland", + "size": "Large" + }, + "age": 3 + } + ], + "total": 2 +} +```` + +### Extending Multiple Properties + +You can extend multiple properties in two ways: + +1. Using a comma-separated list: + +Request: +````GET /api/dogs?extend=breed,owner```` + +2. Using an array format: + +Request: +````GET /api/dogs?extend[]=breed&extend[]=owner```` + +Both approaches will produce the same result, extending multiple properties in a single request. + +### Extending Nested Properties + +You can extend properties of already extended objects using dot notation. This allows you to access nested data structures. + +For example, to extend the breed's parent breed information: + +Request: +````GET /api/dogs?extend=breed.parent_breed```` + +Response: +````json +{ + "results": [ + { + "id": "550e8400-e29b-41d4-a716-446655440000", + "name": "Max", + "breed": { + "id": "german-shepherd", + "name": "German Shepherd", + "origin": "Germany", + "size": "Large", + "parent_breed": { + "id": "herding-dog", + "name": "Herding Dog", + "category": "Working Dog" + } + }, + "age": 5 + } + ] +} +```` + +You can combine nested property extension with multiple property extension: + +Using comma separation: +````GET /api/dogs?extend=breed.parent_breed,owner.address```` + +Using array notation: +````GET /api/dogs?extend[]=breed.parent_breed&extend[]=owner.address```` + +Or combining both approaches: +````GET /api/dogs?extend[]=breed.parent_breed,owner.contact&extend=owner.address```` + +This gives you powerful flexibility to fetch exactly the nested data you need in a single request. + + +### Using the object service + +The ObjectService provides a PHP interface to use data extension programmatically: + +// Extend a single property +````php +$dogs = $objectService->find('dogs', ['extend' => 'breed']); +```` + +// Extend multiple properties using array +````php +$dogs = $objectService->find('dogs', [ + 'extend' => ['breed', 'owner'] +]); +```` + +// Extend nested properties +````php +$dogs = $objectService->find('dogs', [ + 'extend' => ['breed.parent_breed', 'owner.address'] +]); +```` + +// Combine with other query parameters +````php +$dogs = $objectService->find('dogs', [ + 'extend' => ['breed', 'owner'], + 'filter' => ['age' => 5], + 'limit' => 10 +]); + + +## Related Features + +- [Object Relations](object-relations.md) - Base functionality for relations +- [Data Filtering](data-filtering.md) - Combine with filtering for precise data selection +- [Access Control](access-control.md) - Security for extended data \ No newline at end of file diff --git a/docs/data-filtering.md b/docs/data-filtering.md new file mode 100644 index 0000000..52988cb --- /dev/null +++ b/docs/data-filtering.md @@ -0,0 +1,86 @@ +# Data Filtering (_filter) + +Data Filtering allows API consumers to select specific properties they want to receive, supporting data minimalization principles and optimizing response sizes. + +## Overview + +The _filter parameter enables: +- Selection of specific properties +- Nested property filtering +- Exclusion of sensitive data +- Response optimization +- GDPR compliance support + +## Filter Types + +### Property Selection +- Individual fields +- Multiple properties +- Required fields +- Optional fields + +### Nested Properties +- Dot notation support +- Deep property access +- Relationship traversal +- Conditional inclusion + +### Special Filters +- System properties +- Metadata fields +- Computed properties +- Virtual attributes + +## Key Benefits + +1. **Data Minimalization** + - Request only needed data + - Reduce sensitive data exposure + - Support GDPR compliance + - Optimize response size + +2. **Performance** + - Reduce bandwidth usage + - Faster responses + - Efficient processing + - Reduced server load + +3. **Privacy** + - Control data exposure + - Implement need-to-know + - Support data protection + - Audit data access + +## Privacy & Compliance + +- Supports GDPR data minimalization +- Helps implement need-to-know principle +- Reduces unnecessary data exposure +- Provides audit trail integration +- Enforces data access policies + +## Best Practices + +1. **Field Selection** + - Choose fields deliberately + - Consider data sensitivity + - Think about use cases + - Plan for scalability + +2. **Performance** + - Limit nested filters + - Consider caching + - Monitor response times + - Optimize common patterns + +3. **Security** + - Review filtered data + - Check access rights + - Log usage patterns + - Monitor for abuse + +## Related Features + +- [Data Extension](data-extension.md) - Combine with extension for precise data retrieval +- [Access Control](access-control.md) - Security for filtered data +- [Audit Trails](audit-trails.md) - Track data access patterns \ No newline at end of file diff --git a/docs/metadata.md b/docs/metadata.md index 0cdb0bb..edc63fa 100644 --- a/docs/metadata.md +++ b/docs/metadata.md @@ -10,25 +10,71 @@ The metadata system supports: - Searchable attributes - Version tracking +## System Metadata Fields + +| Field | Type | Description | Purpose | +|-------|------|-------------|----------| +| uuid | string | Unique universal identifier | Globally unique object identification | +| uri | string | Uniform Resource Identifier | Unique addressable location | +| version | string | Semantic version number | Track object versions | +| register | string | Register identifier | Object categorization/grouping | +| schema | string | Schema identifier | Data validation reference | +| textRepresentation | text | Text representation of object | Search and display optimization | +| locked | json | Lock information object | Concurrent access control | +| owner | string | Nextcloud user identifier | Object ownership | +| authorization | json | Authorization rules | Access control configuration | +| updated | datetime | Last modification timestamp | Change tracking | +| created | datetime | Creation timestamp | Lifecycle management | +| folder | string | Storage folder path | File organization | + +## Relationship Metadata + +| Field | Type | Description | Purpose | +|-------|------|-------------|----------| +| files | json | Related file IDs | Track associated files | +| relations | json | Related object IDs | Track object relationships | + +## Lock Information Structure +When an object is locked, the following metadata is stored: + +{ + 'user': 'user_id', + 'process': 'optional_process_name', + 'created': 'timestamp', + 'duration': 'seconds', + 'expiration': 'timestamp' +} + +## Authorization Structure +The authorization metadata can contain: +- Access rules +- Permission levels +- User/group assignments +- Role definitions +- Custom policies + ## Key Benefits 1. **Enhanced Organization** - - Improved searchability - - Better categorization - - Flexible classification + - Improved searchability through metadata fields + - Better categorization using registers and schemas + - Flexible classification via custom fields 2. **Process Support** - - Workflow status tracking - - Process automation - - Integration support + - Workflow status tracking through version and lock info + - Process automation using metadata triggers + - Integration support via standardized fields 3. **Data Management** - - Rich context storage - - Extended object information - - Custom attributes + - Rich context storage in metadata fields + - Extended object information tracking + - Custom attributes support + - Complete audit capability ## Related Features - [Schema Validation](schema-validation.md) - Validate metadata - [Content Search](content-search.md) - Search metadata -- [Automatic Facets](automatic-facets.md) - Generate facets from metadata \ No newline at end of file +- [Automatic Facets](automatic-facets.md) - Generate facets from metadata +- [Object Locking](object-locking.md) - Concurrent access control +- [Access Control](access-control.md) - Authorization management \ No newline at end of file From 0aef91ab8175d0ed243a73d90efc6bc2c0cc70e7 Mon Sep 17 00:00:00 2001 From: Ruben van der Linde Date: Wed, 5 Feb 2025 09:00:28 +0100 Subject: [PATCH 06/14] small updates --- docs/metadata.md | 2 ++ docs/object-locking.md | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/docs/metadata.md b/docs/metadata.md index edc63fa..ded35c3 100644 --- a/docs/metadata.md +++ b/docs/metadata.md @@ -45,6 +45,8 @@ When an object is locked, the following metadata is stored: 'expiration': 'timestamp' } +You can read more about locking objects [here](object-locking.md). + ## Authorization Structure The authorization metadata can contain: - Access rules diff --git a/docs/object-locking.md b/docs/object-locking.md index 97ff4f4..b4bc3d3 100644 --- a/docs/object-locking.md +++ b/docs/object-locking.md @@ -27,6 +27,38 @@ The locking system provides: - Transparent lock status - Managed access control +## Lock Information Structure +When an object is locked, the following metadata is stored: + +```json +{ + 'user': 'user_id', + 'process': 'optional_process_name', + 'created': 'timestamp', + 'duration': 'seconds', + 'expiration': 'timestamp' +} +``` + +| Field | Type | Description | +|-------|------|-------------| +| user | string | The ID of the user who created the lock | +| system | string | The ID of the system that created the lock | +| process | string | Optional name of the process that created the lock | +| created | timestamp | When the lock was created | +| duration | integer | How long the lock should last in seconds | +| expiration | timestamp | When the lock will automatically expire | + +The user, system and process fields follow the definition set by [Audit Trails](audit-trails.md) and SHOULD match the same input. + +## Lock Logic + +When an object is locked, the following logic is applied: + +1. The lock is created with the specified duration. +2. The lock is automatically extended if the user is still active. +3. The lock is automatically released if the user is inactive for a specified period. + ## Related Features - [Audit Trails](audit-trails.md) - Track lock operations From 48f9181a1fa3ee510192f1aa7a94419492c0fc60 Mon Sep 17 00:00:00 2001 From: Ruben van der Linde Date: Wed, 5 Feb 2025 10:50:15 +0100 Subject: [PATCH 07/14] Started adding diagrams --- README.md | 2 + docs/diagrams/object-deletion-states.puml | 15 +++ docs/diagrams/object-locking-sequence.puml | 108 +++++++++++++++++++++ docs/object-deletion.md | 69 +++++++++++++ docs/object-locking.md | 16 ++- 5 files changed, 205 insertions(+), 5 deletions(-) create mode 100644 docs/diagrams/object-deletion-states.puml create mode 100644 docs/diagrams/object-locking-sequence.puml create mode 100644 docs/object-deletion.md diff --git a/README.md b/README.md index 437f3c2..0d09062 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ Registers can also apply additional logic to objects, such as validation that is - 💾 **Flexible Storage**: Store objects in Nextcloud, external databases, or object stores. - 🔄 **APIs**: Provide APIs for consumption. - 🧩 **Additional Logic**: Apply extra validation and logic beyond [`schema.json`](https://json-schema.org/). +- 🗑️ [Object Deletion](docs/object-deletion.md) | Soft deletion with retention and recovery | Data safety, compliance, lifecycle management ## Documentation @@ -77,6 +78,7 @@ Open Register makes these principles accessible to any organization by providing | 🔄 [Data Extension](docs/data-extension.md) | Automatically include related entities in responses | Efficient data retrieval, reduced API calls, complete context | | ✂️ [Data Filtering](docs/data-filtering.md) | Select specific properties to return | Data minimalization, GDPR compliance, efficient responses | | 🔍 [Advanced Search](docs/advanced-search.md) | Filter objects using flexible property-based queries | Precise filtering, complex conditions, efficient results | +| 🗑️ [Object Deletion](docs/object-deletion.md) | Soft deletion with retention and recovery | Data safety, compliance, lifecycle management | ## Documentation diff --git a/docs/diagrams/object-deletion-states.puml b/docs/diagrams/object-deletion-states.puml new file mode 100644 index 0000000..804f4b7 --- /dev/null +++ b/docs/diagrams/object-deletion-states.puml @@ -0,0 +1,15 @@ +@startuml Object Deletion States +skinparam monochrome true +skinparam defaultTextAlignment center + +[*] --> Active +Active --> SoftDeleted : delete() +SoftDeleted --> Active : restore() +SoftDeleted --> PendingPurge : retention period expired +PendingPurge --> [*] : purge() + +note right of Active : Object is fully accessible\ndeleted = null +note right of SoftDeleted : Object hidden from queries\ndeleted = timestamp +note right of PendingPurge : Object ready for removal\ncurrent date > purgeDate + +@enduml \ No newline at end of file diff --git a/docs/diagrams/object-locking-sequence.puml b/docs/diagrams/object-locking-sequence.puml new file mode 100644 index 0000000..9e16949 --- /dev/null +++ b/docs/diagrams/object-locking-sequence.puml @@ -0,0 +1,108 @@ +@startuml Object Locking Process +skinparam monochrome true + +actor User +participant "API" as API +participant "ObjectService" as Service +participant "ObjectEntity" as Entity +database "Database" as DB + +== Lock Object == +User -> API: Request lock object +activate API + +API -> Service: lockObject(id, process?, duration?) +activate Service + +Service -> DB: find object +activate DB +DB --> Service: return object +deactivate DB + +Service -> Entity: lock(user, process?, duration?) +activate Entity + +Entity -> Entity: check if already locked +Entity -> Entity: validate user permissions +Entity -> Entity: create/update lock metadata +Entity --> Service: return success +deactivate Entity + +Service -> DB: update object +activate DB +DB --> Service: confirm update +deactivate DB + +Service --> API: return locked object +deactivate Service + +API --> User: return success +deactivate API + +== Auto Expiration == +note over Entity + Lock automatically expires + after duration passes +end note + +== Update Locked Object == +User -> API: Update object +activate API + +API -> Service: updateObject(id, data) +activate Service + +Service -> DB: find object +activate DB +DB --> Service: return object +deactivate DB + +Service -> Entity: validate lock +activate Entity +Entity -> Entity: check lock ownership +Entity -> Entity: extend lock duration +Entity --> Service: validation ok +deactivate Entity + +Service -> DB: update object +activate DB +DB --> Service: confirm update +deactivate DB + +Service --> API: return updated object +deactivate Service + +API --> User: return success +deactivate API + +== Unlock Object == +User -> API: Request unlock object +activate API + +API -> Service: unlockObject(id) +activate Service + +Service -> DB: find object +activate DB +DB --> Service: return object +deactivate DB + +Service -> Entity: unlock(user) +activate Entity +Entity -> Entity: validate lock ownership +Entity -> Entity: remove lock metadata +Entity --> Service: return success +deactivate Entity + +Service -> DB: update object +activate DB +DB --> Service: confirm update +deactivate DB + +Service --> API: return unlocked object +deactivate Service + +API --> User: return success +deactivate API + +@enduml \ No newline at end of file diff --git a/docs/object-deletion.md b/docs/object-deletion.md new file mode 100644 index 0000000..d6dec83 --- /dev/null +++ b/docs/object-deletion.md @@ -0,0 +1,69 @@ +# Object Deletion + +Open Register implements a soft deletion strategy for objects, ensuring data can be recovered and maintaining referential integrity. + +## Overview + +The deletion system provides: +- Soft deletion of objects +- Retention of relationships +- Configurable retention periods +- Recovery capabilities +- Audit trail preservation + +## Deletion Metadata + +When an object is deleted, the following metadata is stored: + +| Field | Type | Description | +|-------|------|-------------| +| deleted | datetime | When the object was marked as deleted | +| deletedBy | string | User ID who performed the deletion | +| deletedReason | string | Optional reason for deletion | +| retentionPeriod | integer | How long to keep the deleted object (in days) | +| purgeDate | datetime | When the object will be permanently deleted | + +## Deletion States + +Objects can exist in the following states: +- Active (deleted = null) +- Soft Deleted (deleted = timestamp) +- Pending Purge (current date > purgeDate) +- Purged (permanently deleted) + +## Deletion Logic + +1. Objects are never immediately deleted from the database +2. Deletion sets the 'deleted' timestamp and related metadata +3. Deleted objects are excluded from normal queries +4. Relations to deleted objects are preserved +5. Files linked to deleted objects are moved to a trash folder +6. Deleted objects can be restored until purge date +7. Objects are only permanently deleted after retention period + +## Key Benefits + +1. **Data Safety** + - Prevent accidental data loss + - Maintain data relationships + - Support data recovery + - Preserve audit history + +2. **Compliance** + - Meet retention requirements + - Support legal holds + - Track deletion reasons + - Document deletion process + +3. **Management** + - Flexible retention policies + - Controlled purge process + - Recovery options + - Clean data lifecycle + +## Related Features + +- [Audit Trails](audit-trails.md) - Track deletion operations +- [Object Locking](object-locking.md) - Prevent deletions of locked objects +- [Access Control](access-control.md) - Manage deletion permissions +- [Time Travel](time-travel.md) - View objects at points before deletion \ No newline at end of file diff --git a/docs/object-locking.md b/docs/object-locking.md index b4bc3d3..3b3d83c 100644 --- a/docs/object-locking.md +++ b/docs/object-locking.md @@ -43,7 +43,7 @@ When an object is locked, the following metadata is stored: | Field | Type | Description | |-------|------|-------------| | user | string | The ID of the user who created the lock | -| system | string | The ID of the system that created the lock | +| system | string | TODO: The ID of the system that created the lock | | process | string | Optional name of the process that created the lock | | created | timestamp | When the lock was created | | duration | integer | How long the lock should last in seconds | @@ -55,11 +55,17 @@ The user, system and process fields follow the definition set by [Audit Trails]( When an object is locked, the following logic is applied: -1. The lock is created with the specified duration. -2. The lock is automatically extended if the user is still active. -3. The lock is automatically released if the user is inactive for a specified period. +1. On object is considerd 'locked' when the object has a lock metadata field set to a diferend vallue then null. +2. When an object is locked, the object is only editable or deletable by the user who created the lock. +3. Only the user who created the lock OR the system that created the lock can unlock the object. +4. When an locked objets is updated, the lock is automatically extended by the duration of the update. +5. Locks my be set for any duration, but the default duration is 1 hour. ## Related Features - [Audit Trails](audit-trails.md) - Track lock operations -- [Access Control](access-control.md) - Manage lock permissions \ No newline at end of file +- [Access Control](access-control.md) - Manage lock permissions + +## Lock Process Flow + +![Object Locking Sequence](diagrams/object-locking-sequence.svg) \ No newline at end of file From f24d9fde8e7d7d0f7781950f7c62ac064cbdc0cf Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Wed, 5 Feb 2025 10:30:11 +0000 Subject: [PATCH 08/14] Bump version to 0.1.43 [skip ci] --- appinfo/info.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appinfo/info.xml b/appinfo/info.xml index c4f099c..d070ab9 100644 --- a/appinfo/info.xml +++ b/appinfo/info.xml @@ -22,7 +22,7 @@ Create a [bug report](https://github.com/OpenRegister/.github/issues/new/choose) Create a [feature request](https://github.com/OpenRegister/.github/issues/new/choose) ]]> - 0.1.42 + 0.1.43 agpl Conduction OpenRegister From 1c86d4c3797a427b53f266bf0c7c89fdf72f21a8 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Wed, 12 Feb 2025 10:43:41 +0000 Subject: [PATCH 09/14] Bump version to 0.1.44 [skip ci] --- appinfo/info.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appinfo/info.xml b/appinfo/info.xml index d070ab9..e5d7c20 100644 --- a/appinfo/info.xml +++ b/appinfo/info.xml @@ -22,7 +22,7 @@ Create a [bug report](https://github.com/OpenRegister/.github/issues/new/choose) Create a [feature request](https://github.com/OpenRegister/.github/issues/new/choose) ]]> - 0.1.43 + 0.1.44 agpl Conduction OpenRegister From 37cd275d1d59f7f8a04039b6b53a140efe56d026 Mon Sep 17 00:00:00 2001 From: Robert Zondervan Date: Wed, 12 Feb 2025 14:36:20 +0100 Subject: [PATCH 10/14] Hotfix: base64 decode before calling function to write decoded data --- lib/Service/ObjectService.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Service/ObjectService.php b/lib/Service/ObjectService.php index c4dda3b..c673e4d 100644 --- a/lib/Service/ObjectService.php +++ b/lib/Service/ObjectService.php @@ -1349,7 +1349,7 @@ public function addFile(ObjectEntity|string $object, string $fileName, string $b $object = $this->objectEntityMapper->find($object); } - return $file = $this->fileService->addFile($object, $fileName, $base64Content); + return $file = $this->fileService->addFile($object, $fileName, base64_decode($base64Content)); } /** From 5d13980f68127e414181717b65d880e9e4b77bed Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Wed, 12 Feb 2025 13:49:22 +0000 Subject: [PATCH 11/14] Bump version to 0.1.45 [skip ci] --- appinfo/info.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appinfo/info.xml b/appinfo/info.xml index e5d7c20..31d148b 100644 --- a/appinfo/info.xml +++ b/appinfo/info.xml @@ -22,7 +22,7 @@ Create a [bug report](https://github.com/OpenRegister/.github/issues/new/choose) Create a [feature request](https://github.com/OpenRegister/.github/issues/new/choose) ]]> - 0.1.44 + 0.1.45 agpl Conduction OpenRegister From 6dcb810e996d7f20bcdcdf4215475c39a980661a Mon Sep 17 00:00:00 2001 From: Robert Zondervan Date: Thu, 13 Feb 2025 16:10:24 +0100 Subject: [PATCH 12/14] Add reshare flag on finding shares --- lib/Service/FileService.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/Service/FileService.php b/lib/Service/FileService.php index 8590f3e..6266018 100644 --- a/lib/Service/FileService.php +++ b/lib/Service/FileService.php @@ -464,7 +464,7 @@ public function findShares(Node $file, int $shareType = 3): array $currentUser = $this->userSession->getUser(); $userId = $currentUser ? $currentUser->getUID() : 'Guest'; - return $this->shareManager->getSharesBy(userId: $userId, shareType: $shareType, path: $file); + return $this->shareManager->getSharesBy(userId: $userId, shareType: $shareType, path: $file, reshares: true); } /** @@ -497,7 +497,7 @@ public function findShare(string $path, ?int $shareType = 3): ?IShare } if ($file instanceof File) { - $shares = $this->shareManager->getSharesBy(userId: $userId, shareType: $shareType, path: $file); + $shares = $this->shareManager->getSharesBy(userId: $userId, shareType: $shareType, path: $file, reshares: true); if (count($shares) > 0) { return $shares[0]; } From bccff4cd9bfa4a7e14388fdd89f2f9ce0dcc53af Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Thu, 13 Feb 2025 15:43:02 +0000 Subject: [PATCH 13/14] Bump version to 0.1.46 [skip ci] --- appinfo/info.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appinfo/info.xml b/appinfo/info.xml index 31d148b..f55940b 100644 --- a/appinfo/info.xml +++ b/appinfo/info.xml @@ -22,7 +22,7 @@ Create a [bug report](https://github.com/OpenRegister/.github/issues/new/choose) Create a [feature request](https://github.com/OpenRegister/.github/issues/new/choose) ]]> - 0.1.45 + 0.1.46 agpl Conduction OpenRegister From c2affb390b3d88dc71ed3a83f1c131120a339df9 Mon Sep 17 00:00:00 2001 From: Robert Zondervan Date: Thu, 13 Feb 2025 16:43:02 +0100 Subject: [PATCH 14/14] Fix tag objecttypes --- lib/Service/FileService.php | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/lib/Service/FileService.php b/lib/Service/FileService.php index 8590f3e..1f99bc0 100644 --- a/lib/Service/FileService.php +++ b/lib/Service/FileService.php @@ -42,7 +42,7 @@ class FileService const ROOT_FOLDER = 'Open Registers'; const APP_GROUP = 'openregister'; const APP_USER = 'OpenRegister'; - const FILE_TAG_TYPE = 'file'; + const FILE_TAG_TYPE = 'files'; /** * Constructor for FileService * @@ -354,16 +354,16 @@ private function getCurrentDomain(): string private function getUser(): IUser { $openCatalogiUser = $this->userManager->get(self::APP_USER); - + if (!$openCatalogiUser) { // Create OpenCatalogi user if it doesn't exist $password = bin2hex(random_bytes(16)); // Generate random password $openCatalogiUser = $this->userManager->createUser(self::APP_USER, $password); - + if (!$openCatalogiUser) { throw new \Exception('Failed to create OpenCatalogi user account.'); } - + // Add user to OpenCatalogi group $group = $this->groupManager->get(self::APP_GROUP); if (!$group) { @@ -371,7 +371,7 @@ private function getUser(): IUser } $group->addUser($openCatalogiUser); } - + return $openCatalogiUser; } @@ -420,7 +420,7 @@ public function formatFiles(array $files): array 'size' => $file->getSize(), 'hash' => $file->getEtag(), 'published' => (new DateTime())->setTimestamp($file->getCreationTime())->format('c'), - 'modified' => (new DateTime())->setTimestamp($file->getUploadTime())->format('c'), + 'modified' => (new DateTime())->setTimestamp($file->getUploadTime())->format('c'), 'labels' => $this->getFileTags(fileId: $file->getId()) ]; @@ -517,7 +517,7 @@ public function findShare(string $path, ?int $shareType = 3): ?IShare private function createShare(array $shareData): IShare { $userId = $this->getUser()->getUID(); - + // Create a new share $share = $this->shareManager->newShare(); $share->setTarget(target: '/'.$shareData['path']); @@ -565,7 +565,7 @@ public function createShareLink(string $path, ?int $shareType = 3, ?int $permiss } $userId = $this->getUser()->getUID(); - + try { $userFolder = $this->rootFolder->getUserFolder(userId: $userId); } catch (NotPermittedException) { @@ -819,14 +819,14 @@ public function addFile(ObjectEntity $objectEntity, string $fileName, string $co $this->userSession->setUser($this->getUser()); $file = $folder->newFile($fileName); - + // Write content to the file $file->putContent($content); $this->userSession->setUser($currentUser); - + return $file; - + } catch (NotPermittedException $e) { $this->logger->error("Permission denied creating file $fileName: " . $e->getMessage()); throw new NotPermittedException("Cannot create file $fileName: " . $e->getMessage());