ENH: Add schema migration support for versioned resources (#298)

Author: GibranAlfa

ARCHITECTURE.md

@@ -65,7 +65,10 @@ classDiagram
 
     class UserFMUDirectory
 
-    class CacheManager
+    class CacheManager {
+        +get_revision_content(..., migration_manager)
+        +restore_revision(..., migration_manager)
+    }
     class LockManager {
         +acquire()
         +ensure_can_write()
@@ -75,12 +78,20 @@ classDiagram
     }
 
     class PydanticResourceManager~PydanticResource~ {
+        +migration_manager
         +load(force, store_cache)
         +save(model)
         +get_resource_diff(incoming_resource)
         +get_structured_model_diff(current_model, incoming_model)
     }
 
+    class MigrationManager~MigratableResource~ {
+        +model_class
+        +current_version: int
+        +migrate_resource(data)
+        +requires_migration(data)
+    }
+
     class MutablePydanticResourceManager~MutablePydanticResource~ {
         +get(key, default)
         +set(key, value)
@@ -129,6 +140,8 @@ classDiagram
 
     FMUDirectoryBase *-- LockManager
     FMUDirectoryBase *-- CacheManager
+    PydanticResourceManager o-- MigrationManager
+    CacheManager ..> MigrationManager
     ProjectFMUDirectory *-- ProjectConfigManager
     ProjectFMUDirectory *-- ChangelogManager
     ProjectFMUDirectory *-- MappingsManager
@@ -140,10 +153,53 @@ The main library split is:
 - `FMUDirectoryBase` is the filesystem-centered abstraction. It owns the `.fmu` path, lock manager, cache manager, and generic read/write helpers.
 - `ProjectFMUDirectory` and `UserFMUDirectory` specialize the base class for project-local `.fmu/` directories and `$HOME/.fmu/`.
 - `PydanticResourceManager` is the generic resource engine for loading, saving, diffing, and caching JSON-backed Pydantic models.
+- `MigrationManager` applies registered, forward-only schema migrations and validates the result as the current Pydantic model.
 - `MutablePydanticResourceManager` adds dot-notation `get`, `set`, `update`, `reset`, and merge behavior for editable resources.
 - `ProjectConfigManager`, `UserConfigManager`, `MappingsManager`, and `LogManager` bind specific Pydantic models to managed files inside `.fmu/`.
 - Directory objects compose the correct managers and delegate resource operations to them.
 
+## Schema Migration
+
+`ProjectConfig`, `UserConfig`, and `InternalMappings` are versioned resources. Each
+model declares its current schema version, and each resource manager has a
+`MigrationManager` with the migration registry for that model.
+
+Migration is forward-only. Data without `schema_version` is treated as version 1.
+Each registered migration function increments the schema version by one. The manager
+rejects a newer schema, a missing migration step, an invalid version, or data that
+does not validate as the current model.
+
+The migration boundary depends on the resource operation:
+
+- **Load:** `PydanticResourceManager` decodes the stored JSON and asks
+  `MigrationManager` for a validated current model. Migration happens in memory.
+  Loading does not write the resource, create a cache revision, or add a changelog
+  entry.
+- **Save:** The resource manager checks the write lock first. If the stored data is
+  older, it attempts to store the exact original JSON under
+  `.fmu/migration-backups/` before writing the current model. This backup is
+  best-effort: a file-system failure is logged and does not stop the save. The
+  original JSON is also stored as a cache revision. The newly written current
+  data is added as another cache revision.
+- **Cache read:** `CacheManager` uses the migration manager to return old revisions
+  as current validated models.
+- **Restore:** `CacheManager` migrates the selected revision before writing it, so
+  the restored resource uses the current schema. If you roll back to an older
+  release while the pre-migration revision is still retained, restore that revision with
+  the older release to return the resource file to the older schema. Project restore
+  operations use the existing restore changelog entry.
+
+There is no backward migration. After a current-schema resource is saved, an older
+`fmu-settings` release can reject it as newer than supported. If the pre-migration
+cache revision is still retained, restore it with the older release. Migration
+backups are not loaded or restored by the library. If the cache revision is no longer
+available, we should help users copy the appropriate migration backup back to the
+resource file before running the older release.
+
+See the
+[schema migration guide](src/fmu/settings/_migrations/README.md)
+for the implementation, test, and coordinated release process.
+
 ## Runtime Flow
 
 The full runtime spans multiple repositories, but `fmu-settings` owns the `.fmu/` directory operations used by the API and CLI.

CONTRIBUTING.md

@@ -32,3 +32,8 @@ ruff check
 ruff format --check
 mypy src tests
 ```
+
+If you need to change the schema of `ProjectConfig`, `UserConfig`, or
+`InternalMappings`, see the
+[schema migration guide](src/fmu/settings/_migrations/README.md) for implementation
+and testing details.

README.md

@@ -81,6 +81,11 @@ ruff format --check
 mypy src tests
 ```
 
+If you need to change the schema of `ProjectConfig`, `UserConfig`, or
+`InternalMappings`, see the
+[schema migration guide](src/fmu/settings/_migrations/README.md) for implementation
+and testing details.
+
 See [CONTRIBUTING.md](CONTRIBUTING.md) for more.
 
 > [!NOTE]

src/fmu/settings/__init__.py

@@ -21,6 +21,7 @@
     init_fmu_directory,
     init_user_fmu_directory,
 )
+from ._migrations import MigrationError
 from .models._enums import CacheResource
 from .models.mappings import (
     InternalBaseMapping,
@@ -47,6 +48,7 @@
     "InternalWellboreIdentifierMapping",
     "InternalWellboreMappings",
     "InvalidFMUProjectPathError",
+    "MigrationError",
     "ProjectFMUDirectory",
     "REQUIRED_FMU_PROJECT_SUBDIRS",
     "UserFMUDirectory",

src/fmu/settings/_fmu_dir.py

@@ -28,6 +28,9 @@
 )
 from .models.user_config import UserConfig
 
+if TYPE_CHECKING:
+    from ._migrations import MigrationManager
+
 logger: Final = null_logger(__name__)
 
 FMUConfigManager: TypeAlias = ProjectConfigManager | UserConfigManager
@@ -44,7 +47,7 @@ class FMUDirectoryBase:
     def __init__(
         self: Self,
         base_path: str | Path,
-        cache_revisions: int = CacheManager.MIN_REVISIONS,
+        cache_revisions: int = 10,
         *,
         lock_timeout_seconds: int = DEFAULT_LOCK_TIMEOUT,
     ) -> None:
@@ -53,7 +56,8 @@ def __init__(
         Args:
             base_path: The directory containing the .fmu directory or one of its parent
                 dirs
-            cache_revisions: Number of revisions to retain in the cache. Minimum is 5.
+            cache_revisions: Number of revisions to retain in the cache. Default is 10.
+                Minimum is 5.
             lock_timeout_seconds: Lock expiration time in seconds. Default 20 minutes.
 
         Raises:
@@ -331,20 +335,18 @@ def __init__(
         self.config = ProjectConfigManager(self)
         super().__init__(
             base_path,
-            CacheManager.MIN_REVISIONS,
+            10,
             lock_timeout_seconds=lock_timeout_seconds,
         )
         self._changelog = ChangelogManager(self)
         self._mappings = MappingsManager(self)
         try:
-            max_revisions = self.config.get(
-                "cache_max_revisions", CacheManager.MIN_REVISIONS
-            )
+            max_revisions = self.config.get("cache_max_revisions", 10)
             self._cache_manager.max_revisions = max_revisions
         except (FileNotFoundError, ValueError) as e:
             logger.warning(
                 f"Failed to load 'cache_max_revisions' from project config. "
-                f"Using default value '{CacheManager.MIN_REVISIONS}'. Error: {e}"
+                f"Using default value '10'. Error: {e}"
             )
 
     def update_validation_metadata(
@@ -494,13 +496,19 @@ def restore_from_cache(
         if manager is self.config:
             previous_max_revisions = self._cache_manager.max_revisions
             restored_config = self.cache.get_revision_content(
-                relative_path, revision_id, model_class=ProjectConfig
+                relative_path,
+                revision_id,
+                model_class=ProjectConfig,
+                migration_manager=self.config.migration_manager,
             )
             self._cache_manager.max_revisions = restored_config.cache_max_revisions
 
             try:
                 self.cache.restore_revision(
-                    relative_path, revision_id, model_class=ProjectConfig
+                    relative_path,
+                    revision_id,
+                    model_class=ProjectConfig,
+                    migration_manager=self.config.migration_manager,
                 )
             except Exception:
                 # Restore the previous runtime retention if config restore fails
@@ -519,7 +527,13 @@ def restore_from_cache(
             return
 
         self.cache.restore_revision(
-            relative_path, revision_id, model_class=manager.model_class
+            relative_path,
+            revision_id,
+            model_class=cast("type[BaseModel]", manager.model_class),
+            migration_manager=cast(
+                "MigrationManager[BaseModel] | None",
+                manager.migration_manager,
+            ),
         )
 
         # Refresh the resource manager's in-memory cache
@@ -555,7 +569,13 @@ def get_cache_content(
             )
 
         return self.cache.get_revision_content(
-            relative_path, revision_id, model_class=manager.model_class
+            relative_path,
+            revision_id,
+            model_class=cast("type[BaseModel]", manager.model_class),
+            migration_manager=cast(
+                "MigrationManager[BaseModel] | None",
+                manager.migration_manager,
+            ),
         )
 
     def _cacheable_resource_managers(