Implement plugin/modular system for FMF metadata loaders (Phase 1)

docs/concept.rst

@@ -123,6 +123,91 @@ In the example above files or directories named ``.plans`` or
 the ``.fmf`` directory cannot be used for storing metadata.
 
 
+Plugins
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Starting with fmf 2.0, a plugin system allows reading metadata from
+multiple file formats beyond ``.fmf`` (YAML) files. This enables
+extracting test metadata directly from source files like Python tests
+or Bash scripts.
+
+.. _config-plugins:
+
+Plugin Configuration
+--------------------
+
+Plugins can be enabled in the ``.fmf/config`` file:
+
+.. code-block:: yaml
+
+    plugins:
+      - fmf  # Short name for FmfPlugin
+      # or use full path:
+      - fmf.plugins.fmf.FmfPlugin
+      # Future phases:
+      # - bash
+      # - pytest
+
+The ``plugins`` section lists built-in plugins to load. Each plugin
+handles specific file types (e.g., ``.sh`` for Bash, ``.py`` for Python).
+
+**Security**: Only built-in plugins from ``fmf/plugins/`` can be loaded.
+Arbitrary code execution from config files is prevented.
+
+If no ``plugins`` section is present, only the default ``FmfPlugin``
+is loaded (for backward compatibility), which handles ``.fmf`` files.
+
+Plugin Priority
+---------------
+
+When multiple plugins can handle the same file extension, the plugin
+with the highest priority (0-200) is selected. Built-in plugins use
+these priorities:
+
+* FmfPlugin: 100 (default format)
+* BashPlugin: 50 (future)
+* PytestPlugin: 50 (future)
+
+You can override plugin priorities in ``.fmf/config``:
+
+.. code-block:: yaml
+
+    plugins:
+      - fmf
+      - bash
+
+    # Override priorities to prefer bash over fmf
+    bash:
+      priority: 120  # Higher than FmfPlugin (100)
+
+    # Or lower fmf priority
+    fmf:
+      priority: 30   # Lower than default (100)
+
+This allows you to control which plugin takes precedence when
+multiple formats are present in the same tree.
+
+File Pattern Override
+---------------------
+
+Future enhancement: Plugins will support custom file patterns via
+configuration to filter which files are processed.
+
+Available Plugins
+-----------------
+
+**Phase 1 (Current):**
+
+* ``fmf.plugins.fmf.FmfPlugin`` - YAML-based ``.fmf`` files (default)
+
+**Future Phases:**
+
+* ``fmf.plugins.bash.BashPlugin`` - Bash scripts with ``#:FMF:`` comments
+* ``fmf.plugins.pytest.PytestPlugin`` - Python tests with pytest marks
+
+See ``PLUGIN_FUTURE.md`` for detailed plugin implementation roadmap.
+
+
 Names
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

fmf/base.py

@@ -12,19 +12,21 @@
 from typing import Any, Dict, Optional, Protocol
 
 from ruamel.yaml import YAML
-from ruamel.yaml.constructor import DuplicateKeyError
 from ruamel.yaml.error import YAMLError
 
 import fmf.context
+import fmf.plugins  # noqa: F401  # Load built-in plugins
 import fmf.utils as utils
+from fmf.plugin_loader import get_registry
+# Re-export constants for backward compatibility
+from fmf.plugins.fmf import MAIN, SUFFIX
 from fmf.utils import dict_to_yaml, log
 
 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 #  Constants
 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-SUFFIX = ".fmf"
-MAIN = "main" + SUFFIX
+# SUFFIX and MAIN moved to fmf.plugins.fmf and re-exported above
 IGNORED_DIRECTORIES = ['/dev', '/proc', '/sys']
 ADJUST_CONTROL_KEYS = ['because', 'continue', 'when']
 
@@ -210,6 +212,18 @@ def _initialize(self, path):
         except YAMLError as error:
             raise utils.FileError(f"Failed to parse '{config_file_path}'.\n{error}")
 
+        # Initialize plugin registry
+        registry = get_registry()
+
+        # Load plugins from config if specified
+        if "plugins" in self.config:
+            registry.load_from_config(self.config)
+            log.debug("Plugins loaded from config.")
+        else:
+            # Default: only FmfPlugin (already auto-registered)
+            # This ensures backward compatibility when no config exists
+            log.debug("No plugin config found, using default FmfPlugin.")
+
     def _merge_plus(self, data, key, value, prepend=False):
         """
         Handle extending attributes using the '+' suffix
@@ -716,35 +730,48 @@ def grow(self, path):
             log.debug("Skipping '{0}' (not accessible).".format(path))
             return
 
-        # Investigate main.fmf as the first file (for correct inheritance)
-        filenames = sorted(
-            [filename for filename in filenames if filename.endswith(SUFFIX)])
-        try:
-            filenames.insert(0, filenames.pop(filenames.index(MAIN)))
-        except ValueError:
-            pass
+        # Get plugin registry
+        registry = get_registry()
+
+        # Prioritize main.fmf first if it exists (for correct inheritance)
+        if MAIN in filenames:
+            filenames = sorted([f for f in filenames if f != MAIN])
+            filenames.insert(0, MAIN)
+        else:
+            filenames = sorted(filenames)
 
-        # Check every metadata file and load data (ignore hidden)
+        # Check every file and load data if a plugin can handle it
         for filename in filenames:
+            # Skip hidden files (unless enabled in config)
             if filename.startswith(".") and filename not in self.explore_include:
                 continue
+
             fullpath = os.path.abspath(os.path.join(dirpath, filename))
-            log.info("Checking file {0}".format(fullpath))
+
+            # Find appropriate plugin for this file
+            # This uses can_handle() which can check regex patterns
+            plugin_class = registry.get_plugin_for_file(filename)
+            if not plugin_class:
+                # No plugin can handle this file, skip silently
+                continue
+
+            # Read file using plugin
+            log.info(f"Processing '{fullpath}' with {plugin_class.__name__}")
             try:
-                with open(fullpath, encoding='utf-8') as datafile:
-                    # Workadound ruamel s390x read issue - fmf/issues/164
-                    content = datafile.read()
-                    data = YAML(typ="safe").load(content)
-            except (YAMLError, DuplicateKeyError) as error:
+                plugin = plugin_class()
+                data = plugin.read(fullpath)
+            except Exception as error:
                 raise utils.FileError(
-                    f"Failed to parse '{fullpath}'.\n{error}")
+                    f"Failed to read '{fullpath}' with {plugin_class.__name__}.\n{error}")
+
             log.data(pretty(data))
-            # Handle main.fmf as data for self
+
+            # Handle main.fmf specially (backward compatibility)
             if filename == MAIN:
                 self.sources.append(fullpath)
                 self._raw_data = copy.deepcopy(data)
                 self.update(data)
-            # Handle other *.fmf files as children
+            # Handle other files as children
             else:
                 self.child(os.path.splitext(filename)[0], data, fullpath)
 
@@ -1075,9 +1102,31 @@ def __enter__(self):
     def __exit__(self, exc_type, exc_val, exc_tb):
         """
         Experimental: Store modified metadata to disk
+
+        Uses the plugin system to write data back to the source file.
+        Falls back to direct YAML write if no plugin is found.
         """
 
         _, full_data, source = self._locate_raw_data()
+
+        # Try to use plugin for writing
+        registry = get_registry()
+        plugin_class = registry.get_plugin_for_file(source)
+
+        if plugin_class:
+            try:
+                plugin = plugin_class()
+                # For now, we pass full_data as the data parameter
+                # hierarchy, append_dict, modified_dict, deleted_items are not used yet
+                plugin.write(source, [], full_data, {}, {}, [])
+                return
+            except NotImplementedError:
+                # Plugin doesn't support write, fall back to default
+                log.debug(f"Plugin {plugin_class.__name__} doesn't support write, using default")
+            except Exception as error:
+                log.warning(f"Plugin write failed: {error}, using default")
+
+        # Fallback: direct YAML write (backward compatibility)
         with open(source, "w", encoding='utf-8') as file:
             file.write(dict_to_yaml(full_data))
 

fmf/plugin.py

@@ -0,0 +1,151 @@
+"""
+Abstract Plugin Base Class for FMF Metadata Loaders
+"""
+
+from abc import ABC, abstractmethod
+from typing import Any, Dict, List, Optional
+
+
+class Plugin(ABC):
+    """
+    Abstract base class for FMF metadata loaders.
+
+    Each plugin handles one or more file extensions and provides
+    methods to read and write metadata in those formats.
+
+    Subclasses must define:
+        - extensions: List of file extensions (e.g., [".fmf", ".sh"])
+        - file_patterns: List of regex patterns to match filenames
+        - priority: Integer 0-200, higher values preferred for conflicts
+        - can_handle(): Method to determine if plugin can handle a file
+        - read(): Method to extract metadata from a file
+
+    Optional:
+        - write(): Method to write metadata back (default: fallback to .fmf)
+        - CONFIG_SECTION: Name of config section in .fmf/config
+    """
+
+    # Class attributes to be defined by subclasses
+    extensions: List[str] = []  # File extensions, e.g., [".fmf"]
+    file_patterns: List[str] = []  # Regex patterns for filenames
+    priority: int = 50  # 0-200, higher = preferred when conflicts occur
+    CONFIG_SECTION: Optional[str] = None  # Config section name
+
+    @abstractmethod
+    def can_handle(self, filename: str) -> bool:
+        """
+        Determine if this plugin can handle the given file.
+
+        Args:
+            filename: Absolute or relative path to file
+
+        Returns:
+            True if plugin can read this file, False otherwise
+        """
+        pass
+
+    @abstractmethod
+    def read(self, filename: str) -> Dict[str, Any]:
+        """
+        Read metadata from file and return as dictionary.
+
+        Args:
+            filename: Path to file to read
+
+        Returns:
+            Dictionary with fmf metadata structure. Can be nested
+            for hierarchical metadata (e.g., test classes with methods).
+
+        Raises:
+            FileError: If file cannot be read or parsed
+        """
+        pass
+
+    @abstractmethod
+    def write(
+            self,
+            filename: str,
+            hierarchy: List[str],
+            data: Dict[str, Any],
+            append_dict: Dict[str, Any],
+            modified_dict: Dict[str, Any],
+            deleted_items: List[str]) -> None:
+        """
+        Write modified metadata back to file.
+
+        Args:
+            filename: Original file path
+            hierarchy: Path components from tree root (e.g., ["/parent", "/child"])
+            data: Complete node data dictionary
+            append_dict: Keys with + suffix (merge operations)
+            modified_dict: Modified keys
+            deleted_items: List of removed keys
+
+        Raises:
+            NotImplementedError: If plugin doesn't support writing
+
+        Note:
+            If your plugin cannot write back to the original format,
+            you can use self._write_fmf_fallback() to create a .fmf file
+            with the same base name instead.
+        """
+        pass
+
+    def _write_fmf_fallback(
+            self,
+            filename: str,
+            hierarchy: List[str],
+            modified_dict: Dict[str, Any],
+            append_dict: Dict[str, Any]) -> None:
+        """
+        Create a .fmf file as fallback for plugins that can't write.
+
+        This method constructs the hierarchical structure based on
+        the hierarchy path and writes it to a .fmf file alongside
+        the original file.
+
+        Args:
+            filename: Original file path
+            hierarchy: Path components from tree root
+            modified_dict: Modified keys to write
+            append_dict: Append operations (keys with +)
+        """
+        import os
+
+        from ruamel.yaml import YAML
+
+        # Build hierarchical dictionary from hierarchy path
+        output = {}
+        current = output
+
+        for key in hierarchy:
+            if key not in current or current[key] is None:
+                current[key] = {}
+            current = current[key]
+
+        # Add modified data to the leaf node
+        current.update(modified_dict)
+
+        # Add append operations
+        for key, value in append_dict.items():
+            # Use key+ notation for append operations
+            current[key + '+'] = value
+
+        # Generate .fmf filename from original file
+        path = os.path.dirname(filename)
+        basename = os.path.basename(filename)
+
+        # Remove original extension
+        for ext in self.extensions:
+            if basename.endswith(ext):
+                basename = basename[:-len(ext)]
+                break
+
+        fmf_file = os.path.join(path, basename + ".fmf")
+
+        # Write YAML to .fmf file
+        yaml = YAML()
+        yaml.default_flow_style = False
+
+        with open(fmf_file, 'w', encoding='utf-8') as f:
+            yaml.dump(output, f)