Merge pull request #11 from untergeek/rel/0.11.0

Release 0.11.0 prep

Author: untergeek

pyproject.toml

@@ -33,9 +33,9 @@ keywords = [
 dependencies = [
     'gitpython==3.1.43',
     'dotmap==1.3.30',
-    'es_client>=8.17.5',
-    'es-wait>=0.12.0',
-    'tiered_debug==1.0.0',
+    'es_client>=8.18.0',
+    'es-wait>=0.14.0',
+    'tiered_debug==1.1.0',
 ]
 
 [project.optional-dependencies]

src/es_testbed/__init__.py

@@ -1,8 +1,9 @@
 """Make classes easier to import here"""
 
-__version__ = '0.10.0'
+__version__ = '0.11.0'
 
 from es_testbed._base import TestBed
 from es_testbed._plan import PlanBuilder
+from es_testbed.debug import debug
 
-__all__ = ['TestBed', 'PlanBuilder']
+__all__ = ['TestBed', 'PlanBuilder', 'debug']

src/es_testbed/_base.py

@@ -5,13 +5,13 @@
 from importlib import import_module
 from datetime import datetime, timezone
 from shutil import rmtree
-import tiered_debug as debug
-from es_testbed.exceptions import ResultNotExpected
-from es_testbed.defaults import NAMEMAPPER
-from es_testbed.helpers.es_api import delete, get
-from es_testbed.helpers.utils import prettystr, process_preset
-from es_testbed._plan import PlanBuilder
-from es_testbed.mgrs import (
+from .debug import debug, begin_end
+from .defaults import NAMEMAPPER
+from .es_api import delete, get
+from .exceptions import ResultNotExpected
+from .utils import prettystr, process_preset
+from ._plan import PlanBuilder
+from .mgrs import (
     ComponentMgr,
     DataStreamMgr,
     IlmMgr,
@@ -107,8 +107,8 @@ def __init__(
         #: The data_stream entity manager
         self.data_streammgr = None
 
+    @begin_end()
     def _erase(self, kind: str, lst: t.Sequence[str]) -> None:
-        debug.lv2('Starting method...')
         overall_success = True
         if not lst:
             debug.lv3(f'{kind}: nothing to delete.')
@@ -118,15 +118,14 @@ def _erase(self, kind: str, lst: t.Sequence[str]) -> None:
             overall_success = False not in ilm  # No False values == True
         else:
             overall_success = self._while(kind, ','.join(lst))
-        debug.lv3('Exiting method, returning value')
-        debug.lv5(f'Value = {overall_success}')
+        debug.lv5(f'Return value = {overall_success}')
         return overall_success
 
-    def _fodder_generator(
+    @begin_end()
+    def _erase_all(
         self,
     ) -> t.Generator[str, t.Sequence[str], None]:
         """Method to delete everything matching our pattern(s)"""
-        debug.lv2('Starting method...')
         items = ['index', 'data_stream', 'snapshot', 'template', 'component', 'ilm']
         for i in items:
             if i == 'snapshot' and self.plan.repository is None:
@@ -135,10 +134,9 @@ def _fodder_generator(
             pattern = f'*{self.plan.prefix}-{NAMEMAPPER[i]}-{self.plan.uniq}*'
             entities = get(self.client, i, pattern, repository=self.plan.repository)
             yield (i, entities)
-        debug.lv3('Exiting method')
 
+    @begin_end()
     def _while(self, kind: str, item: str) -> bool:
-        debug.lv2('Starting method...')
         count = 1
         success = False
         exc = None
@@ -160,11 +158,11 @@ def _while(self, kind: str, item: str) -> bool:
             )
         return success
 
+    @begin_end()
     def get_ilm_polling(self) -> None:
         """
         Get current ILM polling settings and store them in self.plan.polling_interval
         """
-        debug.lv2('Starting method...')
         debug.lv3('Storing current ILM polling settings, if any...')
         try:
             debug.lv4('TRY: Getting cluster settings')
@@ -191,25 +189,17 @@ def get_ilm_polling(self) -> None:
             retval = None  # Must be an actual value to go into a DotMap
         self.plan.ilm_polling_interval = retval
         debug.lv3(f'Stored ILM Polling Interval: {retval}')
-        debug.lv3('Exiting method')
 
+    @begin_end()
     def ilm_polling(self, interval: t.Union[str, None] = None) -> t.Dict:
         """Return persistent cluster settings to speed up ILM polling during testing"""
-        debug.lv2('Starting method...')
         retval = {'indices.lifecycle.poll_interval': interval}
-        debug.lv3('Exiting method, returning value')
-        debug.lv5(f'Value = {retval}')
+        debug.lv5(f'Return value = {retval}')
         return retval
 
-    def set_debug_tier(self, tier: int) -> None:
-        """
-        Set the debug tier globally for this module
-        """
-        debug.set_level(tier)
-
+    @begin_end()
     def setup(self) -> None:
         """Setup the instance"""
-        debug.lv2('Starting method...')
         start = datetime.now(timezone.utc)
         # If we build self.plan here, then we can modify settings before setup()
         self.plan = PlanBuilder(settings=self.settings).plan
@@ -219,13 +209,12 @@ def setup(self) -> None:
         self.setup_entitymgrs()
         end = datetime.now(timezone.utc)
         debug.lv1(f'Testbed setup elapsed time: {(end - start).total_seconds()}')
-        debug.lv3('Exiting method')
 
+    @begin_end()
     def setup_entitymgrs(self) -> None:
         """
         Setup each EntityMgr child class
         """
-        debug.lv2('Starting method...')
         kw = {'client': self.client, 'plan': self.plan}
 
         self.ilmmgr = IlmMgr(**kw)
@@ -242,17 +231,16 @@ def setup_entitymgrs(self) -> None:
         if self.plan.type == 'data_stream':
             self.data_streammgr = DataStreamMgr(**kw, snapmgr=self.snapshotmgr)
             self.data_streammgr.setup()
-        debug.lv3('Exiting method')
 
+    @begin_end()
     def teardown(self) -> None:
         """Tear down anything we created"""
-        debug.lv2('Starting method...')
         start = datetime.now(timezone.utc)
         successful = True
         if self.plan.tmpdir:
             debug.lv3(f'Removing tmpdir: {self.plan.tmpdir}')
             rmtree(self.plan.tmpdir)  # Remove the tmpdir stored here
-        for kind, list_of_kind in self._fodder_generator():
+        for kind, list_of_kind in self._erase_all():
             if not self._erase(kind, list_of_kind):
                 successful = False
         persist = self.ilm_polling(interval=self.plan.ilm_polling_interval)
@@ -268,4 +256,3 @@ def teardown(self) -> None:
         else:
             logger.error('Cleanup was unsuccessful/incomplete')
         self.plan.cleanup = successful
-        debug.lv3('Exiting method')

src/es_testbed/_plan.py

@@ -3,9 +3,9 @@
 import typing as t
 import logging
 from dotmap import DotMap
-import tiered_debug as debug
-from es_testbed.defaults import TESTPLAN
-from es_testbed.helpers.utils import build_ilm_policy, prettystr, randomstr
+from .debug import debug, begin_end
+from .defaults import TESTPLAN
+from .utils import build_ilm_policy, prettystr, randomstr
 
 logger = logging.getLogger(__name__)
 
@@ -114,8 +114,8 @@ def plan(self) -> DotMap:
         """Return the Plan"""
         return self._plan
 
+    @begin_end()
     def _create_lists(self) -> None:
-        debug.lv2('Starting method...')
         names = [
             'indices',
             'data_stream',
@@ -126,30 +126,27 @@ def _create_lists(self) -> None:
         ]
         for name in names:
             self._plan[name] = []
-        debug.lv3('Exiting method')
 
+    @begin_end()
     def setup(self) -> None:
         """Do initial setup of the Plan DotMap"""
-        debug.lv2('Starting method...')
         self._plan.uniq = randomstr(length=8, lowercase=True)
         self._create_lists()
         self.update(self.settings)  # Override with settings.
         self.update_rollover_alias()
         debug.lv3('Rollover alias updated')
         self.update_ilm()
         debug.lv5(f'FINAL PLAN: {prettystr(self._plan.toDict())}')
-        debug.lv3('Exiting method')
 
+    @begin_end()
     def update(self, settings: t.Dict) -> None:
         """Update the Plan DotMap"""
-        debug.lv2('Starting method...')
         self._plan.update(DotMap(settings))
         debug.lv5(f'Updated plan: {prettystr(self._plan.toDict())}')
-        debug.lv3('Exiting method')
 
+    @begin_end()
     def update_ilm(self) -> None:
         """Update the ILM portion of the Plan DotMap"""
-        debug.lv2('Starting method...')
         setdefault = False
         if 'ilm' not in self._plan:
             debug.lv3('key "ilm" is not in plan')
@@ -194,14 +191,12 @@ def update_ilm(self) -> None:
             }
             debug.lv5(f'KWARGS = {kwargs}')
             self._plan.ilm.policy = build_ilm_policy(**kwargs)
-            debug.lv3('Exiting method')
 
+    @begin_end()
     def update_rollover_alias(self) -> None:
         """Update the Rollover Alias value"""
-        debug.lv2('Starting method...')
         if self._plan.rollover_alias:
             self._plan.rollover_alias = f'{self._plan.prefix}-idx-{self._plan.uniq}'
         else:
             self._plan.rollover_alias = None
         debug.lv5(f'Updated rollover_alias = {self._plan.rollover_alias}')
-        debug.lv3('Exiting method')

src/es_testbed/debug.py

@@ -0,0 +1,38 @@
+"""Tiered Debugging module"""
+
+import typing as t
+from functools import wraps
+from tiered_debug import TieredDebug
+
+debug = TieredDebug()
+
+
+def begin_end(begin: t.Optional[int] = 2, end: t.Optional[int] = 3) -> t.Callable:
+    """Decorator to log the beginning and end of a function.
+
+    This decorator will log the beginning and end of a function call at the
+    specified levels, defaulting to 2 for the beginning and 3 for the ending.
+
+    :return: The decorated function.
+    :rtype: Callable
+    """
+    mmap = {
+        1: debug.lv1,
+        2: debug.lv2,
+        3: debug.lv3,
+        4: debug.lv4,
+        5: debug.lv5,
+    }
+
+    def decorator(func: t.Callable) -> t.Callable:
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            common = f"CALL: {func.__name__}()"
+            mmap[begin](f"BEGIN {common}", stklvl=debug.stacklevel + 1)
+            result = func(*args, **kwargs)
+            mmap[end](f"END {common}", stklvl=debug.stacklevel + 1)
+            return result
+
+        return wrapper
+
+    return decorator

src/es_testbed/defaults.py

@@ -3,31 +3,19 @@
 import typing as t
 
 EPILOG: str = 'Learn more at https://github.com/untergeek/es-testbed'
+"""Epilog for the Click CLI"""
 
 HELP_OPTIONS: dict = {'help_option_names': ['-h', '--help']}
-
-ARGSCLASSES: list = ['IlmBuilder', 'IlmExplain', 'TestPlan']
+"""Default help options for Click commands"""
 
 COLD_PREFIX: str = 'restored-'
+"""Prefix for cold searchable snapshot indices"""
+
 FROZEN_PREFIX: str = 'partial-'
+"""Prefix for frozen searchable snapshot indices"""
 
 SS_PREFIX: t.Dict[str, str] = {'cold': COLD_PREFIX, 'frozen': FROZEN_PREFIX}
-
-MAPPING: dict = {
-    'properties': {
-        '@timestamp': {'type': 'date'},
-        'message': {'type': 'keyword'},
-        'number': {'type': 'long'},
-        'nested': {'properties': {'key': {'type': 'keyword'}}},
-        'deep': {
-            'properties': {
-                'l1': {
-                    'properties': {'l2': {'properties': {'l3': {'type': 'keyword'}}}}
-                }
-            }
-        },
-    }
-}
+"""Dictionary of prefixes for searchable snapshot indices"""
 
 NAMEMAPPER: t.Dict[str, str] = {
     'index': 'idx',
@@ -38,14 +26,18 @@
     'template': 'tmpl',
     'snapshot': 'snp',
 }
+"""Mapping of names to abbreviations for use in the CLI"""
 
 PAUSE_DEFAULT: str = '1.0'
+"""Default value for the pause time in seconds"""
 PAUSE_ENVVAR: str = 'ES_TESTBED_PAUSE'
+"""Environment variable for the pause time"""
 
 PLURALMAP: t.Dict[str, str] = {
     'ilm': 'ILM Policies',
     'index': 'indices',
 }
+"""Mapping of singular to plural names for use in the CLI"""
 
 TESTPLAN: dict = {
     'type': 'indices',
@@ -77,9 +69,12 @@
         'storage': 'shared_cache',
     },
 }
+"""Default values for the ILM tiers"""
 
 TIMEOUT_DEFAULT: str = '30'
+"""Default timeout for the testbed in seconds"""
 TIMEOUT_ENVVAR: str = 'ES_TESTBED_TIMEOUT'
+"""Environment variable for the testbed timeout"""
 
 # Define IlmPhase as a typing alias to be reused multiple times
 #
@@ -97,6 +92,7 @@
 IlmPhase = t.Dict[
     str, t.Union[str, t.Dict[str, str], t.Dict[str, t.Dict[str, t.Dict[str, str]]]]
 ]
+"""ILM Phase type alias"""
 
 
 def ilmhot() -> IlmPhase:

src/es_testbed/entities/alias.py

@@ -3,10 +3,10 @@
 # pylint: disable=C0115,C0116,R0902,R0904,R0913,R0917
 import typing as t
 import logging
-import tiered_debug as debug
-from es_testbed.entities.entity import Entity
-from es_testbed.helpers.es_api import resolver, rollover
-from es_testbed.helpers.utils import prettystr
+from ..debug import debug, begin_end
+from ..es_api import resolver, rollover
+from ..utils import prettystr
+from .entity import Entity
 
 if t.TYPE_CHECKING:
     from elasticsearch8 import Elasticsearch
@@ -25,14 +25,13 @@ def __init__(
         super().__init__(client=client, name=name)
         debug.lv3('Alias entity object initialized')
 
+    @begin_end()
     def rollover(self) -> None:
         """Rollover the alias"""
-        debug.lv2('Starting method...')
         rollover(self.client, self.name)
-        debug.lv3('Exiting method')
 
+    @begin_end()
     def verify(self, index_list: t.Sequence[str]) -> bool:
-        debug.lv2('Starting method...')
         retval = False
         res = resolver(self.client, self.name)
         debug.lv5(f'resolver response: {prettystr(res)}')
@@ -48,6 +47,5 @@ def verify(self, index_list: t.Sequence[str]) -> bool:
                 debug.lv3(f'Confirm match of indices backed by alias {self.name}')
                 retval = True
                 break
-        debug.lv3('Exiting method, returning value')
-        debug.lv5(f'Value = {retval}')
+        debug.lv5(f'Return value = {retval}')
         return retval