Simplify middleware audit startup

Delay certain expensive parts of middlewared auditing startup (an
initial scan for file changes and removing unnecessary refreservations
on old audit datasets) until after the system READY event, and only
perform it on first boot after upgrade or fresh install.

This commit also removes the AuditSetupAlertClass. It could only be
called during the first boot for a boot environment and would never
be dismissed. Removal decreases complexity of operations during
system ready events on first boot.

Author: anodos325

docs/source/middleware/index.rst

@@ -6,6 +6,7 @@ Middleware Daemon
     :caption: Contents:
 
     development.rst
+    state.rst
     process_pool.rst
     jobs.rst
     roles.rst

docs/source/middleware/state.rst

@@ -0,0 +1,43 @@
+Middleware State Directories
+############################
+
+.. contents:: Table of Contents
+    :depth: 4
+
+The middlewared process stores state in several directories on the local filesystem. There
+are many situations in which the developer may opt to store state information outside of the
+configuration database. In this case the onus is on the developer to choose an appropriate
+location for this state based on persistence requirements.
+
+
+Volatile state
+**************
+
+Volatile middleware state is stored in the middleware run directory `/var/run/middleware`.
+The expected permissions on the volatile state directory are 0o755. This is typically where
+sentinel files should be placed. This is defined by `MIDDLEWARE_RUN_DIR` in `middlewared/utils`.
+
+
+Persistent state
+****************
+
+There are several directories that are used to store persistent state related to the middlewared
+process and TrueNAS servers.
+
+`/data` -- this ZFS dataset contains the TrueNAS configuration file `freenas-v1.db` and various install-specific
+configuration files that must persist across TrueNAS upgrades. Items that need to be included in the configuration
+tarball should generally be placed here. Permissions on this directory must be 0o755, but many files here should
+be set to 0o700. All files and directories should be owned by root:root.
+
+`/data/subsystems` -- this directory contains application-specific configuration that must persist between
+installs that is not suitable for datastore insertion. The convention is to create a new directory with the name of
+the middleware plugin that needs persistent state. Configuration information stored in these directories must be
+included in the TrueNAS configuration backup and restored on configuration upload.
+
+`/var/lib/truenas-middleware` -- this directory contains persistent middleware state that is applicable to the
+current boot environment only. It is a safe place to store data that we want to persist across reboots, but not
+across upgrades. This is defined by the `MIDDLEWARE_BOOT_ENV_STATE_DIR` in `middlewared/utils`. The permissions
+on this directory should be 0o755 and it should be owned by root:root.
+
+`/root` -- this dataset contains the middleware directory services cache. The permissions on this directory
+should be 0o700 and it should be owned by root:root.

src/middlewared/middlewared/alert/source/audit.py

@@ -14,13 +14,6 @@ class AuditBackendSetupAlertClass(AlertClass, SimpleOneShotAlertClass):
     text = "Audit service failed backend setup: %(service)s. See /var/log/middlewared.log"
 
 
-class AuditSetupAlertClass(AlertClass, SimpleOneShotAlertClass):
-    category = AlertCategory.AUDIT
-    level = AlertLevel.ERROR
-    title = "Audit Service Setup Failed"
-    text = "Audit service failed to complete setup. See /var/log/middlewared.log"
-
-
 # --------------- Monitored Alerts ----------------
 class AuditServiceHealthAlertClass(AlertClass):
     category = AlertCategory.AUDIT

src/middlewared/middlewared/plugins/audit/__init__.py

@@ -1,13 +1,16 @@
+async def init_audit(middleware, event_type, args):
+    if await middleware.call('system.boot_env_first_boot'):
+        try:
+            await middleware.call("audit.setup")
+        except Exception:
+            middleware.logger.error("Failed to perform setup tasks for auditing.", exc_info=True)
+
+
 async def setup(middleware):
+    middleware.event_subscribe('system.ready', init_audit)
+
     try:
         # Set up connections to the auditing databases
         await middleware.call("auditbackend.setup")
     except Exception:
         middleware.logger.error("Failed to set up auditing backend.", exc_info=True)
-    if await middleware.call("keyvalue.get", "run_migration", False):
-        # If this is an upgrade then free up space used by refreservation on
-        # deactivated boot environments
-        try:
-            await middleware.call("audit.setup")
-        except Exception:
-            middleware.logger.error("Failed to perform setup tasks for auditing.", exc_info=True)

src/middlewared/middlewared/plugins/audit/audit.py

@@ -550,9 +550,9 @@ async def setup(self):
         configuration to the current boot environment.
         """
         try:
-            os.mkdir(AUDIT_REPORTS_DIR, 0o700)
+            await self.middleware.run_in_thread(os.mkdir, AUDIT_REPORTS_DIR, 0o700)
         except FileExistsError:
-            os.chmod(AUDIT_REPORTS_DIR, 0o700)
+            await self.middleware.run_in_thread(os.chmod, AUDIT_REPORTS_DIR, 0o700)
 
         cur = await self.middleware.call('audit.get_audit_dataset')
         parent = os.path.dirname(cur['id'])
@@ -589,13 +589,10 @@ async def setup(self):
                     'cleanup may be required', ds['id'], exc_info=True
                 )
 
-        # Dismiss any existing AuditSetup one-shot alerts
-        await self.middleware.call('alert.oneshot_delete', 'AuditSetup', None)
         audit_config = await self.middleware.call('audit.config')
         try:
             await self.middleware.call('audit.update_audit_dataset', audit_config)
         except Exception:
-            await self.middleware.call('alert.oneshot_create', 'AuditSetup', None)
             self.logger.error('Failed to apply auditing dataset configuration.', exc_info=True)
 
         # Generate the initial truenas_verify file

src/middlewared/middlewared/plugins/system/__init__.py

@@ -3,7 +3,7 @@
 
 from middlewared.utils import BOOTREADY
 
-from .utils import FIRST_INSTALL_SENTINEL, lifecycle_conf
+from .utils import FIRST_INSTALL_SENTINEL, BOOTENV_FIRSTBOOT_SENTINEL, lifecycle_conf
 
 
 def firstboot(middleware):
@@ -20,6 +20,15 @@ def firstboot(middleware):
             middleware.call_sync('datastore.update', 'system.advanced', config['id'], {'adv_autotune': True})
 
 
+def firstboot_after_upgrade(middleware):
+    if not os.path.exists(BOOTENV_FIRSTBOOT_SENTINEL):
+        os.makedirs(os.path.dirname(BOOTENV_FIRSTBOOT_SENTINEL), mode=0o700)
+        with open(BOOTENV_FIRSTBOOT_SENTINEL, 'w'):
+            pass
+
+        lifecycle_conf.SYSTEM_BOOT_ENV_FIRST_BOOT = True
+
+
 def read_system_boot_id(middleware):
     try:
         with open('/proc/sys/kernel/random/boot_id', 'r') as f:
@@ -36,6 +45,7 @@ async def setup(middleware):
     middleware.event_register('system.shutdown', 'Started shutdown process', roles=['SYSTEM_GENERAL_READ'])
 
     await middleware.run_in_thread(firstboot, middleware)
+    await middleware.run_in_thread(firstboot_after_upgrade, middleware)
 
     settings = await middleware.call('system.general.config')
     middleware.logger.debug('Setting timezone to %r', settings['timezone'])

src/middlewared/middlewared/plugins/system/lifecycle.py

@@ -23,6 +23,11 @@ class SystemService(Service):
     async def first_boot(self):
         return lifecycle_conf.SYSTEM_FIRST_BOOT
 
+    @private
+    async def boot_env_first_boot(self):
+        # First boot after upgrading server
+        return lifecycle_conf.SYSTEM_BOOT_ENV_FIRST_BOOT
+
     @api_method(
         SystemBootIdArgs,
         SystemBootIdResult,

src/middlewared/middlewared/plugins/system/utils.py

@@ -1,11 +1,12 @@
 import os
 import re
 
-from middlewared.utils import MIDDLEWARE_RUN_DIR
+from middlewared.utils import MIDDLEWARE_RUN_DIR, MIDDLEWARE_BOOT_ENV_STATE_DIR
 
 
 DEBUG_MAX_SIZE = 30
 FIRST_INSTALL_SENTINEL = '/data/first-boot'
+BOOTENV_FIRSTBOOT_SENTINEL = os.path.join(MIDDLEWARE_BOOT_ENV_STATE_DIR, '.first-boot')
 RE_KDUMP_CONFIGURED = re.compile(r'current state\s*:\s*(ready to kdump)', flags=re.M)
 
 
@@ -17,6 +18,8 @@ def __init__(self):
         self.SYSTEM_READY = False
         # Flag telling whether the system is shutting down
         self.SYSTEM_SHUTTING_DOWN = False
+        self.SYSTEM_BOOT_ENV_FIRST_BOOT = False
+        # Flag telling whether post-install
 
 
 def get_debug_execution_dir(system_dataset_path: str, iteration: int = 0) -> str:

src/middlewared/middlewared/utils/__init__.py

@@ -34,6 +34,7 @@ class ProductNames:
 
 MID_PID = None
 MIDDLEWARE_RUN_DIR = '/var/run/middleware'
+MIDDLEWARE_BOOT_ENV_STATE_DIR = '/var/lib/truenas-middleware'
 MIDDLEWARE_STARTED_SENTINEL_PATH = f'{MIDDLEWARE_RUN_DIR}/middlewared-started'
 BOOTREADY = f'{MIDDLEWARE_RUN_DIR}/.bootready'
 BOOT_POOL_NAME_VALID = ['freenas-boot', 'boot-pool']

tests/api2/test_audit_alerts.py

@@ -81,29 +81,6 @@ def test_audit_backend_alert(setup_state):
     assert class_alerts[0]['formatted'].startswith("Audit service failed backend setup"), class_alerts
 
 
-@pytest.mark.parametrize(
-    'setup_state', [
-        [None, 'AuditSetup', 'audit.setup']
-    ],
-    indirect=True
-)
-def test_audit_setup_alert(setup_state):
-    with mock("audit.update_audit_dataset", """
-        from middlewared.service import private
-        @private
-        async def mock(self, new):
-            raise Exception()
-    """):
-        unused, alert_class, audit_method = setup_state
-        call(audit_method)
-        sleep(1)
-        alerts = call("alert.list")
-        class_alerts = [alert for alert in alerts if alert['klass'] == alert_class]
-        assert len(class_alerts) > 0, class_alerts
-        assert class_alerts[0]['klass'] == 'AuditSetup', class_alerts
-        assert class_alerts[0]['formatted'].startswith("Audit service failed to complete setup"), class_alerts
-
-
 def test_audit_health_monitor_alert():
     with mock("auditbackend.query", """
         from middlewared.service import private