Merge pull request xcat2#239 from Obihoernchen/ci

Add CI pipeline

Author: jjohnson42

.github/workflows/ci.yml

@@ -0,0 +1,64 @@
+name: CI
+
+on:
+  push:
+  pull_request:
+
+permissions:
+  contents: read
+
+jobs:
+  ShellCheck:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+      - name: Run ShellCheck (errors only)
+        # Check every tracked file that has a .sh extension or an sh/bash
+        # shebang. SC2148 (missing shebang) is excluded because many .sh
+        # files are sourced fragments or dracut hooks; ShellCheck then
+        # falls back to checking them as bash.
+        run: |
+          {
+            git ls-files '*.sh'
+            git ls-files | while IFS= read -r f; do
+              [ -f "$f" ] || continue
+              head -c 200 "$f" | head -n 1 | \
+                grep -qE '^#!.*[/ ](sh|bash|dash|ash|ksh)([ \t]|$)' && echo "$f"
+            done
+          } | sort -u | xargs -d '\n' shellcheck --severity=error --exclude=SC2148
+
+  python-compileall:
+    name: Python compileall
+    runs-on: ubuntu-latest
+    env:
+      # One entry per Python version shipped by the distros confluent
+      # targets, limited to versions actions/setup-python still provides
+      # on current runners (sles15/alma8 ship 3.6, which is unavailable).
+      # Newline-separated so it feeds both setup-python (multiline input)
+      # and the shell loop below (word-split on whitespace).
+      PYTHON_VERSIONS: |
+        3.8
+        3.9
+        3.10
+        3.12
+        3.13
+        3.14
+    steps:
+      - uses: actions/checkout@v7
+      - uses: actions/setup-python@v6
+        with:
+          python-version: ${{ env.PYTHON_VERSIONS }}
+      - name: Compile all Python files
+        run: |
+          rc=0
+          for v in $PYTHON_VERSIONS; do
+            echo "::group::Python $v"
+            if "python$v" -W error -m compileall -q -x '/\.git/' .; then
+              echo "::endgroup::"
+            else
+              echo "::endgroup::"
+              echo "::error::Python $v compileall failed"
+              rc=1
+            fi
+          done
+          exit "$rc"

confluent_osdeploy/el7-diskless/profiles/default/scripts/image2disk.py

@@ -173,8 +173,8 @@ def fixup(rootdir, vols):
     for vol in vols:
         if vol['mount'] == '/boot/efi':
             targdev = vol['targetdisk']
-            partnum = re.search('(\d+)$', targdev).group(1)
-            targblock = re.search('(.*)\d+$', targdev).group(1)
+            partnum = re.search(r'(\d+)$', targdev).group(1)
+            targblock = re.search(r'(.*)\d+$', targdev).group(1)
     if targblock:
         if targblock.endswith('p') and 'nvme' in targblock:
             targblock = targblock[:-1]
@@ -231,7 +231,7 @@ def install_to_disk(imgpath):
             deflvmsize += fs['initsize']
             minlvmsize += fs['minsize']
         else:
-            plainvols[int(re.search('(\d+)$', fs['device'])[0])] = fs
+            plainvols[int(re.search(r'(\d+)$', fs['device'])[0])] = fs
     with open('/tmp/installdisk') as diskin:
         instdisk = diskin.read()
     instdisk = '/dev/' + instdisk

confluent_osdeploy/ubuntu20.04-diskless/profiles/default/scripts/image2disk.py

@@ -204,8 +204,8 @@ def fixup(rootdir, vols):
     for vol in vols:
         if vol['mount'] == '/boot/efi':
             targdev = vol['targetdisk']
-            partnum = re.search('(\d+)$', targdev).group(1)
-            targblock = re.search('(.*)\d+$', targdev).group(1)
+            partnum = re.search(r'(\d+)$', targdev).group(1)
+            targblock = re.search(r'(.*)\d+$', targdev).group(1)
     if targblock:
         if targblock.endswith('p') and 'nvme' in targblock:
             targblock = targblock[:-1]
@@ -263,7 +263,7 @@ def install_to_disk(imgpath):
             deflvmsize += fs['initsize']
             minlvmsize += fs['minsize']
         else:
-            plainvols[int(re.search('(\d+)$', fs['device'])[0])] = fs
+            plainvols[int(re.search(r'(\d+)$', fs['device'])[0])] = fs
     with open('/tmp/installdisk') as diskin:
         instdisk = diskin.read()
     instdisk = '/dev/' + instdisk

confluent_server/confluent/collective/manager.py

@@ -206,28 +206,30 @@ async def follow_leader(remote, leader):
         exitcause = await cfm.follow_channel(remote)
         newleader = exitcause.get('newleader', None)
     finally:
+        handled = False
         if cleanexit:
             log.log({'info': 'Previous following cleanly closed',
                      'subsystem': 'collective'})
-            return
-        if newleader:
+            handled = True
+        if not handled and newleader:
             log.log(
                 {'info': 'Previous leader directed us to join new leader {}'.format(newleader)})
             try:
                 if await connect_to_leader(None, get_myname(), newleader):
-                    return
+                    handled = True
             except Exception:
                 log.log({'error': 'Unknown error attempting to connect to {}, check trace log'.format(newleader), 'subsystem': 'collective'})
                 cfm.logException()
-        log.log({'info': 'Current leader ({0}) has disappeared, restarting '
-                         'collective membership'.format(leader), 'subsystem': 'collective'})
-        # The leader has folded, time to startup again...
-        follower = None
-        await cfm.stop_following()
-        currentleader = None
-        if retrythread is None:  # start a recovery
-            retrythread = tasks.spawn_task_after(
-                random.random(), start_collective)
+        if not handled:
+            log.log({'info': 'Current leader ({0}) has disappeared, restarting '
+                             'collective membership'.format(leader), 'subsystem': 'collective'})
+            # The leader has folded, time to startup again...
+            follower = None
+            await cfm.stop_following()
+            currentleader = None
+            if retrythread is None:  # start a recovery
+                retrythread = tasks.spawn_task_after(
+                    random.random(), start_collective)
 
 async def _create_tls_connection(host, port):
     cloop = asyncio.get_running_loop()

confluent_server/confluent/config/configmanager.py

@@ -782,6 +782,9 @@ async def relay_slaved_requests(name, listener):
                     msg = await lh.get_next_msg()
                 except Exception:
                     msg = None
+        except Exception:
+            # unexpected loss or misbehavior of the follower ends the relay
+            logException()
         finally:
             try:
                 listener.close()
@@ -798,10 +801,10 @@ async def relay_slaved_requests(name, listener):
                     *[_push_rpc(cfgstreams[s]['stream'], payload) for s in cfgstreams])
             if membership_callback:
                 membership_callback()
-            if not cfgstreams and not cfgleader:  # last one out, set cfgleader to boolean to mark dead collective
-                await stop_following(True)
-                return False
-            return True
+        if not cfgstreams and not cfgleader:  # last one out, set cfgleader to boolean to mark dead collective
+            await stop_following(True)
+            return False
+        return True
 
 lastheartbeat = None
 async def check_leader():

confluent_server/confluent/log.py

@@ -396,7 +396,7 @@ def initSizeRollingCount(self):
         prefix = baseName + "."
         if dirName not in self.dirContents or self.dirContents[dirName][1] < time.time():
             self.dirContents[dirName] = (os.listdir(dirName), time.time() + 5)
-        matchexp = re.compile(f'^{prefix}\.\d+$')
+        matchexp = re.compile(rf'^{prefix}\.\d+$')
         fileNames = [f for f in self.dirContents[dirName][0]
                      if matchexp.match(f)]
         plen = len(prefix)

confluent_server/confluent/plugins/hardwaremanagement/enos.py

@@ -152,12 +152,12 @@ def gather_data(configmanager, creds, node):
     nssh = enos_login(node=node, configmanager=configmanager, creds=creds)
     switch_lines = enos_version(ssh=nssh)
     switch_data = {}
-    sysinfo = {"Product name": {"regex": ".*RackSwitch (\w+)"},
-               "Serial Number": {"regex": "ESN\s*\w*\s*: ([\w-]+)"},
-               "Board Serial Number": {"regex": "Switch Serial No: (\w+)"},
-               "Model": {"regex": "MTM\s*\w*\s*: ([\w-]+)"},
-               "FRU Number": {"regex": "Hardware Part\s*\w*\s*: (\w+)"},
-               "Airflow": {"regex": "System Fan Airflow\s*\w*\s*: ([\w-]+)"},
+    sysinfo = {"Product name": {"regex": r".*RackSwitch (\w+)"},
+               "Serial Number": {"regex": r"ESN\s*\w*\s*: ([\w-]+)"},
+               "Board Serial Number": {"regex": r"Switch Serial No: (\w+)"},
+               "Model": {"regex": r"MTM\s*\w*\s*: ([\w-]+)"},
+               "FRU Number": {"regex": r"Hardware Part\s*\w*\s*: (\w+)"},
+               "Airflow": {"regex": r"System Fan Airflow\s*\w*\s*: ([\w-]+)"},
               }
 
     invinfo = {
@@ -233,7 +233,7 @@ def gather_data(configmanager, creds, node):
     sysfw = {"Software Version": "Unknown", "Boot kernel": "Unknown"}
     for line in switch_lines:
         for key in sysfw.keys():
-            regex = f"{key}\s*\w*\s* ([0-9.]+)"
+            regex = rf"{key}\s*\w*\s* ([0-9.]+)"
             match = re.match(re.compile(regex), line)
             if match:
                 sysfw[key] = match.group(1)
@@ -250,7 +250,7 @@ def gather_psus(data):
         # others are:
         # Internal  Power Supply: On
         if "Power Supply" in line:
-            match = re.match(re.compile("Power Supply (\d)+.*"), line)
+            match = re.match(re.compile(r"Power Supply (\d)+.*"), line)
             if match:
                 psu = match.group(1)
                 if psu not in psus:
@@ -280,7 +280,7 @@ def gather_fans(data):
     for line in data:
         # look for presence of fans
         if "Fan" in line:
-            match = re.match(re.compile("Fan (\d)+.*"), line)
+            match = re.match(re.compile(r"Fan (\d)+.*"), line)
             if match:
                 fan = match.group(1)
                 if match: