Make this code run on python3.9 (October, 2020), the default on rockylinux9 and

the os that runs our agents.

Should be no functional difference in the changes made below.

joshua_model.py was barfing on call to getchildren removed in 3.9
triggered by attempts at changing xml processing.

Most of the below changes came via

   pip install pyupgrade
   pyupgrade --py39-plus .
   find . -name "*.py" -exec pyupgrade --py39-plus {} +

On changes like the below change:

  diff --git a/joshua/joshua.py b/joshua/joshua.py
  index 8065be3..659821d 100755
  --- a/joshua/joshua.py
  +++ b/joshua/joshua.py
  @@ -41,7 +41,7 @@ def get_username():
   def format_ensemble(e, props):
       return "  %-50s %s" % (
           e,
  -        " ".join("{}={}".format(k, v) for k, v in sorted(props.items())),
  +        " ".join(f"{k}={v}" for k, v in sorted(props.items())),
       )

On these sorts of changes:

  @@ -116,11 +116,11 @@ def start_ensemble(
           properties["timeout"] = timeout

       if fail_fast > 0 and not no_fail_fast:
  -        print("Note: Ensemble will complete after {} failed results.".format(fail_fast))
  +        print(f"Note: Ensemble will complete after {fail_fast} failed results.")
         properties["fail_fast"] = fail_fast

Replaces str.format() with an f-string for a print() statement. Modern,
idiomtic python.

Nothing wrong w/ using set in the below but pyupgrade thinks the change
more 'direct':

     if not yes and not dryrun:
         response = input("Do you want to delete these ensembles [y/n]? ")
-        if response.strip().lower() not in set(["y", "yes"]):
+        if response.strip().lower() not in {"y", "yes"}:
             print("Negative response received. Not performing deletion.")
             return

On the below change:

  -class TimeoutFuture(object):
  +class TimeoutFuture:
       def __init__(self, timeout):
           self.cb_list = []
           self.timer = threading.Timer(timeout, self._do_on_ready)

In Python 3, classes implicitly inherit from object if no other base class is specified. So, (object) is redundant.

On the below, 'r' is default:

           if env.get('HOSTNAME', False) and os.path.isfile(k8s_namespace_file):
               pod_name = env['HOSTNAME']
  -            with open(k8s_namespace_file, 'r') as f:
  +            with open(k8s_namespace_file) as f:
                   namespace = f.read()

Below is main change. Element.getchildren() was deprecated in earlier
Python versions and removed in Python 3.9.

 def unwrap_message(text):
     root = ET.fromstring(text)
-    return root.getchildren()[0].attrib
+    return next(iter(root)).attrib

On the blow, PEP 585 - Type Hinting Generics In Standard Collections: This PEP,
implemented starting in Python 3.9, allows the direct use of built-in collection
types (like list, dict, tuple, set, etc.) as generic types in type annotations.

  -def list_and_watch_active_ensembles(tr) -> Tuple[List[str], fdb.Future]:
  +def list_and_watch_active_ensembles(tr) -> tuple[list[str], fdb.Future]:
       return _list_and_watch_ensembles(tr, dir_active, dir_active_changes)

On the below, in Python 3 (specifically starting from Python 3.3), IOError
became an alias for OSError. This means that IOError is essentially the same
as OSError. OSError is now the canonical name for this broad category of
OS-related errors (which includes I/O errors).

-    except IOError:
+    except OSError:
         # This is not our process, so we can't open the file.
         return dict()

On the below, PEP 8 Compliance: The unittest module in Python's standard
library has adopted snake_case method names (like assertEqual) as the
standard, following PEP 8 guidelines.

     def test_mark_env(self):
         env = mark_environment(dict())
-        self.assertEquals(os.getpid(), int(env[VAR_NAME]))
+        self.assertEqual(os.getpid(), int(env[VAR_NAME]))

Author: michael stack

joshua/joshua.py

@@ -41,7 +41,7 @@ def get_username():
 def format_ensemble(e, props):
     return "  %-50s %s" % (
         e,
-        " ".join("{}={}".format(k, v) for k, v in sorted(props.items())),
+        " ".join(f"{k}={v}" for k, v in sorted(props.items())),
     )
 
 
@@ -75,7 +75,7 @@ def list_active_ensembles(
             for props in joshua_model.show_in_progress(e):
                 print(
                     "\t{}".format(
-                        " ".join("{}={}".format(k, v) for k, v in sorted(props.items()))
+                        " ".join(f"{k}={v}" for k, v in sorted(props.items()))
                     )
                 )
 
@@ -116,11 +116,11 @@ def start_ensemble(
         properties["timeout"] = timeout
 
     if fail_fast > 0 and not no_fail_fast:
-        print("Note: Ensemble will complete after {} failed results.".format(fail_fast))
+        print(f"Note: Ensemble will complete after {fail_fast} failed results.")
         properties["fail_fast"] = fail_fast
 
     if max_runs > 0 and not no_max_runs:
-        print("Note: Ensemble will complete after {} runs.".format(max_runs))
+        print(f"Note: Ensemble will complete after {max_runs} runs.")
         properties["max_runs"] = max_runs
 
     if command:
@@ -304,7 +304,7 @@ def _delete_helper(to_delete, yes=False, dryrun=False, sanity=False):
 
     if not yes and not dryrun:
         response = input("Do you want to delete these ensembles [y/n]? ")
-        if response.strip().lower() not in set(["y", "yes"]):
+        if response.strip().lower() not in {"y", "yes"}:
             print("Negative response received. Not performing deletion.")
             return
 
@@ -381,10 +381,10 @@ def download_ensemble(ensemble, out=None, force=False, sanity=False, **args):
 
     if out is None:
         out_file = os.path.abspath(
-            os.path.join(os.getcwd(), "{}.tar.gz".format(ensemble))
+            os.path.join(os.getcwd(), f"{ensemble}.tar.gz")
         )
     elif os.path.isdir(out):
-        out_file = os.path.abspath(os.path.join(out, "{}.tar.gz".format(ensemble)))
+        out_file = os.path.abspath(os.path.join(out, f"{ensemble}.tar.gz"))
     else:
         out_file = os.path.abspath(out)
         if not os.path.isdir(os.path.dirname(out_file)):
@@ -401,15 +401,15 @@ def download_ensemble(ensemble, out=None, force=False, sanity=False, **args):
             return
 
     if not force and os.path.isfile(out_file):
-        print("File {} already exists. Refusing to overwrite file.".format(out_file))
+        print(f"File {out_file} already exists. Refusing to overwrite file.")
         return
 
     # Treat slash characters as subdirectories
     ensemble_dir = os.path.dirname(ensemble)
     if ensemble_dir:
         os.mkdir(ensemble_dir)
 
-    print("Downloading ensemble {} into {}...".format(ensemble, out_file))
+    print(f"Downloading ensemble {ensemble} into {out_file}...")
     with open(out_file, "wb") as fout:
         joshua_model.get_ensemble_data(ensemble_id=ensemble, outfile=fout)
     print("Download completed")

joshua/joshua_agent.py

@@ -72,7 +72,7 @@ def __repr__(self):
 
 
 # This is used to handle waiting for a given amount of time.
-class TimeoutFuture(object):
+class TimeoutFuture:
     def __init__(self, timeout):
         self.cb_list = []
         self.timer = threading.Timer(timeout, self._do_on_ready)
@@ -263,7 +263,7 @@ def tar_artifacts(ensemble, seed, sources, dest, work_dir=None):
         )
     try:
         # Create a temporary directory in the destination where we will store the results.
-        out_name = "joshua-run-{0}-{1}".format(ensemble, seed)
+        out_name = f"joshua-run-{ensemble}-{seed}"
         tmpdir = os.path.join(work_dir, out_name)
         os.makedirs(tmpdir)
 
@@ -413,7 +413,7 @@ def _cancelled(self):
     def run(self, command, cwd, env):
         cmd_path = os.path.join(cwd, command[0])
         if not os.path.exists(cmd_path):
-            log("{} doesn't exist".format(cmd_path))
+            log(f"{cmd_path} doesn't exist")
             return
         process = subprocess.Popen(
             command,
@@ -528,7 +528,7 @@ def _run_ensemble(
         k8s_namespace_file = "/var/run/secrets/kubernetes.io/serviceaccount/namespace"
         if env.get('HOSTNAME', False) and os.path.isfile(k8s_namespace_file):
             pod_name = env['HOSTNAME']
-            with open(k8s_namespace_file, 'r') as f:
+            with open(k8s_namespace_file) as f:
                 namespace = f.read()
 
             # Get the cluster config
@@ -554,7 +554,7 @@ def _run_ensemble(
         # Set environment variable to use the created temporary directory as its temporary directory.
         env["TMP"] = os.path.join(where, "tmp")
 
-        log("{} {} {}".format(ensemble, seed, command))
+        log(f"{ensemble} {seed} {command}")
 
         # Run the test and log output
         process = subprocess.Popen(
@@ -577,7 +577,7 @@ def _run_ensemble(
             try:
                 output, _ = process.communicate(timeout=1)
                 retcode = process.poll()
-                log("exit code: {}".format(retcode))
+                log(f"exit code: {retcode}")
                 # output = output.decode('utf-8')
 
                 break
@@ -642,7 +642,7 @@ def _run_ensemble(
         try:
             i = 0
             while os.path.exists(to_write):
-                to_write = os.path.join(where, "tmp", "console-{0}.log".format(i))
+                to_write = os.path.join(where, "tmp", f"console-{i}.log")
                 i += 1
 
             with open(to_write, "wb") as fout:
@@ -808,7 +808,7 @@ def agent(
         while True:
             # Break if the stop file is defined and present
             if stop_file and os.path.exists(stop_file):
-                log("Exiting due to existing stopfile: {}".format(stop_file))
+                log(f"Exiting due to existing stopfile: {stop_file}")
                 break
             # Break if requested
             if stopAgent():
@@ -885,7 +885,7 @@ def agent(
             # Throw away local state for ensembles that are no longer active
             local_ensemble_dirs = set(os.listdir(ensemble_dir(basepath=work_dir)))
             for e in (local_ensemble_dirs - set(ensembles)) - set(sanity_ensembles):
-                log("removing {} {}".format(e, ensemble_dir(e, basepath=work_dir)))
+                log(f"removing {e} {ensemble_dir(e, basepath=work_dir)}")
                 shutil.rmtree(
                     ensemble_dir(e, basepath=work_dir), True
                 )  # SOMEDAY: this sometimes throws errors, but we don't know why and it isn't that important
@@ -906,7 +906,7 @@ def agent(
                 try:
                     watch.wait_for_any(watch, sanity_watch, TimeoutFuture(1.0))
                 except Exception as e:
-                    log("watch error: {}".format(e))
+                    log(f"watch error: {e}")
                     watch = None
                     time.sleep(1.0)
 

joshua/joshua_model.py

@@ -183,7 +183,7 @@ def is_message(text):
 
 def unwrap_message(text):
     root = ET.fromstring(text)
-    return root.getchildren()[0].attrib
+    return next(iter(root)).attrib
 
 
 def load_datetime(string):
@@ -224,7 +224,7 @@ def identify_existing_ensembles(tr, ensembles):
 
 
 @transactional
-def list_and_watch_active_ensembles(tr) -> Tuple[List[str], fdb.Future]:
+def list_and_watch_active_ensembles(tr) -> tuple[list[str], fdb.Future]:
     return _list_and_watch_ensembles(tr, dir_active, dir_active_changes)
 
 
@@ -258,7 +258,7 @@ def _unpack_property(ensemble, key, value, into):
         into[t[1]] = struct.unpack("<Q", value)[0]
 
 
-def _list_ensembles(tr, dir) -> List[Tuple[str, Dict]]:
+def _list_ensembles(tr, dir) -> list[tuple[str, dict]]:
     prop_reads = []
     for k, v in tr[dir.range()]:
         (ensemble,) = dir.unpack(k)
@@ -281,17 +281,17 @@ def _list_ensembles(tr, dir) -> List[Tuple[str, Dict]]:
 
 
 @transactional
-def list_active_ensembles(tr) -> List[Tuple[str, Dict]]:
+def list_active_ensembles(tr) -> list[tuple[str, dict]]:
     return _list_ensembles(tr, dir_active)
 
 
 @transactional
-def list_sanity_ensembles(tr) -> List[Tuple[str, Dict]]:
+def list_sanity_ensembles(tr) -> list[tuple[str, dict]]:
     return _list_ensembles(tr, dir_sanity)
 
 
-def list_all_ensembles() -> List[Tuple[str, Dict]]:
-    ensembles: List[Tuple[str, Dict]] = []
+def list_all_ensembles() -> list[tuple[str, dict]]:
+    ensembles: list[tuple[str, dict]] = []
     r = dir_all_ensembles.range()
     start = r.start
     tr = db.create_transaction()
@@ -405,7 +405,7 @@ def _create_ensemble(tr, ensemble_id, properties, sanity=False):
     dir, changes = get_dir_changes(sanity)
 
     if tr[dir_all_ensembles[ensemble_id]] != None:
-        print("{} already inserted".format(ensemble_id))
+        print(f"{ensemble_id} already inserted")
         return  # Already inserted
     tr[dir_all_ensembles[ensemble_id]] = b""
     for k, v in properties.items():
@@ -458,7 +458,7 @@ def stop_user_ensembles(username, sanity=False):
 
 def get_active_ensembles(
     stopped, sanity=False, username=None
-) -> List[Tuple[str, Dict]]:
+) -> list[tuple[str, dict]]:
     if stopped:
         ensemble_list = list_all_ensembles()
     elif sanity:
@@ -655,7 +655,7 @@ def _get_snap_counter(tr: fdb.Transaction, ensemble_id: str, counter: str) -> in
 
 def _get_seeds_and_heartbeats(
     ensemble_id: str, tr: fdb.Transaction
-) -> List[Tuple[int, float]]:
+) -> list[tuple[int, float]]:
     result = []
     for k, v in tr.snapshot[dir_ensemble_incomplete[ensemble_id]["heartbeat"].range()]:
         (seed,) = dir_ensemble_incomplete[ensemble_id]["heartbeat"].unpack(k)
@@ -718,7 +718,7 @@ def should_run_ensemble(tr: fdb.Transaction, ensemble_id: str) -> bool:
 
 
 @transactional
-def show_in_progress(tr: fdb.Transaction, ensemble_id: str) -> List[Tuple[int, Dict]]:
+def show_in_progress(tr: fdb.Transaction, ensemble_id: str) -> list[tuple[int, dict]]:
     """
     Returns a list of properties for in progress tests
     """

joshua/process_handling.py

@@ -107,7 +107,7 @@ def get_environment(pid):
                     var_strs,
                 )
             )
-    except IOError:
+    except OSError:
         # This is not our process, so we can't open the file.
         return dict()
 
@@ -238,7 +238,7 @@ def test_check_alive(self):
 
     def test_mark_env(self):
         env = mark_environment(dict())
-        self.assertEquals(os.getpid(), int(env[VAR_NAME]))
+        self.assertEqual(os.getpid(), int(env[VAR_NAME]))
 
     def test_get_all_pids(self):
         if sys.platform != "linux2":
@@ -268,9 +268,9 @@ def test_get_environment(self):
             # Make sure the environment for this process is the same
             # as we know it to be.
             env = get_environment(str(os.getpid()))
-            self.assertEquals(env, os.environ)
+            self.assertEqual(env, os.environ)
             env = get_environment(os.getpid())
-            self.assertEquals(env, os.environ)
+            self.assertEqual(env, os.environ)
 
     def test_retrieve_children(self):
         if sys.platform != "linux2":
@@ -285,7 +285,7 @@ def test_retrieve_children(self):
                     stderr=subprocess.PIPE,
                 )
             pids = retrieve_children()
-            self.assertEquals(len(pids), 10)
+            self.assertEqual(len(pids), 10)
 
     def test_kill_all_children(self):
         if sys.platform != "linux2":
@@ -300,7 +300,7 @@ def test_kill_all_children(self):
                     stderr=subprocess.PIPE,
                 )
             self.assertTrue(kill_all_children())
-            self.assertEquals(len(retrieve_children()), 0)
+            self.assertEqual(len(retrieve_children()), 0)
 
     def test_wait_for_death(self):
         process = subprocess.Popen(

joshua_done_test.py

@@ -10,7 +10,7 @@
 from typing import List
 
 @fdb.transactional
-def write_ensemble_data(tr, path: List[str]):
+def write_ensemble_data(tr, path: list[str]):
     root_dir = fdb.directory.create_or_open(tr, tuple(path))
     num = os.getenv('JOSHUA_SEED', None)
     has_joshua_seed = num is not None
@@ -28,8 +28,8 @@ def write_ensemble_data(tr, path: List[str]):
     parser.add_argument('ensemble_dir')
     parser.add_argument('cluster_file')
     args = parser.parse_args()
-    dirPath: List[str] = args.ensemble_dir.split(',')
+    dirPath: list[str] = args.ensemble_dir.split(',')
     print('dirPath = ({})'.format(",".join(dirPath)))
-    print('clusterFile = {}'.format(args.cluster_file))
+    print(f'clusterFile = {args.cluster_file}')
     db = fdb.open(args.cluster_file if args.cluster_file != 'None' else None)
     write_ensemble_data(db, dirPath)

setup.cfg

@@ -1,2 +1,2 @@
 [bdist_wheel]
-python-tag = py38
+python-tag = py39

setup.py

@@ -70,7 +70,7 @@
         + module.platforms
         + [
             "Programming Language :: Python :: 3",
-            "Programming Language :: Python :: 3.8",
+            "Programming Language :: Python :: 3.9",
             "Programming Language :: Python :: Implementation :: CPython",
         ],
     )

test_joshua_model.py

@@ -50,7 +50,7 @@ def with_script(tmp_path, script_contents):
         return factory
 
     def add_bash_script(self, name, contents):
-        script = "#!/bin/bash\n{}".format(contents).encode("utf-8")
+        script = f"#!/bin/bash\n{contents}".encode("utf-8")
         entry = tarfile.TarInfo(name)
         entry.mode = 0o755
         entry.size = len(script)
@@ -119,9 +119,9 @@ def fdb_cluster():
     port = getFreePort()
     cluster_file = os.path.join(tmp_dir, "fdb.cluster")
     with open(cluster_file, "w") as f:
-        f.write("abdcefg:[email protected]:{}".format(port))
+        f.write(f"abdcefg:[email protected]:{port}")
     proc = subprocess.Popen(
-        ["fdbserver", "-p", "auto:{}".format(port), "-C", cluster_file], cwd=tmp_dir
+        ["fdbserver", "-p", f"auto:{port}", "-C", cluster_file], cwd=tmp_dir
     )
 
     subprocess.check_output(

webapp/config.py

@@ -25,7 +25,7 @@
 load_dotenv(os.path.join(basedir, '.env'))
 
 
-class Config(object):
+class Config:
     SECRET_KEY = os.environ.get('SECRET_KEY') or 'you-will-never-guess'
     SQLALCHEMY_DATABASE_URI = os.environ.get('DATABASE_URL') or \
         'sqlite:///' + os.path.join(basedir, 'db.sqlite')

webapp/joshua_webapp/api.py

@@ -87,9 +87,9 @@ def upload_ensemble():
     try:
         properties = schema.load(request_data)
     except Exception as err:
-        app.logger.info('api_upload: Validation error: {}'.format(err))
+        app.logger.info(f'api_upload: Validation error: {err}')
         return {
-            "message": 'api_upload: Post field validation error: {}'.format(err)
+            "message": f'api_upload: Post field validation error: {err}'
         }, 422
     fileobj = request_files['file']
     filename = secure_filename(fileobj.filename)
@@ -102,7 +102,7 @@ def upload_ensemble():
     if not tarfile.is_tarfile(saved_file):
         os.remove(saved_file)
         app.logger.info(
-            'api_upload: not a valid tar file: {}'.format(saved_file))
+            f'api_upload: not a valid tar file: {saved_file}')
         return {"error": 'api_upload: not a valid tar file'}, 400
 
     # convert to non-unicode string for username
@@ -128,7 +128,7 @@ def stop_ensemble(ensemble):
     properties = joshua_model.get_ensemble_properties(ensemble)
     sanity = properties[
         'sanity'] if properties and 'sanity' in properties else True
-    app.logger.info('Stop ensemble {} {}'.format(ensemble, sanity))
+    app.logger.info(f'Stop ensemble {ensemble} {sanity}')
     joshua_model.stop_ensemble(ensemble, sanity=sanity)
     return jsonify('OK'), 200
 
@@ -139,6 +139,6 @@ def resume_ensemble(ensemble):
     properties = joshua_model.get_ensemble_properties(ensemble)
     sanity = properties[
         'sanity'] if properties and 'sanity' in properties else True
-    app.logger.info('Resume ensemble {} {}'.format(ensemble, sanity))
+    app.logger.info(f'Resume ensemble {ensemble} {sanity}')
     joshua_model.resume_ensemble(ensemble, sanity=sanity)
     return jsonify('OK'), 200