Test pipeline fixes and refresh. (#1305)

Author: bcipriano

.github/workflows/testing-pipeline.yml

@@ -7,90 +7,57 @@ on:
     branches: [ master ]
 
 jobs:
-  test_python_2019:
-    name: Run Python Unit Tests (CY2019)
-    runs-on: ubuntu-latest
-    container: aswf/ci-opencue:2019
-    steps:
-      - uses: actions/checkout@v3
-      - name: Run Python Tests
-        run: ci/run_python_tests.sh
-
-  test_cuebot_2019:
-    name: Build Cuebot and Run Unit Tests (CY2019)
-    runs-on: ubuntu-latest
-    container:
-      image: aswf/ci-opencue:2019
-    steps:
-    - uses: actions/checkout@v3
-    - name: Build with Gradle
-      run: |
-        chown -R aswfuser:aswfgroup .
-        su -c "cd cuebot && ./gradlew build --stacktrace --info" aswfuser
-
-  test_python_2020:
-    name: Run Python Unit Tests (CY2020)
+  test_python_2022:
+    name: Run Python Unit Tests (CY2022)
     runs-on: ubuntu-latest
-    container: aswf/ci-opencue:2020
+    container: aswf/ci-opencue:2022
     steps:
       - uses: actions/checkout@v3
       - name: Run Python Tests
-        run: ci/run_python_tests.sh
+        run: ci/run_python_tests.sh --no-gui
 
-  test_cuebot_2020:
-    name: Build Cuebot and Run Unit Tests (CY2020)
+  test_cuebot_2022:
+    name: Build Cuebot and Run Unit Tests (CY2022)
     runs-on: ubuntu-latest
     container:
-      image: aswf/ci-opencue:2020
+      image: aswf/ci-opencue:2022
     steps:
       - uses: actions/checkout@v3
       - name: Build with Gradle
         run: |
           chown -R aswfuser:aswfgroup .
           su -c "cd cuebot && ./gradlew build --stacktrace --info" aswfuser
 
-  test_python_2021:
-    name: Run Python Unit Tests (CY2021)
+  test_python_2023:
+    name: Run Python Unit Tests (CY2023)
     runs-on: ubuntu-latest
-    container: aswf/ci-opencue:2021
+    container: aswf/ci-opencue:2023
     steps:
       - uses: actions/checkout@v3
       - name: Run Python Tests
         run: ci/run_python_tests.sh
 
-  test_cuebot_2021:
-    name: Build Cuebot and Run Unit Tests (CY2021)
+  test_cuebot_2023:
+    name: Build Cuebot and Run Unit Tests (CY2023)
     runs-on: ubuntu-latest
     container:
-      image: aswf/ci-opencue:2021
+      image: aswf/ci-opencue:2023
     steps:
       - uses: actions/checkout@v3
       - name: Build with Gradle
         run: |
           chown -R aswfuser:aswfgroup .
           su -c "cd cuebot && ./gradlew build --stacktrace --info" aswfuser
 
-  test_python_2022:
-    name: Run Python Unit Tests (CY2022)
+  test_python2:
+    name: Run Python Unit Tests using Python2
     runs-on: ubuntu-latest
-    container: aswf/ci-opencue:2022
+    container: aswf/ci-opencue:2019
     steps:
       - uses: actions/checkout@v3
       - name: Run Python Tests
         run: ci/run_python_tests.sh
 
-  test_cuebot_2022:
-    name: Build Cuebot and Run Unit Tests (CY2022)
-    runs-on: ubuntu-latest
-    container:
-      image: aswf/ci-opencue:2022
-    steps:
-      - uses: actions/checkout@v3
-      - name: Build with Gradle
-        run: |
-          chown -R aswfuser:aswfgroup .
-          su -c "cd cuebot && ./gradlew build --stacktrace --info" aswfuser
-
   test_pyside6:
     name: Run CueGUI Tests using PySide6
     runs-on: ubuntu-latest
@@ -113,7 +80,7 @@ jobs:
     name: Test Documentation Build
     runs-on: ubuntu-latest
     container:
-      image: aswf/ci-opencue:2020
+      image: aswf/ci-opencue:2023
     steps:
       - uses: actions/checkout@v3
       - name: Run Sphinx build

ci/run_python_tests.sh

@@ -1,7 +1,13 @@
 #!/bin/bash
 
+# Script for running OpenCue unit tests with PySide2.
+#
+# This script is written to be run within the OpenCue GitHub Actions environment.
+# See `.github/workflows/testing-pipeline.yml`.
+
 set -e
 
+args=("$@")
 python_version=$(python -V 2>&1)
 echo "Will run tests using ${python_version}"
 
@@ -23,6 +29,6 @@ PYTHONPATH=pycue:pyoutline python cuesubmit/setup.py test
 python rqd/setup.py test
 
 # Xvfb no longer supports Python 2.
-if [[ "$python_version" =~ "Python 3" ]]; then
+if [[ "$python_version" =~ "Python 3" && ${args[0]} != "--no-gui" ]]; then
   ci/run_gui_test.sh
 fi

ci/run_python_tests_pyside6.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+# Script for running OpenCue unit tests with PySide6.
+#
+# This script is written to be run within the OpenCue GitHub Actions environment.
+# See `.github/workflows/testing-pipeline.yml`.
+
+set -e
+
+python_version=$(python -V 2>&1)
+echo "Will run tests using ${python_version}"
+
+# NOTE: To run this in an almalinux environment, install these packages:
+# yum -y install \
+#   dbus-libs \
+#   fontconfig \
+#   gcc \
+#   libxkbcommon-x11 \
+#   mesa-libEGL-devel \
+#   python-devel \
+#   which \
+#   xcb-util-keysyms \
+#   xcb-util-image \
+#   xcb-util-renderutil \
+#   xcb-util-wm \
+#   Xvfb
+
+# Install Python requirements.
+python3 -m pip install --user -r requirements.txt -r requirements_gui.txt
+# Replace PySide2 with PySide6.
+python3 -m pip uninstall -y PySide2
+python3 -m pip install --user PySide6==6.3.2
+
+# Protos need to have their Python code generated in order for tests to pass.
+python -m grpc_tools.protoc -I=proto/ --python_out=pycue/opencue/compiled_proto --grpc_python_out=pycue/opencue/compiled_proto proto/*.proto
+python -m grpc_tools.protoc -I=proto/ --python_out=rqd/rqd/compiled_proto --grpc_python_out=rqd/rqd/compiled_proto proto/*.proto
+
+# Fix compiled proto code for Python 3.
+2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py
+2to3 -wn -f import rqd/rqd/compiled_proto/*_pb2*.py
+
+python pycue/setup.py test
+PYTHONPATH=pycue python pyoutline/setup.py test
+PYTHONPATH=pycue python cueadmin/setup.py test
+PYTHONPATH=pycue:pyoutline python cuesubmit/setup.py test
+python rqd/setup.py test
+
+ci/run_gui_test.sh

cuebot/src/main/resources/conf/ddl/postgres/seed_data.sql

@@ -1,6 +1,6 @@
 Insert into SHOW (PK_SHOW,STR_NAME,INT_DEFAULT_MAX_CORES,INT_DEFAULT_MIN_CORES,B_BOOKING_ENABLED,B_DISPATCH_ENABLED,B_ACTIVE) values ('00000000-0000-0000-0000-000000000000', 'testing', 200000, 100, true, true, true);
 
-Insert into SHOW_STATS (PK_SHOW,INT_FRAME_INSERT_COUNT,INT_JOB_INSERT_COUNT,INT_FRAME_SUCCESS_COUNT,INT_FRAME_FAIL_COUNT) values ('00000000-0000-0000-0000-000000000000',0,0,0,0)
+Insert into SHOW_STATS (PK_SHOW,INT_FRAME_INSERT_COUNT,INT_JOB_INSERT_COUNT,INT_FRAME_SUCCESS_COUNT,INT_FRAME_FAIL_COUNT) values ('00000000-0000-0000-0000-000000000000',0,0,0,0);
 
 Insert into SHOW_ALIAS (PK_SHOW_ALIAS,PK_SHOW,STR_NAME) values ('00000000-0000-0000-0000-000000000001', '00000000-0000-0000-0000-000000000000', 'test');
 

cuegui/tests/Constants_tests.py

@@ -25,7 +25,7 @@
 
 import mock
 import pyfakefs.fake_filesystem_unittest
-from PySide2 import QtGui
+from qtpy import QtGui
 
 import opencue
 import cuegui.Constants
@@ -40,6 +40,7 @@
 '''
 
 
+# pylint: disable=import-outside-toplevel,redefined-outer-name,reimported
 class ConstantsTests(pyfakefs.fake_filesystem_unittest.TestCase):
     def setUp(self):
         self.setUpPyfakefs()
@@ -53,6 +54,7 @@ def test__should_load_user_config_from_env_var(self):
         self.fs.create_file(config_file_path, contents=CONFIG_YAML)
         os.environ['CUEGUI_CONFIG_FILE'] = config_file_path
 
+        import cuegui.Constants
         result = importlib.reload(cuegui.Constants)
 
         self.assertEqual('98.707.68', result.VERSION)
@@ -65,6 +67,7 @@ def test__should_load_user_config_from_user_profile(self):
         config_file_path = '/home/username/.config/opencue/cuegui.yaml'
         self.fs.create_file(config_file_path, contents=CONFIG_YAML)
 
+        import cuegui.Constants
         result = importlib.reload(cuegui.Constants)
 
         self.assertEqual('98.707.68', result.VERSION)
@@ -73,6 +76,7 @@ def test__should_load_user_config_from_user_profile(self):
 
     @mock.patch('platform.system', new=mock.Mock(return_value='Linux'))
     def test__should_use_default_values(self):
+        import cuegui.Constants
         result = importlib.reload(cuegui.Constants)
 
         self.assertNotEqual('98.707.68', result.VERSION)
@@ -161,6 +165,7 @@ def test__should_use_default_values(self):
 
     @mock.patch('platform.system', new=mock.Mock(return_value='Darwin'))
     def test__should_use_mac_editor(self):
+        import cuegui.Constants
         result = importlib.reload(cuegui.Constants)
 
         self.assertEqual('open -t', result.DEFAULT_EDITOR)

requirements.txt

@@ -12,7 +12,8 @@ packaging==20.9
 pathlib==1.0.1;python_version<"3.4"
 protobuf==3.17.3;python_version<"3.0"
 psutil==5.6.7
-pyfakefs==3.6
+pyfakefs==3.6;python_version<"3.7"
+pyfakefs==5.2.3;python_version>="3.7"
 pylint==2.6.0;python_version>="3.7"
 pynput==1.7.6
 PyYAML==5.1

rqd/rqd/rqcore.py

@@ -1181,8 +1181,10 @@ def pipe_to_file(stdout, stderr, outfile):
     outfile.flush()
     os.fsync(outfile)
 
+    # pylint: disable=import-outside-toplevel
     import select
     import errno
+    # pylint: enable=import-outside-toplevel
 
     fd2file = {}
     fd2output = {}
@@ -1213,8 +1215,7 @@ def print_and_flush_ln(fd, last_timestamp):
         # There are no line breaks
         if len(lines) < 2:
             return curr_line_timestamp
-        else:
-            next_line_timestamp = datetime.datetime.now().strftime("%H:%M:%S")
+        next_line_timestamp = datetime.datetime.now().strftime("%H:%M:%S")
 
         remainder = lines[-1]
         for line in lines[0:-1]:
@@ -1225,8 +1226,7 @@ def print_and_flush_ln(fd, last_timestamp):
 
         if next_line_timestamp is None:
             return curr_line_timestamp
-        else:
-            return next_line_timestamp
+        return next_line_timestamp
 
     def translate_newlines(data):
         data = data.decode("utf-8", "ignore")