actions: provision python from conda-forge

.github/workflows/test_fast.yml

@@ -12,6 +12,10 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
+defaults:
+  run:
+    shell: bash -c "exec $CONDA_PREFIX/bin/bash -elo pipefail {0}"
+
 jobs:
   test:
     runs-on: ${{ matrix.os }}
@@ -20,12 +24,10 @@ jobs:
       fail-fast: false # don't stop on first failure
       matrix:
         os: ['ubuntu-latest']
-        python-version: ['3.8', '3.10', '3.11', '3']
+        python-version: ['3.7', '3.8', '3.10', '3.11', '3.12', '3']
         include:
-          - os: 'ubuntu-22.04'
-            python-version: '3.7'
           - os: 'macos-latest'
-            python-version: '3.9'  # oldest supported version
+            python-version: '3.8'  # oldest version with arm64 builds
           # non-utc timezone test
           - os: 'ubuntu-latest'
             python-version: '3.9'  # not the oldest, not the most recent version
@@ -39,24 +41,26 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
 
-      - name: Configure Python
-        uses: actions/setup-python@v5
+      - name: Install System Dependencies
+        uses: mamba-org/setup-micromamba@v2
         with:
-          python-version: ${{ matrix.python-version }}
-
-      - name: Apt-Get Install
-        if: startsWith(matrix.os, 'ubuntu')
+          cache-environment: true
+          post-cleanup: 'all'
+          environment-name: cylc-fast-test
+          create-args: >-
+            python=${{ matrix.python-version }}
+            pip
+            bash
+            subversion
+            sqlite
+
+      - name: Install Python Dependencies
         run: |
-          sudo apt-get update
-          sudo apt-get install -y sqlite3
+          pip install -e ."[all]"
 
       - name: Patch DNS
         uses: cylc/release-actions/patch-dns@v1
 
-      - name: Install
-        run: |
-          pip install -e ."[all]"
-
       - name: Configure git  # Needed by the odd test
         uses: cylc/release-actions/configure-git@v1
 
@@ -99,21 +103,26 @@ jobs:
   lint:
     runs-on: 'ubuntu-latest'
     timeout-minutes: 10
+    strategy:
+      matrix:
+        python-version: ['3']
     steps:
-      - name: Apt-Get Install
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y shellcheck
-
       - name: Checkout
         uses: actions/checkout@v4
 
-      - name: Configure Python
-        uses: actions/setup-python@v5
+      - name: Install System Dependencies
+        uses: mamba-org/setup-micromamba@v2
         with:
-          python-version: 3.x
-
-      - name: Install
+          cache-environment: true
+          post-cleanup: 'all'
+          environment-name: cylc-lint-test
+          create-args: >-
+            python=${{ matrix.python-version }}
+            pip
+            bash
+            shellcheck
+
+      - name: Install Python Dependencies
         run: |
           pip install -e ."[tests]"
 

.github/workflows/test_functional.yml

@@ -32,6 +32,10 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
+defaults:
+  run:
+    shell: bash -c "exec $CONDA_PREFIX/bin/bash -elo pipefail {0}"
+
 jobs:
   test:
     runs-on: ${{ matrix.os }}
@@ -56,34 +60,34 @@ jobs:
             platform: '_local_background*'
           # tests/k
           - name: 'flaky'
-            os: 'ubuntu-22.04'
+            os: 'ubuntu-latest'
             python-version: '3.7'
             test-base: 'tests/k'
             chunk: '1/1'
             platform: '_local_background* _local_at*'
           # remote platforms
           - name: '_remote_background_indep_poll'
-            os: 'ubuntu-22.04'
+            os: 'ubuntu-latest'
             python-version: '3.7'
             test-base: 'tests/f tests/k'
             chunk: '1/1'
             platform: '_remote_background_indep_poll _remote_at_indep_poll'
           - name: '_remote_background_indep_tcp'
-            os: 'ubuntu-22.04'
+            os: 'ubuntu-latest'
             test-base: 'tests/f tests/k'
             python-version: '3.7'
             chunk: '1/1'
             platform: '_remote_background_indep_tcp _remote_at_indep_tcp'
           # macos
           - name: 'macos 1/5'
             os: 'macos-latest'
-            python-version: '3.9'
+            python-version: '3.8'  # oldest available
             test-base: 'tests/f'
             chunk: '1/5'
             platform: '_local_background*'
           - name: 'macos 2/5'
             os: 'macos-latest'
-            python-version: '3.9'
+            python-version: '3'  # newest available
             test-base: 'tests/f'
             chunk: '2/5'
             platform: '_local_background*'
@@ -101,9 +105,21 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Configure Python
-        uses: actions/setup-python@v5
+        uses: mamba-org/setup-micromamba@v2
         with:
-          python-version: ${{ matrix.python-version }}
+          cache-environment: true
+          post-cleanup: 'all'
+          environment-name: cylc-functional-test
+          create-args: >-
+            python=${{ matrix.python-version }}
+            pip
+            bash
+            coreutils
+            grep
+            sed
+            sqlite
+            subversion
+            tree
 
       - name: Create global config
         run: |
@@ -128,36 +144,12 @@ jobs:
           __HERE__
           cat "$GLOBAL_CFG_PATH"
 
-      - name: Brew Install
-        if: startsWith(matrix.os, 'macos')
-        run: |
-          # install system deps
-          brew update
-          brew install bash coreutils gnu-sed grep
-
-          # add GNU coreutils and sed to the user PATH
-          # (see instructions in brew install output)
-          echo "$(brew --prefix)/opt/coreutils/libexec/gnubin" >> "${GITHUB_PATH}"
-          echo "$(brew --prefix)/opt/grep/libexec/gnubin" >> "${GITHUB_PATH}"
-          echo "$(brew --prefix)/opt/gnu-sed/libexec/gnubin" >> "${GITHUB_PATH}"
-
-          # add coreutils to the bashrc too (for jobs)
-          cat >> "${HOME}/.bashrc" <<__HERE__
-          PATH="$(brew --prefix)/opt/coreutils/libexec/gnubin:$PATH"
-          PATH="$(brew --prefix)/opt/grep/libexec/gnubin:$PATH"
-          PATH="$(brew --prefix)/opt/gnu-sed/libexec/gnubin:$PATH"
-          export PATH
-          __HERE__
-
-      - name: Apt-Get Install
-        if: startsWith(matrix.os, 'ubuntu')
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y sqlite3 tree at
-
       - name: Add .github/bin/ to PATH
         # Sets up mocked mail command & any other custom executables
-        run: echo "${{ github.workspace }}/.github/bin" >> $GITHUB_PATH
+        # run: echo "${{ github.workspace }}/.github/bin" >> $GITHUB_PATH
+        run: |
+          echo "export PATH=\"${{ github.workspace }}/.github/bin:$PATH\"" >> ~/.bash_profile
+          cat ~/.bash_profile
 
       - name: Install
         run: |
@@ -167,6 +159,8 @@ jobs:
       - name: Configure Atrun
         if: contains(matrix.platform, '_local_at')
         run: |
+          sudo apt-get update
+          sudo apt-get install -y at
           cat >> "$GLOBAL_CFG_PATH" << __HERE__
             [platforms]
                 [[_local_at_indep_tcp]]

tests/flakyfunctional/database/00-simple.t

@@ -38,8 +38,8 @@ cmp_ok "${NAME}" "${SORTED_ORIG}"
 
 NAME='select-workflow-params.out'
 sqlite3 "${DB_FILE}" \
-    'SELECT key, value FROM workflow_params
-    WHERE key != "uuid_str" AND key != "cycle_point_tz" ORDER BY key' \
+    "SELECT key, value FROM workflow_params
+    WHERE key != 'uuid_str' AND key != 'cycle_point_tz' ORDER BY key" \
     >"${NAME}"
 cmp_ok "${NAME}" << __EOF__
 UTC_mode|0

tests/flakyfunctional/hold-release/15-hold-after.t

@@ -27,7 +27,7 @@ run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}"
 workflow_run_ok "${TEST_NAME_BASE}-1" \
     cylc play --reference-test --debug --no-detach "${WORKFLOW_NAME}"
 sqlite3 "${WORKFLOW_RUN_DIR}/log/db" \
-    'SELECT cycle, name, status FROM task_pool WHERE cycle=="20140102T0000Z" ORDER BY name' \
+    "SELECT cycle, name, status FROM task_pool WHERE cycle=='20140102T0000Z' ORDER BY name" \
     >'taskpool.out'
 cmp_ok 'taskpool.out' <<'__OUT__'
 20140102T0000Z|foo|waiting

tests/flakyfunctional/job-submission/19-chatty.t

@@ -70,7 +70,7 @@ done
 # Task pool in database contains the correct states
 TEST_NAME="${TEST_NAME_BASE}-db-task-pool"
 DB_FILE="${WORKFLOW_RUN_DIR}/log/db"
-QUERY='SELECT cycle, name, status FROM task_states WHERE name LIKE "nh%"'
+QUERY="SELECT cycle, name, status FROM task_states WHERE name LIKE 'nh%'"
 run_ok "$TEST_NAME" sqlite3 "$DB_FILE" "$QUERY"
 sort "${TEST_NAME}.stdout" > "${TEST_NAME}.stdout.sorted"
 cmp_ok "${TEST_NAME}.stdout.sorted" << '__OUT__'

tests/flakyfunctional/restart/46-stop-clock-time.t

@@ -21,7 +21,7 @@
 
 dumpdbtables() {
     sqlite3 "${WORKFLOW_RUN_DIR}/log/db" \
-        'SELECT * FROM workflow_params WHERE key=="stop_clock_time";' \
+        "SELECT * FROM workflow_params WHERE key=='stop_clock_time';" \
         >'stopclocktime.out'
 }
 

tests/functional/cylc-cat-log/05-remote-tail.t

@@ -35,8 +35,7 @@ TEST_NAME="${TEST_NAME_BASE}-validate"
 run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}"
 #-------------------------------------------------------------------------------
 $SCP "${PWD}/bin/my-tailer.sh" \
-    "${CYLC_TEST_HOST}:cylc-run/.bin/my-tailer.sh
-"
+    "${CYLC_TEST_HOST}:cylc-run/.bin/my-tailer.sh"
 #-------------------------------------------------------------------------------
 # Run detached.
 workflow_run_ok "${TEST_NAME_BASE}-run" cylc play "${WORKFLOW_NAME}"

tests/functional/cylc-poll/11-event-time.t

@@ -27,9 +27,9 @@ workflow_run_ok "${TEST_NAME_BASE}" \
 
 RUND="$RUN_DIR/${WORKFLOW_NAME}"
 sed -n 's/CYLC_JOB_EXIT_TIME=//p' "${RUND}/log/job/1/w1/NN/job.status" >'st-time.txt'
-sqlite3 "${RUND}/log/db" '
+sqlite3 "${RUND}/log/db" "
     SELECT time_run_exit FROM task_jobs
-    WHERE cycle=="1" AND name=="w1" AND submit_num=="1"' >'db-time.txt'
+    WHERE cycle=='1' AND name=='w1' AND submit_num=='1'" >'db-time.txt'
 run_ok "${TEST_NAME_BASE}-time-run-exit" diff -u 'st-time.txt' 'db-time.txt'
 
 purge

tests/functional/cylc-set/00-set-succeeded.t

@@ -28,12 +28,12 @@ reftest_run
 for TASK in foo bar
 do
     sqlite3 ~/cylc-run/"${WORKFLOW_NAME}"/log/db \
-       "SELECT status FROM task_states WHERE name is \"$TASK\"" > "${TASK}.1"
+       "SELECT status FROM task_states WHERE name is '$TASK'" > "${TASK}.1"
 
     cmp_ok ${TASK}.1 - <<<succeeded
 
     sqlite3 ~/cylc-run/"${WORKFLOW_NAME}"/log/db \
-       "SELECT outputs FROM task_outputs WHERE name is \"$TASK\"" > "${TASK}.2"
+       "SELECT outputs FROM task_outputs WHERE name is '$TASK'" > "${TASK}.2"
 
     cmp_json \
         "check-${TASK}-outputs" \

tests/functional/cylc-set/03-set-failed.t

@@ -42,7 +42,7 @@ cylc stop --now --now --interval=2 --max-polls=5 "${WORKFLOW_NAME}"
 
 # Check the DB records all the outputs.
 sqlite3 ~/cylc-run/"${WORKFLOW_NAME}"/log/db \
-   "SELECT outputs FROM task_outputs WHERE name is \"foo\"" > db-foo.1
+   "SELECT outputs FROM task_outputs WHERE name is 'foo'" > db-foo.1
 
 # Json string list of outputs from the db may not be ordered correctly.
 python3 - << __END__ > db-foo.2

tests/functional/cylc-set/04-switch.t

@@ -28,7 +28,7 @@ reftest_run
 # The branch-point task foo should be recorded as succeeded.
 
 sqlite3 ~/cylc-run/"${WORKFLOW_NAME}"/log/db \
-   "SELECT status FROM task_states WHERE name is \"foo\"" > db-foo.2
+   "SELECT status FROM task_states WHERE name is 'foo'" > db-foo.2
 
 cmp_ok "db-foo.2" - << __OUT__
 succeeded
@@ -40,7 +40,7 @@ __OUT__
 #   submitted, started
 
 sqlite3 ~/cylc-run/"${WORKFLOW_NAME}"/log/db \
-   "SELECT outputs FROM task_outputs WHERE name is \"foo\"" > db-foo.1
+   "SELECT outputs FROM task_outputs WHERE name is 'foo'" > db-foo.1
 
 # Json string list of outputs from the db may not be ordered correctly.
 python3 - << __END__ > db-foo.2

tests/functional/cylc-set/05-expire.t

@@ -26,14 +26,14 @@ install_and_validate
 reftest_run
 
 sqlite3 ~/cylc-run/"${WORKFLOW_NAME}"/log/db \
-   "SELECT status FROM task_states WHERE name is \"bar\"" > db-bar.1
+   "SELECT status FROM task_states WHERE name is 'bar'" > db-bar.1
 
 cmp_ok "db-bar.1" - << __OUT__
 expired
 __OUT__
 
 sqlite3 ~/cylc-run/"${WORKFLOW_NAME}"/log/db \
-   "SELECT outputs FROM task_outputs WHERE name is \"bar\"" > db-bar.2
+   "SELECT outputs FROM task_outputs WHERE name is 'bar'" > db-bar.2
 
 cmp_ok "db-bar.2" - << __OUT__
 {"expired": "(manually completed)"}

tests/functional/database/06-task-message.t

@@ -27,16 +27,16 @@ workflow_run_ok "${TEST_NAME_BASE}-run" \
 DB_FILE="$RUN_DIR/${WORKFLOW_NAME}/log/db"
 
 NAME='select-task-events.out'
-sqlite3 "${DB_FILE}" '
+sqlite3 "${DB_FILE}" "
     SELECT
         cycle, name, event, message
     FROM
         task_events
     WHERE
-        event GLOB "message *"
+        event GLOB 'message *'
     ORDER BY
         event
-' >"${NAME}"
+" >"${NAME}"
 cmp_ok "${NAME}" <<'__SELECT__'
 1|t1|message critical|You are being critical
 1|t1|message info|You are normal

tests/functional/events/20-workflow-event-handlers.t

@@ -37,7 +37,7 @@ workflow_run_ok "${TEST_NAME_BASE}-run" \
     cylc play --reference-test --debug --no-detach ${OPT_SET} "${WORKFLOW_NAME}"
 
 LOGD="$RUN_DIR/${WORKFLOW_NAME}/log"
-WORKFLOW_UUID="$(sqlite3 "${LOGD}/db" 'SELECT value FROM workflow_params WHERE key=="uuid_str"')"
+WORKFLOW_UUID="$(sqlite3 "${LOGD}/db" "SELECT value FROM workflow_params WHERE key=='uuid_str'")"
 LOG_FILE="${LOGD}/scheduler/log"
 grep_ok "\\[('workflow-event-handler-00', 'startup') ret_code\\] 0" "${LOG_FILE}"
 grep_ok "\\[('workflow-event-handler-00', 'startup') out\\] Your ${WORKFLOW_NAME} workflow has a startup event and URL http://myworkflows.com/${WORKFLOW_NAME}.html and workflow-priority as HIGH and workflow-UUID as ${WORKFLOW_UUID}." "${LOG_FILE}"

tests/functional/job-file-trap/00-sigusr1.t

@@ -44,7 +44,7 @@ run_tests() {
     poll_grep_workflow_log 'vacated/USR1'
     sleep 1  # a bit of extra time for workflow db update to complete
     sqlite3 "${WORKFLOW_RUN_DIR}/log/db" \
-        'SELECT status FROM task_states WHERE name=="t1";' \
+        "SELECT status FROM task_states WHERE name=='t1';" \
         >"${TEST_NAME}-db-t1" 2>'/dev/null'
     grep_ok "^\(submitted\|running\)$" "${TEST_NAME}-db-t1"
     # Start the job again and see what happens
@@ -55,7 +55,7 @@ run_tests() {
     poll_workflow_stopped
     # Test t1 status in DB
     sqlite3 "${WORKFLOW_RUN_DIR}/log/db" \
-        'SELECT status FROM task_states WHERE name=="t1";' >"${TEST_NAME}-db-t1"
+        "SELECT status FROM task_states WHERE name=='t1';" >"${TEST_NAME}-db-t1"
     cmp_ok "${TEST_NAME}-db-t1" - <<<'succeeded'
     # Test reference
     grep_ok 'WORKFLOW REFERENCE TEST PASSED' "${WORKFLOW_RUN_DIR}/log/scheduler/log"