OJ-45476: refactor how we get output dir size (#406)

* OJ-45476: refactor how we get output dir size

* OJ-45476: add back in config outdir

* OJ-45476: attempt to fix dependency problems

Author: gavinpitt-jf

.github/workflows/pr-checks.yml

@@ -5,27 +5,26 @@ name: Python application
 
 on:
   push:
-    branches: [ "master" ]
+    branches: ["master"]
   pull_request:
-    branches: [ "master" ]
+    branches: ["master"]
 
 permissions:
   contents: read
 
 jobs:
   build:
-
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v4
-    - name: Set up Python 3.10
-      uses: actions/setup-python@v3
-      with:
-        python-version: "3.10"
-    - name: Install dependencies
-      run: |
-        sudo apt-get install libkrb5-dev && pip install pdm && pdm install --dev
-    - name: Tests (pytest)
-      run: |
-        pdm run python -m pytest
+      - uses: actions/checkout@v4
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v3
+        with:
+          python-version: "3.10"
+      - name: Install dependencies
+        run: |
+          pip install pdm && pdm install --dev
+      - name: Tests (pytest)
+        run: |
+          pdm run python -m pytest

jf_agent/validation.py

@@ -1,15 +1,12 @@
 import logging
 import os
 import shutil
+import subprocess
 import traceback
 from typing import Optional
 
 import psutil
 import requests
-from jf_agent.config_file_reader import get_ingest_config
-from jf_agent.data_manifests.git.generator import get_instance_slug
-from jf_agent.git import GithubGqlClient, get_git_client, get_nested_repos_from_git
-from jf_agent.util import upload_file
 from jf_ingest.validation import (
     GitConnectionHealthCheckResult,
     IngestionHealthCheckResult,
@@ -20,6 +17,10 @@
 )
 
 from jf_agent import write_file
+from jf_agent.config_file_reader import get_ingest_config
+from jf_agent.data_manifests.git.generator import get_instance_slug
+from jf_agent.git import GithubGqlClient, get_git_client, get_nested_repos_from_git
+from jf_agent.util import upload_file
 
 logger = logging.getLogger(__name__)
 
@@ -173,7 +174,12 @@ def validate_memory(config):
             f"  Available memory: {round(psutil.virtual_memory().available / (1024 * 1024), 2)} MB"
         )
 
-        output_dir_size = os.popen(f'du -hs {config.outdir}').readlines()[0].split("\t")[0]
+        proc = subprocess.run(
+            ['du', '-hs', os.path.expanduser(config.outdir)],
+            encoding='utf-8',
+            stdout=subprocess.PIPE,
+        )
+        output_dir_size = proc.stdout.split("\t")[0]
         usage = shutil.disk_usage(config.outdir)
 
         print(

pdm.lock

@@ -20,7 +20,7 @@ dependencies = [
     "python-dotenv>=1.0.0",
     "structlog>=24.4.0",
     "colorama>=0.4.6",
-    "jf-ingest==0.0.214",
+    "jf-ingest==0.0.218",
 ]
 requires-python = "~=3.10.15"
 readme = "README.md"