Merge pull request #21 from UCL-RITS/enforce_style

Adds style guide enforcement in CI builds

.github/workflows/main.yml

@@ -34,5 +34,8 @@ jobs:
     - name: Test with pytest
       run: |
         pytest data_preprocessing/tests/
- 
+    - name: Style check
+      run: black --check data_preprocessing
+
+
 

data_preprocessing/scripts/convert_xml_to_yolotxt.py

@@ -58,15 +58,20 @@ def convert_xml_annotation_to_yolo(label_path, output_path, is_masati, classes):
 
     in_file.close()
     out_file.close()
+
+
 def main():
     from read_params import read_parameter_file
+
     params = read_parameter_file("params.yaml")
-    labels_dir = os.path.join(os.getcwd(), str(params["get_yolo_labels"]["annotations_dir"]))
+    labels_dir = os.path.join(
+        os.getcwd(), str(params["get_yolo_labels"]["annotations_dir"])
+    )
     classes = ["boat"]
 
     output_path = os.path.join(os.getcwd(), "yolo_annotations")
     print("Will output to: {}".format(output_path))
-    
+
     is_masati = params["get_yolo_labels"]["is_masati"]
     if not os.path.exists(output_path):
         os.makedirs(output_path)

data_preprocessing/scripts/prepare_yolov5.py

@@ -8,17 +8,13 @@
 """
 
 
-def create_yolo_dirs(train_label_dir,
-                     val_label_dir,
-                     train_img_dir,
-                     val_img_dir,
-                     target_directory
-                     ):
+def create_yolo_dirs(
+    train_label_dir, val_label_dir, train_img_dir, val_img_dir, target_directory
+):
 
     output_images_dir = os.path.join(target_directory, "images")
     output_labels_dir = output_images_dir.replace("images", "labels")
 
-
     # Make these directories if they don't exist
     if not os.path.exists(target_directory):
         os.mkdir(target_directory)
@@ -31,8 +27,8 @@ def create_yolo_dirs(train_label_dir,
 
     output_train_label_dir = os.path.join(output_labels_dir, "train")
     output_validation_label_dir = os.path.join(output_labels_dir, "val")
-    output_train_img_dir = os.path.join(output_images_dir,"train")
-    output_validation_image_dir = os.path.join(output_images_dir,"val")
+    output_train_img_dir = os.path.join(output_images_dir, "train")
+    output_validation_image_dir = os.path.join(output_images_dir, "val")
     os.rename(train_label_dir, output_train_label_dir)
     os.rename(val_label_dir, output_validation_label_dir)
     os.rename(train_img_dir, output_train_img_dir)
@@ -42,28 +38,26 @@ def create_yolo_dirs(train_label_dir,
         output_train_label_dir,
         output_validation_label_dir,
         output_train_img_dir,
-        output_validation_image_dir
+        output_validation_image_dir,
     ]
 
+
 def main():
     from read_params import read_parameter_file
+
     params = read_parameter_file("params.yaml")
     workdir = os.getcwd()
     train_label_dir = os.path.join(
-        workdir,
-        params["create_yolov5_filestructure"]["train_labels"]
+        workdir, params["create_yolov5_filestructure"]["train_labels"]
     )
     val_label_dir = os.path.join(
-        workdir,
-        params["create_yolov5_filestructure"]["val_labels"]
+        workdir, params["create_yolov5_filestructure"]["val_labels"]
     )
     train_img_dir = os.path.join(
-        workdir,
-        params["create_yolov5_filestructure"]["train_imgs"]
+        workdir, params["create_yolov5_filestructure"]["train_imgs"]
     )
     val_img_dir = os.path.join(
-        workdir,
-        params["create_yolov5_filestructure"]["val_imgs"]
+        workdir, params["create_yolov5_filestructure"]["val_imgs"]
     )
     dataset = params["create_yolov5_filestructure"]["dataset"]
 
@@ -75,22 +69,23 @@ def main():
         assert os.path.isdir(val_img_dir)
     except AssertionError:
         "Check that the yolo train/val label and image directories exist in this directory!"
-
-    assert os.path.isdir("../../data_preprocessing"), "Run this script from a subdirectory of data_preprocessing"
-
+
+    assert os.path.isdir(
+        "../../data_preprocessing"
+    ), "Run this script from a subdirectory of data_preprocessing"
+
     vetii_directory = str(Path(workdir).parent.parent)
-    data_directory = os.path.join(vetii_directory,"data")
+    data_directory = os.path.join(vetii_directory, "data")
+
+    assert os.path.isdir(
+        data_directory
+    ), "data/ directory does not exist at the top level of this project. Create this directory before running!"
 
-    assert os.path.isdir(data_directory), "data/ directory does not exist at the top level of this project. Create this directory before running!"
-
     target_directory = os.path.join(data_directory, dataset)
     print(target_directory)
-    output_dirs = create_yolo_dirs(train_label_dir,
-                     val_label_dir,
-                     train_img_dir,
-                     val_img_dir,
-                     target_directory
-                     )
+    output_dirs = create_yolo_dirs(
+        train_label_dir, val_label_dir, train_img_dir, val_img_dir, target_directory
+    )
 
     for outdir in output_dirs:
         assert os.path.isdir(outdir), "{} was not found".format(outdir)

data_preprocessing/scripts/sort_yolo_images.py

@@ -22,7 +22,7 @@ def create_yolo_dirs(workdir, imgdir, labeldir):
     if not os.path.exists(output_images_dir):
         os.mkdir(output_images_dir)
 
-    label_list = os.listdir(os.path.join(workdir,labeldir))
+    label_list = os.listdir(os.path.join(workdir, labeldir))
     for imgfile in progress_bar(os.listdir(os.path.join(workdir, imgdir))):
         basefile = imgfile.split(".", 1)[0]
         if basefile + ".txt" in label_list:
@@ -44,7 +44,7 @@ def rename_yolo_files(workdir, label_dir, imgformat):
 
         old_label_path = os.path.join(labels_path, thisfile)
         old_image_path = old_label_path.replace("labels", "images").replace(
-            ".txt", "."+imgformat
+            ".txt", "." + imgformat
         )
         new_label_name = str(counter).zfill(5)
         new_label_path = os.path.join(labels_path, new_label_name + ".txt")
@@ -57,6 +57,7 @@ def rename_yolo_files(workdir, label_dir, imgformat):
 
 def main():
     from read_params import read_parameter_file
+
     yolo_annotations_dir = "yolo_annotations"
     workdir = os.getcwd()
     params = read_parameter_file("params.yaml")

data_preprocessing/scripts/train_test_split.py

@@ -86,6 +86,7 @@ def move_image_files(workdir, train, test, annotations_dir_name, params):
 
 def main():
     from read_params import read_parameter_file
+
     params = read_parameter_file("params.yaml")
     float_fraction(params["split_dataset"]["trainpct"])
     xml_annotations_dir = params["split_dataset"]["annotations_dir"]

data_preprocessing/tests/fixtures/setup_convert_xml_yolo.py

@@ -38,4 +38,4 @@ def create_temp_file_io(tmpdir_factory):
         r" </annotation>"
     )
     test_xml.write(test_xml_string)
-    return test_xml, test_outdir
+    return test_xml, test_outdir

data_preprocessing/tests/fixtures/setup_prepare_yolov5.py

@@ -1,6 +1,7 @@
 import pytest
 from pathlib import Path
 
+
 @pytest.fixture(scope="session")
 def setup_target_dir(tmpdir_factory):
     test_target_dir = tmpdir_factory.mktemp("data_testset")
@@ -15,6 +16,6 @@ def setup_target_dir(tmpdir_factory):
         "val_label_dir": str(val_label_dir),
         "train_img_dir": str(train_img_dir),
         "val_img_dir": str(val_img_dir),
-        "target_dir": str(test_target_dir)
+        "target_dir": str(test_target_dir),
     }
-    return setup_dict
+    return setup_dict

data_preprocessing/tests/fixtures/setup_sort_yolo_images.py

@@ -1,6 +1,7 @@
 import pytest
 from pathlib import Path
 
+
 @pytest.fixture(scope="session")
 def setup_yolo_dirs(tmpdir_factory):
 
@@ -10,7 +11,6 @@ def setup_yolo_dirs(tmpdir_factory):
     test_label_two = test_label_dir.join("test_2.txt")
     test_label_two.write("test_label_2")
 
-
     test_img_dir = tmpdir_factory.mktemp("train")
     test_img_one = test_img_dir.join("test_1.png")
     test_img_one.write("test_img_1")
@@ -22,9 +22,9 @@ def setup_yolo_dirs(tmpdir_factory):
     assert test_img_one.read() == "test_img_1"
     assert test_img_two.read() == "test_img_2"
 
-
     return test_label_dir, test_img_dir
 
+
 @pytest.fixture(scope="session")
 def setup_yolo_rename_dirs(tmpdir_factory):
 
@@ -40,4 +40,3 @@ def setup_yolo_rename_dirs(tmpdir_factory):
     assert test_img_one.read() == "test_img_1"
 
     return test_old_label_dir, test_old_img_dir
-

data_preprocessing/tests/fixtures/setup_train_test_split.py

@@ -16,21 +16,23 @@ def setup_file_list(tmpdir_factory):
         str(test_xml_two),
         str(test_xml_three),
         str(test_xml_four),
-        ]
+    ]
     return test_file_list
 
+
 @pytest.fixture(scope="session")
 def setup_out_dir(tmpdir_factory):
     temp_dir = tmpdir_factory.mktemp("test_outdir")
     return temp_dir
 
+
 @pytest.fixture(scope="session")
 def setup_move_img_files(tmpdir_factory):
 
     test_labels_dir = tmpdir_factory.mktemp("label")
     test_labels_dir.join("test_1.xml")
     test_labels_dir.join("test_2.xml")
-    
+
     test_train_dir = tmpdir_factory.mktemp("train")
     test_valid_dir = tmpdir_factory.mktemp("valid")
 
@@ -42,30 +44,24 @@ def setup_move_img_files(tmpdir_factory):
     assert test_img_one.read() == "test1"
     assert test_img_two.read() == "test2"
 
-    test_val_dict = {
-            "Filename": [str(test_img_dir)+"/test_1.png"]
-        }
-
-    test_train_dict = {
-            "Filename": [str(test_img_dir)+"/test_2.png"]
-        }
+    test_val_dict = {"Filename": [str(test_img_dir) + "/test_1.png"]}
+
+    test_train_dict = {"Filename": [str(test_img_dir) + "/test_2.png"]}
     test_train_set = pd.DataFrame.from_dict(test_train_dict)
     test_val_set = pd.DataFrame.from_dict(test_val_dict)
 
-
     params = {
         "split_dataset": {
             "train_dir": str(test_train_dir),
             "valid_dir": str(test_valid_dir),
             "img_dir": str(test_img_dir),
-            "file_type": "png"
+            "file_type": "png",
         }
     }
     setup_dict = {
         "train": test_train_set,
         "test": test_val_set,
         "labels": str(test_labels_dir),
-        "params": params
+        "params": params,
     }
     return setup_dict
-

data_preprocessing/tests/test_convert_xml_to_yolotxt.py

@@ -13,20 +13,20 @@ def test_yolo_conversion():
 
     boundaries = (469, 472, 28, 37)
 
-
     width = 512.0
     height = 512.0
     bounding_box = funcs.yolo_conversion((width, height), boundaries)
     expected_bounding_box = [0.9189453125, 0.0634765625, 0.005859375, 0.017578125]
     assert list(bounding_box) == expected_bounding_box
 
+
 @pytest.mark.usefixtures("create_temp_file_io")
 def test_convert_xml_annotation_to_yolo(create_temp_file_io):
     test_xml, test_outdir = create_temp_file_io
     print(test_xml, test_outdir)
-    funcs.convert_xml_annotation_to_yolo(test_xml, test_outdir, True, ['boat'])
+    funcs.convert_xml_annotation_to_yolo(test_xml, test_outdir, True, ["boat"])
     assert len(os.listdir(test_outdir)) == 1
     assert os.listdir(test_outdir)[0] == "test.txt"
-    output_file = os.path.join(test_outdir,"test.txt")
+    output_file = os.path.join(test_outdir, "test.txt")
     output_text = Path(output_file).read_text().replace("\n", "")
-    assert output_text == "0 0.9189453125 0.0634765625 0.005859375 0.017578125"
+    assert output_text == "0 0.9189453125 0.0634765625 0.005859375 0.017578125"

data_preprocessing/tests/test_prepare_yolov5.py

@@ -1,10 +1,10 @@
-
 import pytest
 import os
 from pathlib import Path
 from ..scripts import prepare_yolov5 as funcs
 from .fixtures.setup_prepare_yolov5 import setup_target_dir
 
+
 @pytest.mark.usefixtures("setup_target_dir")
 def test_create_yolo_dirs(setup_target_dir):
     setup_dict = setup_target_dir
@@ -13,11 +13,9 @@ def test_create_yolo_dirs(setup_target_dir):
         setup_dict["val_label_dir"],
         setup_dict["train_img_dir"],
         setup_dict["val_img_dir"],
-        setup_dict["target_dir"]
+        setup_dict["target_dir"],
     )
     assert len(output_dirs) == 4
 
 
-#@pytest.mark.usefixtures("setup_yolo_rename_dirs")
-
-
+# @pytest.mark.usefixtures("setup_yolo_rename_dirs")

data_preprocessing/tests/test_sort_yolo_images.py

@@ -1,21 +1,21 @@
-
 import pytest
 import os
 from pathlib import Path
 from ..scripts import sort_yolo_images as funcs
 from .fixtures.setup_sort_yolo_images import setup_yolo_dirs, setup_yolo_rename_dirs
 
+
 @pytest.mark.usefixtures("setup_yolo_dirs")
 def test_create_yolo_dirs(setup_yolo_dirs):
     test_label_dir, _ = setup_yolo_dirs
     workdir = str(Path(test_label_dir).parent)
     assert str(test_label_dir) == os.path.join(workdir, test_label_dir)
-    assert len(os.listdir(os.path.join(workdir,"label_sortyolo0"))) == 2
-    assert len(os.listdir(os.path.join(workdir,"train0"))) == 2
+    assert len(os.listdir(os.path.join(workdir, "label_sortyolo0"))) == 2
+    assert len(os.listdir(os.path.join(workdir, "train0"))) == 2
 
     out_img_dir, out_label_dir = funcs.create_yolo_dirs(
-        workdir, 
-        "train0", 
+        workdir,
+        "train0",
         "label_sortyolo0",
     )
     print(out_img_dir, out_label_dir)
@@ -24,6 +24,7 @@ def test_create_yolo_dirs(setup_yolo_dirs):
     assert len(os.listdir(out_img_dir)) == 2
     assert len(os.listdir(out_label_dir)) == 2
 
+
 @pytest.mark.usefixtures("setup_yolo_rename_dirs")
 def test_rename_yolo_files(setup_yolo_rename_dirs):
     test_label_dir, test_img_dir = setup_yolo_rename_dirs
@@ -32,7 +33,6 @@ def test_rename_yolo_files(setup_yolo_rename_dirs):
     assert os.listdir(test_label_dir)[0] == "test_1.txt"
     assert os.listdir(test_img_dir)[0] == "test_1.jpg"
     funcs.rename_yolo_files(str(workdir), test_label_dir, "jpg")
-    
+
     assert os.listdir(test_label_dir)[0] == "00001.txt"
     assert os.listdir(test_img_dir)[0] == "00001.jpg"
-