From d23115023170ec7574dd76421ed3fc5ed89150dd Mon Sep 17 00:00:00 2001 From: Harry Moss Date: Thu, 17 Dec 2020 17:57:47 +0000 Subject: [PATCH 1/2] :lipstick: Style changes * Changes to scripts/ and tests/ after black check --- .../scripts/convert_xml_to_yolotxt.py | 9 ++- data_preprocessing/scripts/prepare_yolov5.py | 55 +++++++++---------- .../scripts/sort_yolo_images.py | 5 +- .../scripts/train_test_split.py | 1 + .../tests/fixtures/setup_convert_xml_yolo.py | 2 +- .../tests/fixtures/setup_prepare_yolov5.py | 5 +- .../tests/fixtures/setup_sort_yolo_images.py | 5 +- .../tests/fixtures/setup_train_test_split.py | 22 +++----- .../tests/test_convert_xml_to_yolotxt.py | 8 +-- .../tests/test_prepare_yolov5.py | 8 +-- .../tests/test_sort_yolo_images.py | 14 ++--- .../tests/test_train_test_split.py | 26 ++++++--- 12 files changed, 84 insertions(+), 76 deletions(-) diff --git a/data_preprocessing/scripts/convert_xml_to_yolotxt.py b/data_preprocessing/scripts/convert_xml_to_yolotxt.py index 6253469..11dc67c 100644 --- a/data_preprocessing/scripts/convert_xml_to_yolotxt.py +++ b/data_preprocessing/scripts/convert_xml_to_yolotxt.py @@ -58,15 +58,20 @@ def convert_xml_annotation_to_yolo(label_path, output_path, is_masati, classes): in_file.close() out_file.close() + + def main(): from read_params import read_parameter_file + params = read_parameter_file("params.yaml") - labels_dir = os.path.join(os.getcwd(), str(params["get_yolo_labels"]["annotations_dir"])) + labels_dir = os.path.join( + os.getcwd(), str(params["get_yolo_labels"]["annotations_dir"]) + ) classes = ["boat"] output_path = os.path.join(os.getcwd(), "yolo_annotations") print("Will output to: {}".format(output_path)) - + is_masati = params["get_yolo_labels"]["is_masati"] if not os.path.exists(output_path): os.makedirs(output_path) diff --git a/data_preprocessing/scripts/prepare_yolov5.py b/data_preprocessing/scripts/prepare_yolov5.py index 9592f4b..02aaca2 100644 --- a/data_preprocessing/scripts/prepare_yolov5.py +++ b/data_preprocessing/scripts/prepare_yolov5.py @@ -8,17 +8,13 @@ """ -def create_yolo_dirs(train_label_dir, - val_label_dir, - train_img_dir, - val_img_dir, - target_directory - ): +def create_yolo_dirs( + train_label_dir, val_label_dir, train_img_dir, val_img_dir, target_directory +): output_images_dir = os.path.join(target_directory, "images") output_labels_dir = output_images_dir.replace("images", "labels") - # Make these directories if they don't exist if not os.path.exists(target_directory): os.mkdir(target_directory) @@ -31,8 +27,8 @@ def create_yolo_dirs(train_label_dir, output_train_label_dir = os.path.join(output_labels_dir, "train") output_validation_label_dir = os.path.join(output_labels_dir, "val") - output_train_img_dir = os.path.join(output_images_dir,"train") - output_validation_image_dir = os.path.join(output_images_dir,"val") + output_train_img_dir = os.path.join(output_images_dir, "train") + output_validation_image_dir = os.path.join(output_images_dir, "val") os.rename(train_label_dir, output_train_label_dir) os.rename(val_label_dir, output_validation_label_dir) os.rename(train_img_dir, output_train_img_dir) @@ -42,28 +38,26 @@ def create_yolo_dirs(train_label_dir, output_train_label_dir, output_validation_label_dir, output_train_img_dir, - output_validation_image_dir + output_validation_image_dir, ] + def main(): from read_params import read_parameter_file + params = read_parameter_file("params.yaml") workdir = os.getcwd() train_label_dir = os.path.join( - workdir, - params["create_yolov5_filestructure"]["train_labels"] + workdir, params["create_yolov5_filestructure"]["train_labels"] ) val_label_dir = os.path.join( - workdir, - params["create_yolov5_filestructure"]["val_labels"] + workdir, params["create_yolov5_filestructure"]["val_labels"] ) train_img_dir = os.path.join( - workdir, - params["create_yolov5_filestructure"]["train_imgs"] + workdir, params["create_yolov5_filestructure"]["train_imgs"] ) val_img_dir = os.path.join( - workdir, - params["create_yolov5_filestructure"]["val_imgs"] + workdir, params["create_yolov5_filestructure"]["val_imgs"] ) dataset = params["create_yolov5_filestructure"]["dataset"] @@ -75,22 +69,23 @@ def main(): assert os.path.isdir(val_img_dir) except AssertionError: "Check that the yolo train/val label and image directories exist in this directory!" - - assert os.path.isdir("../../data_preprocessing"), "Run this script from a subdirectory of data_preprocessing" - + + assert os.path.isdir( + "../../data_preprocessing" + ), "Run this script from a subdirectory of data_preprocessing" + vetii_directory = str(Path(workdir).parent.parent) - data_directory = os.path.join(vetii_directory,"data") + data_directory = os.path.join(vetii_directory, "data") + + assert os.path.isdir( + data_directory + ), "data/ directory does not exist at the top level of this project. Create this directory before running!" - assert os.path.isdir(data_directory), "data/ directory does not exist at the top level of this project. Create this directory before running!" - target_directory = os.path.join(data_directory, dataset) print(target_directory) - output_dirs = create_yolo_dirs(train_label_dir, - val_label_dir, - train_img_dir, - val_img_dir, - target_directory - ) + output_dirs = create_yolo_dirs( + train_label_dir, val_label_dir, train_img_dir, val_img_dir, target_directory + ) for outdir in output_dirs: assert os.path.isdir(outdir), "{} was not found".format(outdir) diff --git a/data_preprocessing/scripts/sort_yolo_images.py b/data_preprocessing/scripts/sort_yolo_images.py index a273132..414f074 100644 --- a/data_preprocessing/scripts/sort_yolo_images.py +++ b/data_preprocessing/scripts/sort_yolo_images.py @@ -22,7 +22,7 @@ def create_yolo_dirs(workdir, imgdir, labeldir): if not os.path.exists(output_images_dir): os.mkdir(output_images_dir) - label_list = os.listdir(os.path.join(workdir,labeldir)) + label_list = os.listdir(os.path.join(workdir, labeldir)) for imgfile in progress_bar(os.listdir(os.path.join(workdir, imgdir))): basefile = imgfile.split(".", 1)[0] if basefile + ".txt" in label_list: @@ -44,7 +44,7 @@ def rename_yolo_files(workdir, label_dir, imgformat): old_label_path = os.path.join(labels_path, thisfile) old_image_path = old_label_path.replace("labels", "images").replace( - ".txt", "."+imgformat + ".txt", "." + imgformat ) new_label_name = str(counter).zfill(5) new_label_path = os.path.join(labels_path, new_label_name + ".txt") @@ -57,6 +57,7 @@ def rename_yolo_files(workdir, label_dir, imgformat): def main(): from read_params import read_parameter_file + yolo_annotations_dir = "yolo_annotations" workdir = os.getcwd() params = read_parameter_file("params.yaml") diff --git a/data_preprocessing/scripts/train_test_split.py b/data_preprocessing/scripts/train_test_split.py index ec080fb..c6a4c80 100644 --- a/data_preprocessing/scripts/train_test_split.py +++ b/data_preprocessing/scripts/train_test_split.py @@ -86,6 +86,7 @@ def move_image_files(workdir, train, test, annotations_dir_name, params): def main(): from read_params import read_parameter_file + params = read_parameter_file("params.yaml") float_fraction(params["split_dataset"]["trainpct"]) xml_annotations_dir = params["split_dataset"]["annotations_dir"] diff --git a/data_preprocessing/tests/fixtures/setup_convert_xml_yolo.py b/data_preprocessing/tests/fixtures/setup_convert_xml_yolo.py index 61a9f6a..c4aca94 100644 --- a/data_preprocessing/tests/fixtures/setup_convert_xml_yolo.py +++ b/data_preprocessing/tests/fixtures/setup_convert_xml_yolo.py @@ -38,4 +38,4 @@ def create_temp_file_io(tmpdir_factory): r" " ) test_xml.write(test_xml_string) - return test_xml, test_outdir \ No newline at end of file + return test_xml, test_outdir diff --git a/data_preprocessing/tests/fixtures/setup_prepare_yolov5.py b/data_preprocessing/tests/fixtures/setup_prepare_yolov5.py index 0b2699c..426c429 100644 --- a/data_preprocessing/tests/fixtures/setup_prepare_yolov5.py +++ b/data_preprocessing/tests/fixtures/setup_prepare_yolov5.py @@ -1,6 +1,7 @@ import pytest from pathlib import Path + @pytest.fixture(scope="session") def setup_target_dir(tmpdir_factory): test_target_dir = tmpdir_factory.mktemp("data_testset") @@ -15,6 +16,6 @@ def setup_target_dir(tmpdir_factory): "val_label_dir": str(val_label_dir), "train_img_dir": str(train_img_dir), "val_img_dir": str(val_img_dir), - "target_dir": str(test_target_dir) + "target_dir": str(test_target_dir), } - return setup_dict \ No newline at end of file + return setup_dict diff --git a/data_preprocessing/tests/fixtures/setup_sort_yolo_images.py b/data_preprocessing/tests/fixtures/setup_sort_yolo_images.py index 4865199..4b83442 100644 --- a/data_preprocessing/tests/fixtures/setup_sort_yolo_images.py +++ b/data_preprocessing/tests/fixtures/setup_sort_yolo_images.py @@ -1,6 +1,7 @@ import pytest from pathlib import Path + @pytest.fixture(scope="session") def setup_yolo_dirs(tmpdir_factory): @@ -10,7 +11,6 @@ def setup_yolo_dirs(tmpdir_factory): test_label_two = test_label_dir.join("test_2.txt") test_label_two.write("test_label_2") - test_img_dir = tmpdir_factory.mktemp("train") test_img_one = test_img_dir.join("test_1.png") test_img_one.write("test_img_1") @@ -22,9 +22,9 @@ def setup_yolo_dirs(tmpdir_factory): assert test_img_one.read() == "test_img_1" assert test_img_two.read() == "test_img_2" - return test_label_dir, test_img_dir + @pytest.fixture(scope="session") def setup_yolo_rename_dirs(tmpdir_factory): @@ -40,4 +40,3 @@ def setup_yolo_rename_dirs(tmpdir_factory): assert test_img_one.read() == "test_img_1" return test_old_label_dir, test_old_img_dir - diff --git a/data_preprocessing/tests/fixtures/setup_train_test_split.py b/data_preprocessing/tests/fixtures/setup_train_test_split.py index a0ece7a..6121d03 100644 --- a/data_preprocessing/tests/fixtures/setup_train_test_split.py +++ b/data_preprocessing/tests/fixtures/setup_train_test_split.py @@ -16,21 +16,23 @@ def setup_file_list(tmpdir_factory): str(test_xml_two), str(test_xml_three), str(test_xml_four), - ] + ] return test_file_list + @pytest.fixture(scope="session") def setup_out_dir(tmpdir_factory): temp_dir = tmpdir_factory.mktemp("test_outdir") return temp_dir + @pytest.fixture(scope="session") def setup_move_img_files(tmpdir_factory): test_labels_dir = tmpdir_factory.mktemp("label") test_labels_dir.join("test_1.xml") test_labels_dir.join("test_2.xml") - + test_train_dir = tmpdir_factory.mktemp("train") test_valid_dir = tmpdir_factory.mktemp("valid") @@ -42,30 +44,24 @@ def setup_move_img_files(tmpdir_factory): assert test_img_one.read() == "test1" assert test_img_two.read() == "test2" - test_val_dict = { - "Filename": [str(test_img_dir)+"/test_1.png"] - } - - test_train_dict = { - "Filename": [str(test_img_dir)+"/test_2.png"] - } + test_val_dict = {"Filename": [str(test_img_dir) + "/test_1.png"]} + + test_train_dict = {"Filename": [str(test_img_dir) + "/test_2.png"]} test_train_set = pd.DataFrame.from_dict(test_train_dict) test_val_set = pd.DataFrame.from_dict(test_val_dict) - params = { "split_dataset": { "train_dir": str(test_train_dir), "valid_dir": str(test_valid_dir), "img_dir": str(test_img_dir), - "file_type": "png" + "file_type": "png", } } setup_dict = { "train": test_train_set, "test": test_val_set, "labels": str(test_labels_dir), - "params": params + "params": params, } return setup_dict - diff --git a/data_preprocessing/tests/test_convert_xml_to_yolotxt.py b/data_preprocessing/tests/test_convert_xml_to_yolotxt.py index 9e38c9c..e37d357 100644 --- a/data_preprocessing/tests/test_convert_xml_to_yolotxt.py +++ b/data_preprocessing/tests/test_convert_xml_to_yolotxt.py @@ -13,20 +13,20 @@ def test_yolo_conversion(): boundaries = (469, 472, 28, 37) - width = 512.0 height = 512.0 bounding_box = funcs.yolo_conversion((width, height), boundaries) expected_bounding_box = [0.9189453125, 0.0634765625, 0.005859375, 0.017578125] assert list(bounding_box) == expected_bounding_box + @pytest.mark.usefixtures("create_temp_file_io") def test_convert_xml_annotation_to_yolo(create_temp_file_io): test_xml, test_outdir = create_temp_file_io print(test_xml, test_outdir) - funcs.convert_xml_annotation_to_yolo(test_xml, test_outdir, True, ['boat']) + funcs.convert_xml_annotation_to_yolo(test_xml, test_outdir, True, ["boat"]) assert len(os.listdir(test_outdir)) == 1 assert os.listdir(test_outdir)[0] == "test.txt" - output_file = os.path.join(test_outdir,"test.txt") + output_file = os.path.join(test_outdir, "test.txt") output_text = Path(output_file).read_text().replace("\n", "") - assert output_text == "0 0.9189453125 0.0634765625 0.005859375 0.017578125" \ No newline at end of file + assert output_text == "0 0.9189453125 0.0634765625 0.005859375 0.017578125" diff --git a/data_preprocessing/tests/test_prepare_yolov5.py b/data_preprocessing/tests/test_prepare_yolov5.py index ddcb908..a90076c 100644 --- a/data_preprocessing/tests/test_prepare_yolov5.py +++ b/data_preprocessing/tests/test_prepare_yolov5.py @@ -1,10 +1,10 @@ - import pytest import os from pathlib import Path from ..scripts import prepare_yolov5 as funcs from .fixtures.setup_prepare_yolov5 import setup_target_dir + @pytest.mark.usefixtures("setup_target_dir") def test_create_yolo_dirs(setup_target_dir): setup_dict = setup_target_dir @@ -13,11 +13,9 @@ def test_create_yolo_dirs(setup_target_dir): setup_dict["val_label_dir"], setup_dict["train_img_dir"], setup_dict["val_img_dir"], - setup_dict["target_dir"] + setup_dict["target_dir"], ) assert len(output_dirs) == 4 -#@pytest.mark.usefixtures("setup_yolo_rename_dirs") - - +# @pytest.mark.usefixtures("setup_yolo_rename_dirs") diff --git a/data_preprocessing/tests/test_sort_yolo_images.py b/data_preprocessing/tests/test_sort_yolo_images.py index 49ba0ed..39c5604 100644 --- a/data_preprocessing/tests/test_sort_yolo_images.py +++ b/data_preprocessing/tests/test_sort_yolo_images.py @@ -1,21 +1,21 @@ - import pytest import os from pathlib import Path from ..scripts import sort_yolo_images as funcs from .fixtures.setup_sort_yolo_images import setup_yolo_dirs, setup_yolo_rename_dirs + @pytest.mark.usefixtures("setup_yolo_dirs") def test_create_yolo_dirs(setup_yolo_dirs): test_label_dir, _ = setup_yolo_dirs workdir = str(Path(test_label_dir).parent) assert str(test_label_dir) == os.path.join(workdir, test_label_dir) - assert len(os.listdir(os.path.join(workdir,"label_sortyolo0"))) == 2 - assert len(os.listdir(os.path.join(workdir,"train0"))) == 2 + assert len(os.listdir(os.path.join(workdir, "label_sortyolo0"))) == 2 + assert len(os.listdir(os.path.join(workdir, "train0"))) == 2 out_img_dir, out_label_dir = funcs.create_yolo_dirs( - workdir, - "train0", + workdir, + "train0", "label_sortyolo0", ) print(out_img_dir, out_label_dir) @@ -24,6 +24,7 @@ def test_create_yolo_dirs(setup_yolo_dirs): assert len(os.listdir(out_img_dir)) == 2 assert len(os.listdir(out_label_dir)) == 2 + @pytest.mark.usefixtures("setup_yolo_rename_dirs") def test_rename_yolo_files(setup_yolo_rename_dirs): test_label_dir, test_img_dir = setup_yolo_rename_dirs @@ -32,7 +33,6 @@ def test_rename_yolo_files(setup_yolo_rename_dirs): assert os.listdir(test_label_dir)[0] == "test_1.txt" assert os.listdir(test_img_dir)[0] == "test_1.jpg" funcs.rename_yolo_files(str(workdir), test_label_dir, "jpg") - + assert os.listdir(test_label_dir)[0] == "00001.txt" assert os.listdir(test_img_dir)[0] == "00001.jpg" - diff --git a/data_preprocessing/tests/test_train_test_split.py b/data_preprocessing/tests/test_train_test_split.py index 67b835c..7feda76 100644 --- a/data_preprocessing/tests/test_train_test_split.py +++ b/data_preprocessing/tests/test_train_test_split.py @@ -2,7 +2,11 @@ import pytest import pandas as pd from pathlib import Path -from .fixtures.setup_train_test_split import setup_file_list, setup_move_img_files, setup_out_dir +from .fixtures.setup_train_test_split import ( + setup_file_list, + setup_move_img_files, + setup_out_dir, +) from ..scripts import train_test_split as funcs @@ -14,16 +18,22 @@ def test_float_fraction(): with pytest.raises(Exception) as below_zero_exception: funcs.float_fraction(-0.5) - assert below_zero_exception.value.args[0] == "Argument should be a fraction! Must be <= 1.0 and >= 0.0" + assert ( + below_zero_exception.value.args[0] + == "Argument should be a fraction! Must be <= 1.0 and >= 0.0" + ) with pytest.raises(Exception) as above_one_exception: funcs.float_fraction(1.5) - assert above_one_exception.value.args[0] == "Argument should be a fraction! Must be <= 1.0 and >= 0.0" - + assert ( + above_one_exception.value.args[0] + == "Argument should be a fraction! Must be <= 1.0 and >= 0.0" + ) assert funcs.float_fraction(0.75) <= 1.0 assert funcs.float_fraction(0.75) >= 0.0 + @pytest.mark.usefixtures("setup_file_list") def test_split_dataset(setup_file_list): test_file_list = setup_file_list @@ -31,6 +41,7 @@ def test_split_dataset(setup_file_list): assert len(train_set) == 3 assert len(test_set) == 1 + @pytest.mark.usefixtures("setup_out_dir") def test_output_files(setup_out_dir): outdir = setup_out_dir @@ -38,8 +49,9 @@ def test_output_files(setup_out_dir): test_list = pd.DataFrame() funcs.output_files(outdir, train_list, test_list) assert len(os.listdir(outdir)) == 2 - assert 'xml_annotations_train.txt' in os.listdir(outdir) - assert 'xml_annotations_test.txt' in os.listdir(outdir) + assert "xml_annotations_train.txt" in os.listdir(outdir) + assert "xml_annotations_test.txt" in os.listdir(outdir) + @pytest.mark.usefixtures("setup_move_img_files") def test_move_image_files(setup_move_img_files): @@ -50,5 +62,5 @@ def test_move_image_files(setup_move_img_files): test_variables["train"], test_variables["test"], test_variables["labels"], - test_variables["params"] + test_variables["params"], ) From 548b0080c8e59e642aaceb7e16c3b044f35ee939 Mon Sep 17 00:00:00 2001 From: Harry Moss Date: Thu, 17 Dec 2020 18:02:01 +0000 Subject: [PATCH 2/2] :construction_worker: Adds black check to CI build --- .github/workflows/main.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 04d6e86..b252759 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -34,5 +34,8 @@ jobs: - name: Test with pytest run: | pytest data_preprocessing/tests/ - + - name: Style check + run: black --check data_preprocessing + +