Merge pull request #48 from Project-AgML/dev

Preparation for v0.5.0

Author: amogh7joshi

.github/workflows/update-datasets.yml

@@ -7,11 +7,8 @@ on:
   push:
     paths:
       - agml/_assets/public_datasources.json
-  pull_request:
-    paths:
-      - agml/_assets/public_datasources.json
 
-permissions: read-all|write-all
+permissions: write-all
 
 jobs:
   build:
@@ -20,6 +17,8 @@ jobs:
 
     steps:
     - uses: actions/checkout@v3
+      with:
+        ref: dev
     - name: Set up Python 3.8
       uses: actions/setup-python@v3
       with:
@@ -28,14 +27,17 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+        python3 setup.py install
     - name: Configure Git
       run: |
-        git config user.name "AgML Dataset Update Bot"
-        git config user.email "joshi.amoghn@gmail.com"
+        git config user.name 'github-actions[bot]'
+        git config user.email 'github-actions[bot]@users.noreply.github.com'
     - name: Update Datasets
       run: |
         python scripts/run_full_dataset_update.py
-        git add agml/_assets/*
+        git add -f agml/_assets/*
+        git add -f docs/
+        git add README.md
         git commit -m "Updated information for new dataset"
         git push origin dev
-        
+

MANIFEST.in

@@ -4,3 +4,4 @@ include agml/_assets/shape_info.pickle
 include agml/_helios/helios_install.sh
 include agml/synthetic/synthetic_data_generation/*
 prune agml/_internal
+prune .github

README.md

@@ -131,6 +131,12 @@ You're now ready to use AgML for training your own models!
 [peachpear_flower_segmentation](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/peachpear_flower_segmentation.md) | Semantic Segmentation | 42 | 
 [red_grapes_and_leaves_segmentation](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/red_grapes_and_leaves_segmentation.md) | Semantic Segmentation | 258 | 
 [white_grapes_and_leaves_segmentation](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/white_grapes_and_leaves_segmentation.md) | Semantic Segmentation | 273 | 
+[ghai_romaine_detection](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/ghai_romaine_detection.md) | Object Detection | 500 |
+[ghai_green_cabbage_detection](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/ghai_green_cabbage_detection.md) | Object Detection | 500 |
+[ghai_iceberg_lettuce_detection](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/ghai_iceberg_lettuce_detection.md) | Object Detection | 500 |
+[riseholme_strawberry_classification_2021](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/riseholme_strawberry_classification_2021.md) | Image Classification | 3520 |
+[ghai_broccoli_detection](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/ghai_broccoli_detection.md) | Object Detection | 500 |
+[bean_synthetic_earlygrowth_aerial](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/bean_synthetic_earlygrowth_aerial.md) | Semantic Segmentation | 2500 |
 
 ## Usage Information
 
@@ -161,4 +167,4 @@ a bug or feature that you would like to see implemented, please don't hesitate t
 See the [contributing guidelines](/CONTRIBUTING.md) for more information.
 
 ## Funding
-This project is partly funded by the [National AI Institute for Food Systems (AIFS)](https://aifs.ucdavis.edu).
+This project is partly funded by the [National AI Institute for Food Systems (AIFS)](https://aifs.ucdavis.edu

agml/__init__.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 __version__ = '0.4.7'
-__all__ = ['data', 'synthetic', 'backend', 'viz']
+__all__ = ['data', 'synthetic', 'backend', 'viz', 'io']
 
 
 # If AgML is being imported for the first time, then we need to setup
@@ -25,12 +25,13 @@ def _setup():
         _os.makedirs(_os.path.expanduser('~/.agml'))
         with open(_os.path.join(
                 _os.path.expanduser('~/.agml/config.json')), 'w') as f:
-            _json.dump({'data_path': _os.path.expanduser('~/.agml/datasets')}, f)
+            _json.dump({'data_path': _os.path.expanduser('~/.agml/datasets'),
+                        'viz_backend': 'matplotlib'}, f)
 _setup(); del _setup # noqa
 
 
 # There are no top-level imported functions or classes, only the modules.
-from . import data, backend, synthetic, viz
+from . import data, backend, synthetic, viz, io
 
 
 

agml/_assets/public_datasources.json

@@ -627,8 +627,8 @@
         "ml_task": "object_detection",
         "ag_task": "fruit_detection",
         "location": {
-            "continent": "north_america",
-            "country": "usa"
+            "continent": "digital",
+            "country": "digital"
         },
         "sensor_modality": "rgb",
         "real_synthetic": "synthetic",
@@ -1105,5 +1105,130 @@
                 0.276553213596344
             ]
         }
+    },
+    "ghai_iceberg_lettuce_detection": {
+        "classes": {
+            "1": "iceberg_lettuce"
+        },
+        "ml_task": "object_detection",
+        "ag_task": "crop_detection",
+        "location": {
+            "continent": "north_america",
+            "country": "usa"
+        },
+        "sensor_modality": "rgb",
+        "real_synthetic": "real",
+        "platform": "handheld/ground",
+        "input_data_format": "jpg",
+        "annotation_format": "coco_json",
+        "n_images": "500",
+        "docs_url": "https://github.com/AxisAg/GHAIDatasets/blob/main/datasets/iceberg.md",
+        "external_image_sources": [],
+        "stats": {
+            "mean": [
+                0.3193310499191284,
+                0.4143226742744446,
+                0.3003973960876465
+            ],
+            "std": [
+                0.2828165590763092,
+                0.2841264605522156,
+                0.25660744309425354
+            ]
+        }
+    },
+    "riseholme_strawberry_classification_2021": {
+        "classes": {
+            "0": "anomalous",
+            "1": "occluded",
+            "2": "ripe",
+            "3": "unripe"
+        },
+        "ml_task": "image_classification",
+        "ag_task": "disease_classification",
+        "sensor_modality": "rgb",
+        "real_synthetic": "real",
+        "platform": "ground",
+        "input_data_format": "png",
+        "annotation_format": "directory_names",
+        "n_images": "3520",
+        "docs_url": "https://github.com/ctyeong/Riseholme-2021",
+        "external_image_sources": [],
+        "stats": {
+            "mean": [
+                0.5753855109214783,
+                0.4637872874736786,
+                0.3637043833732605
+            ],
+            "std": [
+                0.1983788013458252,
+                0.19222092628479004,
+                0.1773509532213211
+            ]
+        }
+    },
+    "ghai_broccoli_detection": {
+        "classes": {
+            "1": "canopy",
+            "2": "crown"
+        },
+        "ml_task": "object_detection",
+        "ag_task": "crop_detection",
+        "location": {
+            "continent": "north_america",
+            "country": "usa"
+        },
+        "sensor_modality": "rgb",
+        "real_synthetic": "real",
+        "platform": "handheld/ground",
+        "input_data_format": "jpg",
+        "annotation_format": "coco_json",
+        "n_images": "500",
+        "docs_url": "https://github.com/AxisAg/GHAIDatasets/blob/main/datasets/broccoli.md",
+        "external_image_sources": [],
+        "stats": {
+            "mean": [
+                0.42832377552986145,
+                0.45015057921409607,
+                0.43816232681274414
+            ],
+            "std": [
+                0.24546495079994202,
+                0.23975253105163574,
+                0.23829665780067444
+            ]
+        }
+    },
+    "bean_synthetic_earlygrowth_aerial": {
+        "classes": {
+            "1": "leaves",
+            "2": "branches"
+        },
+        "ml_task": "semantic_segmentation",
+        "ag_task": "plant_segmentation",
+        "location": {
+            "continent": "digital",
+            "country": "digital"
+        },
+        "sensor_modality": "rgb",
+        "real_synthetic": "synthetic",
+        "platform": "aerial",
+        "input_data_format": "jpg",
+        "annotation_format": "image",
+        "n_images": "2500",
+        "docs_url": "https://github.com/Project-AgML/AgML/blob/main/docs/datasets/bean_synthetic_earlygrowth_aerial.md",
+        "external_image_sources": [],
+        "stats": {
+            "mean": [
+                0.24804304540157318,
+                0.2218194305896759,
+                0.16611839830875397
+            ],
+            "std": [
+                0.10789012163877487,
+                0.12054657936096191,
+                0.08992248773574829
+            ]
+        }
     }
 }

agml/_assets/shape_info.pickle

@@ -122,5 +122,21 @@
     "ghai_green_cabbage_detection": {
         "license": "CC BY-SA 4.0",
         "citation": ""
+    },
+    "ghai_iceberg_lettuce_detection": {
+        "license": "CC BY-SA 4.0",
+        "citation": ""
+    },
+    "riseholme_strawberry_classification_2021": {
+        "license": "",
+        "citation": "@inproceedings{CWSC21,\n  title={Self-supervised Representation Learning for Reliable Robotic Monitoring of Fruit Anomalies},\n  author={Choi, Taeyeong and Would, Owen and Salazar-Gomez, Adrian and Cielniak, Grzegorz},\n  booktitle={2022 International Conference on Robotics and Automation (ICRA)},\n  pages={2266--2272},\n  year={2022},\n  organization={IEEE}\n}"
+    },
+    "ghai_broccoli_detection": {
+        "license": "CC BY-SA 4.0",
+        "citation": ""
+    },
+    "bean_synthetic_earlygrowth_aerial": {
+        "license": "MIT",
+        "citation": "@ARTICLE{10.3389/fpls.2019.01185,\n  \nAUTHOR={Bailey, Brian N.},   \n\t \nTITLE={Helios: A Scalable 3D Plant and Environmental Biophysical Modeling Framework},      \n\t\nJOURNAL={Frontiers in Plant Science},      \n\t\nVOLUME={10},      \n\t\nYEAR={2019},      \n\t  \nURL={https://www.frontiersin.org/article/10.3389/fpls.2019.01185},       \n\t\nDOI={10.3389/fpls.2019.01185},      \n\t\nISSN={1664-462X},   \n   \nABSTRACT={This article presents an overview of Helios, a new three-dimensional (3D) plant and environmental modeling framework. Helios is a model coupling framework designed to provide maximum flexibility in integrating and running arbitrary 3D environmental system models. Users interact with Helios through a well-documented open-source C++ API. Version 1.0 comes with model plug-ins for radiation transport, the surface energy balance, stomatal conductance, photosynthesis, solar position, and procedural tree generation. Additional plug-ins are also available for visualizing model geometry and data and for processing and integrating LiDAR scanning data. Many of the plug-ins perform calculations on the graphics processing unit, which allows for efficient simulation of very large domains with high detail. An example modeling study is presented in which leaf-level heterogeneity in water usage and photosynthesis of an orchard is examined to understand how this leaf-scale variability contributes to whole-tree and -canopy fluxes.}\n}"
     }
 }

agml/_assets/source_citations.json

@@ -970,11 +970,57 @@ def ghai_green_cabbage_detection(self, dataset_name):
         shutil.move(os.path.join(original_dir, 'coco.json'),
                     os.path.join(processed_dir, 'annotations.json'))
 
+    def ghai_iceberg_lettuce_detection(self, dataset_name):
+        # Create processed directories
+        original_dir = os.path.join(self.data_original_dir, dataset_name)
+        processed_dir = os.path.join(self.data_processed_dir, dataset_name)
+        processed_image_dir = os.path.join(processed_dir, 'images')
+        os.makedirs(processed_image_dir, exist_ok = True)
+
+        # Move images
+        for image in glob.glob(os.path.join(original_dir, '*.jpg')):
+            shutil.move(image, processed_image_dir)
+        shutil.move(os.path.join(original_dir, 'coco.json'),
+                    os.path.join(processed_dir, 'annotations.json'))
+
+    def riseholme_strawberry_classification_2021(self, dataset_name):
+        # Create processed data directory.
+        original_dir = os.path.join(
+            self.data_original_dir, 'Riseholme-2021-main', 'Data')
+        processed_dir = os.path.join(self.data_processed_dir, dataset_name)
+
+        # Load all of the individual images and keep a mapping to their corresponding directory.
+        images = {
+            'anomalous': get_file_list(os.path.join(original_dir, 'Anomalous')),
+            **{dir_.lower(): get_file_list(os.path.join(original_dir, 'Normal', dir_))
+               for dir_ in os.listdir(os.path.join(original_dir, 'Normal'))}
+        }
+
+        # Create the output file structure.
+        for class_name, image_set in images.items():
+            class_dir = os.path.join(processed_dir, class_name)
+            os.makedirs(class_dir, exist_ok = True)
+            for file in image_set:
+                shutil.copyfile(file, os.path.join(class_dir, os.path.basename(file)))
+
+    def ghai_broccoli_detection(self, dataset_name):
+        # Create processed directories
+        original_dir = os.path.join(self.data_original_dir, dataset_name)
+        processed_dir = os.path.join(self.data_processed_dir, dataset_name)
+        processed_image_dir = os.path.join(processed_dir, 'images')
+        os.makedirs(processed_image_dir, exist_ok = True)
+
+        # Move images
+        for image in tqdm(glob.glob(os.path.join(original_dir, '*.jpg'))):
+            shutil.move(image, processed_image_dir)
+        shutil.move(os.path.join(original_dir, 'coco.json'),
+                    os.path.join(processed_dir, 'annotations.json'))
+
 
 if __name__ == '__main__':
     # Initialize program arguments.
     ap = argparse.ArgumentParser()
-    ap.add_argument("--data_dir", type = str, default = '../../data',
+    ap.add_argument("--data_dir", type = str, default = '../../data_new',
                     help = "The directory containing two sub-directories, "
                            "`original` and `processed`, with the data.")
     ap.add_argument("--dataset", type = str,
@@ -983,4 +1029,10 @@ def ghai_green_cabbage_detection(self, dataset_name):
 
     # Execute the preprocessing.
     p = PublicDataPreprocessor(os.path.abspath(args.data_dir))
+    print("Processing dataset")
     p.preprocess(args.dataset)
+    print("Converting dataset")
+    os.chdir(f'{args.data_dir}/processed')
+    os.system(f'zip -r {args.dataset}.zip {args.dataset} -x ".*" -x "__MACOSX"')
+
+

agml/_internal/preprocess.py

@@ -34,7 +34,6 @@
 DATASET_SAVE_DIR: str
 
 
-
 # This is similar to `DATASET_SAVE_DIR`, but is for synthetically generated
 # datasets using Helios. By default, this will be SUPER_BASE_DIR/synthetic,
 # but it can be overridden. The value is set upon instantiation of the module.
@@ -96,11 +95,7 @@ def set_data_save_path(location = None):
         raise NotADirectoryError(
             f"The provided destination {location} does "
             f"not exist, or is not a directory.")
-    with open(os.path.join(SUPER_BASE_DIR, 'config.json'), 'r') as f:
-        contents = json.load(f)
-    contents['data_path'] = os.path.realpath(os.path.abspath(location))
-    with open(os.path.join(SUPER_BASE_DIR, 'config.json'), 'w') as f:
-        json.dump(contents, f)
+    _update_config('data_path', os.path.realpath(os.path.abspath(location)))
     return
 
 
@@ -132,11 +127,7 @@ def set_synthetic_save_path(location = None):
         raise NotADirectoryError(
             f"The provided destination {location} does "
             f"not exist, or is not a directory.")
-    with open(os.path.join(SUPER_BASE_DIR, 'config.json'), 'r') as f:
-        contents = json.load(f)
-    contents['synthetic_data_path'] = os.path.realpath(os.path.abspath(location))
-    with open(os.path.join(SUPER_BASE_DIR, 'config.json'), 'w') as f:
-        json.dump(contents, f)
+    _update_config('synthetic_data_path', os.path.realpath(os.path.abspath(location)))
     return
 
 
@@ -167,13 +158,28 @@ def set_model_save_path(location = None):
         raise NotADirectoryError(
             f"The provided destination {location} does "
             f"not exist, or is not a directory.")
+    _update_config('model_path', os.path.realpath(os.path.abspath(location)))
+    return
+
+
+def _get_config(param):
+    """Update the configuration file with a new parameter."""
+    global SUPER_BASE_DIR
+    with open(os.path.join(SUPER_BASE_DIR, 'config.json'), 'r') as f:
+        contents = json.load(f)
+    if param not in contents.keys():
+        return None
+    return contents[param]
+
+
+def _update_config(param, value):
+    """Update the configuration file with a new parameter."""
+    global SUPER_BASE_DIR
     with open(os.path.join(SUPER_BASE_DIR, 'config.json'), 'r') as f:
         contents = json.load(f)
-    contents['model_path'] = os.path.realpath(os.path.abspath(location))
+    contents[param] = value
     with open(os.path.join(SUPER_BASE_DIR, 'config.json'), 'w') as f:
         json.dump(contents, f)
-    return
-
 
 
 def clear_all_datasets():