Update README.md

README.md

@@ -10,6 +10,8 @@ We chose to use a deep learning convolutional neural network (CNN) to identify a
 These notebooks and scripts are created in python through the VS Code platform. 
 Before running any scripts or notebooks, the user should import the necessary packages listed below, should the necessary packages not be available you can try a 'pip install **package name**' 
 
+## Installation
+
 Relevant packages and libraries to install include :
 
 ```python
@@ -30,6 +32,8 @@ The **goes2go** package is developed as an easy and efficient way to access GOES
 
 [Brian's GitHub](https://github.com/blaylockbk/goes2go)
 
+[AWS CLI Download Instructions](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html)
+
 The next package that may be unfamiliar is subprocess. We use the subprocess package to access the GOES data stored on the AWS server.
 
 To set-up the AWS CLI on your own please follow the links under **GOES on AWS**:

notebooks/Data_Cleaning.ipynb

@@ -150,13 +150,13 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "def CloudImageryProcessingFunction(file):\n",
+    "import numpy as np\n",
+    "import xarray as xr\n",
+    "from netCDF4 import Dataset\n",
     "\n",
-    "    # for file in nc_file_list:\n",
+    "def CloudImageryProcessingFunction(file):\n",
     "    print(file)\n",
-    "\n",
     "    ds = xr.open_dataset(file)\n",
-    "\n",
     "    file_id = Dataset(file)\n",
     "\n",
     "    # Call function to calculate latitude and longitude from GOES ABI fixed grid projection data\n",
@@ -167,6 +167,20 @@
     "    G = ds['CMI_C03'].data\n",
     "    B = ds['CMI_C01'].data\n",
     "\n",
+    "    # Define a function to remove outliers based on IQR\n",
+    "    def remove_outliers(data):\n",
+    "        Q1, Q3 = np.percentile(data, [25, 75])\n",
+    "        IQR = Q3 - Q1\n",
+    "        lower_bound = Q1 - 1.5 * IQR\n",
+    "        upper_bound = Q3 + 1.5 * IQR\n",
+    "        data = np.clip(data, lower_bound, upper_bound)\n",
+    "        return data\n",
+    "\n",
+    "    # Remove outliers from each channel\n",
+    "    R = remove_outliers(R)\n",
+    "    G = remove_outliers(G)\n",
+    "    B = remove_outliers(B)\n",
+    "\n",
     "    # Apply range limits for each channel. RGB values must be between 0 and 1\n",
     "    R = np.clip(R, 0, 1)\n",
     "    G = np.clip(G, 0, 1)\n",
@@ -183,14 +197,10 @@
     "    G_true = np.maximum(G_true, 0)\n",
     "    G_true = np.minimum(G_true, 1)\n",
     "\n",
-    "\n",
     "    # The RGB array for the true color image\n",
     "    RGB = np.dstack([R, G_true, B])\n",
     "    rgb = RGB[:,:-1,:] # reverse the green???\n",
     "\n",
-    "    # Assuming abi_lat and abi_lon have shapes (M, N), and R/G/B are (M, N)\n",
-    "    # rgb = np.dstack([R, G, B])  # Stack the individual R, G, B components        CHECKING THIS\n",
-    "\n",
     "    return abi_lat, abi_lon, rgb\n"
    ]
   },
@@ -234,21 +244,34 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# data processing\n",
-    "def CloudOpticalDepthProcessingFunction(file):\n",
+    "import numpy as np\n",
+    "import xarray as xr\n",
+    "from netCDF4 import Dataset\n",
     "\n",
-    "    # for file in nc_file_list:\n",
+    "# Data processing\n",
+    "def CloudOpticalDepthProcessingFunction(file):\n",
     "    print(file)\n",
-    "\n",
     "    ds = xr.open_dataset(file)\n",
-    "\n",
     "    file_id = Dataset(file)\n",
     "\n",
     "    # Call function to calculate latitude and longitude from GOES ABI fixed grid projection data\n",
     "    abi_lat, abi_lon = calculate_degrees(file_id)\n",
     "\n",
+    "    # Load Cloud Optical Depth (COD) data\n",
     "    cod = ds['COD'].data\n",
     "\n",
+    "    # Define a function to remove outliers based on IQR\n",
+    "    def remove_outliers(data):\n",
+    "        Q1, Q3 = np.percentile(data, [25, 75])\n",
+    "        IQR = Q3 - Q1\n",
+    "        lower_bound = Q1 - 1.5 * IQR\n",
+    "        upper_bound = Q3 + 1.5 * IQR\n",
+    "        data = np.clip(data, lower_bound, upper_bound)\n",
+    "        return data\n",
+    "\n",
+    "    # Remove outliers from the COD data\n",
+    "    cod = remove_outliers(cod)\n",
+    "\n",
     "    return abi_lat, abi_lon, cod\n"
    ]
   },
@@ -292,20 +315,34 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "def ReflectedSWProcessingFunction(file):\n",
+    "import numpy as np\n",
+    "import xarray as xr\n",
+    "from netCDF4 import Dataset\n",
     "\n",
-    "    # for file in nc_file_list:\n",
+    "def ReflectedSWProcessingFunction(file):\n",
     "    print(file)\n",
-    "\n",
     "    ds = xr.open_dataset(file)\n",
-    "\n",
     "    file_id = Dataset(file)\n",
     "\n",
+    "    # Extract latitude and longitude\n",
     "    abi_lat = ds['lat'].values\n",
     "    abi_lon = ds['lon'].values\n",
     "\n",
+    "    # Load Reflected Shortwave Radiation (RSR) data\n",
     "    rsr = ds['RSR'].data\n",
     "\n",
+    "    # Define a function to remove outliers based on IQR\n",
+    "    def remove_outliers(data):\n",
+    "        Q1, Q3 = np.percentile(data, [25, 75])\n",
+    "        IQR = Q3 - Q1\n",
+    "        lower_bound = Q1 - 1.5 * IQR\n",
+    "        upper_bound = Q3 + 1.5 * IQR\n",
+    "        data = np.clip(data, lower_bound, upper_bound)\n",
+    "        return data\n",
+    "\n",
+    "    # Remove outliers from the RSR data\n",
+    "    rsr = remove_outliers(rsr)\n",
+    "\n",
     "    return abi_lat, abi_lon, rsr\n"
    ]
   },
@@ -325,7 +362,7 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "C:\\Users\\adhal\\AppData\\Local\\Temp\\ipykernel_37992\\3872720453.py:20: RuntimeWarning: invalid value encountered in sqrt\n",
+      "C:\\Users\\adhal\\AppData\\Local\\Temp\\ipykernel_29456\\3872720453.py:20: RuntimeWarning: invalid value encountered in sqrt\n",
       "  r_s = (-1.0*b_var - np.sqrt((b_var**2)-(4.0*a_var*c_var)))/(2.0*a_var)\n"
      ]
     },