3A (#11)

* RTL Modifications for AE and AWB

* Bug Fixing for 3A in RTL

* Pylint error fixing

* Moved AWB before WB module

* Bug Fixing for AWB-3A

* RTl Modifications in AE

* Made pixel limit calculated through approximations

* Restoration of YUV image from 422 and 444 formats

* Bug Fixing in restore YUV Function

* Bug Fixing for 3A

* Rolled Back YUV changes

* 3A BUG FIXING

* modified automation file according to new position
of AWB in isp_pipeline and generated test vector.

* Removed Linting Errors.

* modified automation file with no linting errors.

* Finalized Code for YUV Format Conversion

* Removed Linting Errors

* resolved linting errors and removed test_func file from
util.

* Removed AE Redundant Code

* Bug Fixing in AE

* Removed wbg_gen.py

---------

Co-authored-by: Maria_Nadeem-10x <[email protected]>

Author: bakhtawar-10xe

.pylintrc

@@ -1,7 +1,8 @@
 [MESSAGES CONTROL]
-disable=unsubscriptable-object,E1136, E1137, R,E0401, C3001, E1121
+disable=unsubscriptable-object,E1136, E1137, R,E0401, C3001, E1121, W0106
 
 # E0401: Unable to import 'util.utils' (import-error)
 # C3001: Lambda expression assigned to a variable. 
 #        Define a function using the "def" keyword instead. (unnecessary-lambda-assignment)
 # E1121: Too many positional arguments for method call (too-many-function-args)
+# W0106: Expression not assigned to anything (expression-not-assigned).

assets/infinite-isp-architecture-initial.png

@@ -1,70 +1,67 @@
 """
+DESCRIPTION: ...
 """
 
 import os
 import sys
 import subprocess
 from pathlib import Path
 import shutil
-import numpy as np
+from datetime import datetime
 import yaml
 from util import utils
-from datetime import datetime
 
 # define module under test, must be the same as defined in the config
-# DUT options = ['crop', 'dead_pixel_correction', 'hdr_stitching', 'black_level_correction', 'OECF', 'digital_gain', 'lens_shading_correction', 
-#                'bayer_noise_reduction', 'white_balance', 'demosaic', 'auto_white_balance', 'color_correction_matrix', 'gamma_correction', 
-#                'auto_exposure', 'color_space_conversion', 'ldci', 'sharpen', '2d_noise_reduction', 'scale', 'yuv_conversion_format']
+# DUT options = ['crop', 'dead_pixel_correction', 'hdr_stitching', 'black_level_correction',
+#                'OECF', 'digital_gain', 'lens_shading_correction','bayer_noise_reduction',
+#                'auto_white_balance', 'white_balance', 'demosaic', 'color_correction_matrix',
+#                'gamma_correction', 'auto_exposure', 'color_space_conversion', 'ldci', 'sharpen',
+#                '2d_noise_reduction', 'scale', 'yuv_conversion_format']
 
 
-DUT = ["dead_pixel_correction", "black_level_correction", "digital_gain"]
+DUT = ["dead_pixel_correction"]
 
 
 # Modules:
-CROP = {"isEnable": False,
-        "new_width": 1280,
-        "new_height": 720}
+CROP = {"isEnable": False, "new_width": 1280, "new_height": 720}
+
+DPC = {"isEnable": True, "dp_threshold": 80}
 
-DPC = {"isEnable": True,
-       "dp_threshold": 80}
-       
 
 HDR = {"isEnable": False}
 
-BLC = {"isEnable": True,
-       "isLinear": True,
-       "r_offset": 200,
-       "gr_offset": 200,
-       "gb_offset": 200,
-       "b_offset": 200}
+BLC = {
+    "isEnable": False,
+    "isLinear": True,
+    "r_offset": 200,
+    "gr_offset": 200,
+    "gb_offset": 200,
+    "b_offset": 200,
+}
 
 OECF = {"isEnable": False}
-DG  = {"is_save": True}
+DG = {"is_save": False}
 
 LSC = {"isEnable": False}
 
-BNR = {"isEnable": False, 
-       "filt_window": 9}
+BNR = {"isEnable": False, "filt_window": 9}
 
-WB = {"isEnable": False,
-      "isAuto": False}
+AWB = {"isEnable": False}
+WB = {"isEnable": False}
 DEM = {"is_save": False}
-AWB = {}
 CCM = {"isEnable": False}
-GC  = {"isEnable": True}
+GC = {"isEnable": False}
 AE = {"isEnable": False}
 
 CSC = {"is_save": False}
-LDCI = {"isEnable": True}
+LDCI = {"isEnable": False}
 
-SHARPEN ={"isEnable": False}
+SHARPEN = {"isEnable": False}
 NR2D = {"isEnable": False}
 
-SCALE = {"isEnable": False, 
-         "isHardware": True}
+SCALE = {"isEnable": False, "isHardware": True}
 
-YUV= {"isEnable": False,
-      "conv_type": '444'}
+YUV = {"isEnable": False, "conv_type": "444"}
 
 # These modules are currently not a part of isp pipeline, but are included in config
 # PRE_GAMMA = {"isEnable": False}
@@ -74,19 +71,19 @@
 # define folder name to save outputs
 folder_name = datetime.now().strftime("%Y%m%d_%H%M%S")
 
-input_ext = ".raw"
+INPUT_EXT = ".raw"
 
 # Setting this flag to True saves yvu array
-swap_uv = True
+SWAP_UV = False
 
 # Set this flag to True if all input files need to be saved in a single folder
-in_single_folder = True
+IN_SINGLE_FOLDER = True
 
 # The path of the dataset
 DATASET_PATH = "./in_frames/normal/data/"
 
 # Parent folder for Images (Path is relative to ./in_frames/normal/)
-PARENT_FOLDER = DATASET_PATH.rsplit('./in_frames/normal/', maxsplit=1)[-1]
+PARENT_FOLDER = DATASET_PATH.rsplit("./in_frames/normal/", maxsplit=1)[-1]
 
 # The path of the default config file
 CONFIG_PATH = "./config/configs.yml"
@@ -97,52 +94,79 @@
 
 # load, update and resave config file according to above parameters
 with open(CONFIG_PATH, "r", encoding="utf-8") as file:
-    config = yaml.safe_load(file)   
+    config = yaml.safe_load(file)
 
 module_tags = list(config.keys())[2:]
-remove = ["swap_uv_channels","pre_gamma", "tone_mapping", "jpeg_conversion"]
+remove = ["swap_uv_channels", "pre_gamma", "tone_mapping", "jpeg_conversion"]
 [module_tags.remove(module) for module in remove]
 
 # Set is_RTl to True to indicate that automation file is being executed
 config["sensor_info"]["is_RTL"] = True
 
 # ensure that the modules are in same order as they are in the config file
-new_params = [CROP, DPC, HDR, BLC, OECF, DG, LSC, BNR, WB, DEM, AWB, CCM, GC, AE, 
-              CSC, LDCI, SHARPEN, NR2D, SCALE, YUV]
+new_params = [
+    CROP,
+    DPC,
+    HDR,
+    BLC,
+    OECF,
+    DG,
+    LSC,
+    BNR,
+    AWB,
+    WB,
+    DEM,
+    CCM,
+    GC,
+    AE,
+    CSC,
+    LDCI,
+    SHARPEN,
+    NR2D,
+    SCALE,
+    YUV,
+]
 
 default_modules = ["digital_gain", "demosaic", "color_space_conversion"]
 
 for idx, module in enumerate(module_tags):
     # save the input and output arrays of module under test with is_save flag
-    try:        
-        if module in DUT or module_tags[idx+1] in DUT:
-            is_save = True
+    try:
+        if module in DUT or module_tags[idx + 1] in DUT:
+            IS_SAVE = True
         elif module in default_modules:
 
-            is_save = new_params[idx]["is_save"]
-        else:  
-            is_save = False  
-        
+            IS_SAVE = new_params[idx]["is_save"]
+        else:
+            IS_SAVE = False
+
         # update swap_uv flag to save yuv or yvu array
-        if module in module_tags[module_tags.index("color_space_conversion"):] and is_save:
-            config["swap_uv_channels"][str(module)] = swap_uv
+        if (
+            module in module_tags[module_tags.index("color_space_conversion") :]
+            and IS_SAVE
+        ):
+            config["swap_uv_channels"][str(module)] = SWAP_UV
 
         if module in DUT and module not in default_modules:
-            assert new_params[idx]["isEnable"]==True, "DUT not enabled in this script."
-        
-        utils.update_config(config, module,new_params[idx].keys(), new_params[idx].values(), is_save)  
+            assert (
+                new_params[idx]["isEnable"] is True
+            ), "DUT not enabled in this script."
+
+        utils.update_config(
+            config, module, new_params[idx].keys(), new_params[idx].values(), IS_SAVE
+        )
     except IndexError:
         pass
 
-# Empty the Results folder in temp folder 
+# Empty the Results folder in temp folder
 shutil.copytree("./out_for_RTL/Results", "./out_for_RTL/temp")
 shutil.rmtree("./out_for_RTL/Results")
 os.mkdir("./out_for_RTL/Results")
 
 # save the config file along with its results
 yaml.add_representer(list, utils.represent_list)
 
-with open(f"./out_for_RTL/Results/configs_automate.yml", "w", encoding="utf-8") as file:
+with open("./out_for_RTL/Results/configs_automate.yml", "w", encoding="utf-8") as file:
     yaml.dump(config, file, sort_keys=False, Dumper=utils.CustomDumper, width=17000)
 
 # loop over images
@@ -157,14 +181,20 @@
 for i, raw_filename in enumerate(RAW_FILENAMES):
     # update filename in config
     config["platform"]["filename"] = PARENT_FOLDER + raw_filename
-    
+
     # update the config file in cofig file
-    with open(f"config/configs.yml", "w", encoding="utf-8") as file:
+    with open("config/configs.yml", "w", encoding="utf-8") as file:
         yaml.safe_dump(config, file, sort_keys=False, default_flow_style=False)
-    
+
     # direct standard output stream to a text file
-    with open(f"./out_for_RTL/Results/{folder_name}_isp_pipeline_log.txt", "a") as text_file:
-        subprocess.run([sys.executable, "isp_pipeline.py"], check=True, stdout=text_file, text=True)
+    with open(
+        f"./out_for_RTL/Results/{folder_name}_isp_pipeline_log.txt",
+        "a",
+        encoding="utf-8",
+    ) as text_file:
+        subprocess.run(
+            [sys.executable, "isp_pipeline.py"], check=True, stdout=text_file, text=True
+        )
 
 shutil.copy(RETAINED_CONFIG, CONFIG_PATH)
 os.remove(RETAINED_CONFIG)
@@ -174,22 +204,22 @@
 path = f"./out_for_RTL/{folder_name}/"
 
 if "crop" in DUT:
-    utils.rename_for_RTL(path, "Inpipeline_crop_")
+    utils.rename_for_rtl(path, "Inpipeline_crop_")
 else:
-    input_module = module_tags[module_tags.index(DUT[0])-1]
-    utils.rename_for_RTL(path, "Out_" + input_module + "_")
+    input_module = module_tags[module_tags.index(DUT[0]) - 1]
+    utils.rename_for_rtl(path, "Out_" + input_module + "_")
 
-# restructure the directory 
-utils.restructure_dir(path, in_single_folder, DUT[-1])
+# restructure the directory
+utils.restructure_dir(path, IN_SINGLE_FOLDER, DUT[-1])
 
 
 # convert the saved numpy arrays to bin files as required by the RTL
-utils.get_RTL_input(path, in_single_folder, input_ext)
+utils.get_rtl_input(path, IN_SINGLE_FOLDER, INPUT_EXT)
 
 # place back the contents of the Results folder
 shutil.copytree("./out_for_RTL/temp", "./out_for_RTL/Results")
 shutil.rmtree("./out_for_RTL/temp")
 
 
 # with open(f"./out_for_RTL/{folder_name}/{folder_name}_comparison_log.txt", "a") as text_file:
-#     subprocess.run([sys.executable, "compare_output.py"], stdout=text_file, text= True)
+#     subprocess.run([sys.executable, "compare_output.py"], stdout=text_file, text= True)