feat(abr_testing): remove temp data > 60 days (#17022)

# Overview

Before running temperature sensors, all data older than 60 days in the
abr ambient temperature sheet is removed

Author: AnthonyNASC20

abr-testing/abr_testing/tools/abr_setup.py

@@ -4,7 +4,10 @@
 import configparser
 import traceback
 import sys
+from datetime import datetime, timedelta
+from typing import Any
 from hardware_testing.scripts import ABRAsairScript  # type: ignore
+from abr_testing.automation import google_sheets_tool
 from abr_testing.data_collection import (
     get_run_logs,
     abr_google_drive,
@@ -13,15 +16,60 @@
 from abr_testing.tools import sync_abr_sheet
 
 
+def clean_sheet(sheet_name: str, credentials: str) -> Any:
+    """Remove data older than 60 days from sheet."""
+    sheet = google_sheets_tool.google_sheet(
+        credentials=credentials, file_name=sheet_name, tab_number=0
+    )
+    date_columns = sheet.get_column(3)
+    curr_date = datetime.now()
+    cutoff_days = 60  # Cutoff period in days
+    cutoff_date = curr_date - timedelta(days=cutoff_days)
+
+    rem_rows = []
+    for row_id, date in enumerate(date_columns):
+        # Convert to datetime if needed
+        formatted_date = None
+        if isinstance(date, str):  # Assuming dates might be strings
+            try:
+                formatted_date = datetime.strptime(date, "%m/%d/%Y")
+            except ValueError:
+                try:
+                    formatted_date = datetime.strptime(date, "%Y-%m-%d")
+                except ValueError:
+                    continue
+
+        # Check if the date is older than the cutoff
+        if formatted_date < cutoff_date:
+            rem_rows.append(row_id)
+        if len(rem_rows) > 2000:
+            break
+    if len(rem_rows) == 0:
+        # No more rows to remove
+        print("Nothing to remove")
+        return
+    print(f"Rows to be removed: {rem_rows}")
+    try:
+        sheet.batch_delete_rows(rem_rows)
+        print("deleted rows")
+    except Exception:
+        print("could not delete rows")
+        traceback.print_exc()
+        sys.exit(1)
+    clean_sheet(sheet_name, credentials)
+
+
 def run_sync_abr_sheet(
     storage_directory: str, abr_data_sheet: str, room_conditions_sheet: str
 ) -> None:
     """Sync ABR sheet with temp and lifetime percents."""
     sync_abr_sheet.run(storage_directory, abr_data_sheet, room_conditions_sheet)
 
 
-def run_temp_sensor() -> None:
+def run_temp_sensor(ambient_conditions_sheet: str, credentials: str) -> None:
     """Run temperature sensors on all robots."""
+    # Remove entries > 60 days
+    clean_sheet(ambient_conditions_sheet, credentials)
     processes = ABRAsairScript.run()
     for process in processes:
         process.start()
@@ -71,34 +119,27 @@ def main(configurations: configparser.ConfigParser) -> None:
     ambient_conditions_sheet = None
     sheet_url = None
 
-    has_defaults = False
     # If default is not specified get all values
     default = configurations["DEFAULT"]
-    if len(default) > 0:
-        has_defaults = True
-    try:
-        if has_defaults:
-            storage_directory = default["Storage"]
-            email = default["Email"]
-            drive_folder = default["Drive_Folder"]
-            sheet_name = default["Sheet_Name"]
-            sheet_url = default["Sheet_Url"]
-    except KeyError as e:
-        print("Cannot read config file\n" + str(e))
+    credentials = ""
+    if default:
+        try:
+            credentials = default["Credentials"]
+        except KeyError as e:
+            print("Cannot read config file\n" + str(e))
 
     # Run Temperature Sensors
-    if not has_defaults:
-        ambient_conditions_sheet = configurations["TEMP-SENSOR"]["Sheet_Url"]
+    ambient_conditions_sheet = configurations["TEMP-SENSOR"]["Sheet_Url"]
+    ambient_conditions_sheet_name = configurations["TEMP-SENSOR"]["Sheet_Name"]
     print("Starting temp sensors...")
-    run_temp_sensor()
+    run_temp_sensor(ambient_conditions_sheet_name, credentials)
     print("Temp Sensors Started")
     # Get Run Logs and Record
-    if not has_defaults:
-        storage_directory = configurations["RUN-LOG"]["Storage"]
-        email = configurations["RUN-LOG"]["Email"]
-        drive_folder = configurations["RUN-LOG"]["Drive_Folder"]
-        sheet_name = configurations["RUN-LOG"]["Sheet_Name"]
-        sheet_url = configurations["RUN-LOG"]["Sheet_Url"]
+    storage_directory = configurations["RUN-LOG"]["Storage"]
+    email = configurations["RUN-LOG"]["Email"]
+    drive_folder = configurations["RUN-LOG"]["Drive_Folder"]
+    sheet_name = configurations["RUN-LOG"]["Sheet_Name"]
+    sheet_url = configurations["RUN-LOG"]["Sheet_Url"]
     print(sheet_name)
     if storage_directory and drive_folder and sheet_name and email:
         print("Retrieving robot run logs...")
@@ -113,11 +154,10 @@ def main(configurations: configparser.ConfigParser) -> None:
     if storage_directory and sheet_url and ambient_conditions_sheet:
         run_sync_abr_sheet(storage_directory, sheet_url, ambient_conditions_sheet)
     # Collect calibration data
-    if not has_defaults:
-        storage_directory = configurations["CALIBRATION"]["Storage"]
-        email = configurations["CALIBRATION"]["Email"]
-        drive_folder = configurations["CALIBRATION"]["Drive_Folder"]
-        sheet_name = configurations["CALIBRATION"]["Sheet_Name"]
+    storage_directory = configurations["CALIBRATION"]["Storage"]
+    email = configurations["CALIBRATION"]["Email"]
+    drive_folder = configurations["CALIBRATION"]["Drive_Folder"]
+    sheet_name = configurations["CALIBRATION"]["Sheet_Name"]
     if storage_directory and drive_folder and sheet_name and email:
         print("Retrieving and recording robot calibration data...")
         get_calibration_data(storage_directory, drive_folder, sheet_name, email)