Fetch Periscope data

.env.sample

@@ -0,0 +1,3 @@
+LIST="" #directory of your lists. "ct_membership_lists" for example
+PERISCOPE_URL="" #url shared with your chapter to grab data from PERISCOPE_URL
+PERISCOPE_PASS="" #password for the periscope dashboard

pyproject.toml

@@ -17,7 +17,8 @@ dependencies = [
     "plotly~=6.2",
     "python-dotenv~=1.1",
     "ratelimit~=2.2",
-    "tqdm~=4.67"
+    "selenium>=4.40.0",
+    "tqdm~=4.67",
 ]
 
 [project.optional-dependencies]

src/app.py

@@ -1,7 +1,15 @@
+import threading
+
 import dash
 import dash_bootstrap_components as dbc
 import dash_bootstrap_templates
-from dash import Dash, Input, Output, clientside_callback, html
+from dash import Dash, Input, Output, State, clientside_callback, html, callback
+
+from src.utils.fetch_list import fetch_list
+
+# shared state for the background fetch thread
+_fetch_state: dict = {"running": False, "status": ""}
+_fetch_lock = threading.Lock()
 
 FAVICON = {
     "rel": "icon",
@@ -37,5 +45,47 @@
     Input(component_id="color-mode-switch", component_property="value"),
 )
 
+def _run_fetch():
+    try:
+        fetch_list()
+        with _fetch_lock:
+            _fetch_state["status"] = "Done."
+    except Exception as e:
+        with _fetch_lock:
+            _fetch_state["status"] = f"Error: {e}"
+    finally:
+        with _fetch_lock:
+            _fetch_state["running"] = False
+
+
+@callback(
+    Output("fetch-list-poll", "disabled"),
+    Output("fetch-list-status", "children"),
+    Input("fetch-list-button", "n_clicks"),
+    prevent_initial_call=True,
+)
+def start_fetch(n_clicks):
+    with _fetch_lock:
+        if _fetch_state["running"]:
+            return False, _fetch_state["status"]
+        _fetch_state["running"] = True
+        _fetch_state["status"] = "Fetching..."
+    threading.Thread(target=_run_fetch, daemon=True).start()
+    return False, "Fetching..."
+
+
+@callback(
+    Output("fetch-list-poll", "disabled", allow_duplicate=True),
+    Output("fetch-list-status", "children", allow_duplicate=True),
+    Input("fetch-list-poll", "n_intervals"),
+    prevent_initial_call=True,
+)
+def poll_fetch_status(n_intervals):
+    with _fetch_lock:
+        running = _fetch_state["running"]
+        status = _fetch_state["status"]
+    return running, status
+
+
 if __name__ == "__main__":
     app.run(debug=True)

src/components/sidebar.py

@@ -1,6 +1,6 @@
 import dash
 import dash_bootstrap_components as dbc
-from dash import dcc, html
+from dash import dcc, html, Input, Output, callback
 
 from src.utils.scan_lists import MEMB_LISTS
 
@@ -35,7 +35,22 @@
                                     dbc.Col(dbc.Label(className="fa fa-moon", html_for="color-mode-switch")),
                                 ],
                                 className="g-0",
-                            )
+                            ),
+                            dbc.Row(
+                                dbc.Col(
+                                    [
+                                        dbc.Button(
+                                            "Fetch New List",
+                                            id="fetch-list-button",
+                                            size="sm",
+                                            color="secondary",
+                                            className="mt-1 w-100",
+                                        ),
+                                        html.Small(id="fetch-list-status", className="text-muted"),
+                                        dcc.Interval(id="fetch-list-poll", interval=1000, disabled=True),
+                                    ]
+                                ),
+                            ),
                         ],
                         width="auto",
                         align="center",

src/utils/fetch_list.py

@@ -0,0 +1,98 @@
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.common.action_chains import ActionChains
+import dotenv
+import os
+import time
+from pathlib import Path, PurePath
+
+config = dotenv.dotenv_values(Path(PurePath(__file__).parents[2], ".env"))
+DOWNLOAD_DIR = config.get('LIST')
+PERISCOPE_URL = config.get('PERISCOPE_URL')
+PERISCOPE_PASS= config.get('PERISCOPE_PASS')
+
+def fetch_list(download_dir = DOWNLOAD_DIR, periscope_url = PERISCOPE_URL, periscope_pass = PERISCOPE_PASS):
+
+    if download_dir is None:
+        return None
+
+    #make sure the download dir exists
+    os.makedirs(os.path.abspath(download_dir), exist_ok=True)
+
+    options = Options()
+    options.add_argument("--window-size=1920,1080") #set standard window size
+    options.add_argument(argument="--headless=new") #headless
+    options.add_experimental_option("prefs", {
+        "download.default_directory": download_dir,
+        "download.prompt_for_download": False,
+    })
+
+    driver = webdriver.Chrome(options=options)
+
+    # set download directory
+    driver.execute_cdp_cmd("Browser.setDownloadBehavior", {
+        "behavior": "allow",
+        "downloadPath": os.path.abspath(download_dir),
+    })
+
+    wait = WebDriverWait(driver, 15)
+
+    # auth section, fills in periscope pw given url
+    driver.get(periscope_url)
+    password_input = driver.find_element(By.XPATH, """//*[@id="password"]""")
+    password_input.send_keys(periscope_pass)
+    button = driver.find_element(By.XPATH, """//*[@id="submit-button"]""")
+    ActionChains(driver).move_to_element(button).move_by_offset(10, 0).click().perform()
+    print("Authorized periscope.")
+
+    # scrolls to the widget, hovers to reveal controls, then click "More Options"
+    # because periscope can't just give you a link to a csv for some reason
+    title = wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, ".widget-18183666 div.title")))
+    driver.execute_script("arguments[0].scrollIntoView({block: 'center'});", title)
+    time.sleep(0.5)
+    ActionChains(driver).move_to_element(title).perform()
+    print("Found main member widget.")
+
+    # move to the little hamburger button
+    expand = wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, ".widget-18183666 div.expand.button")))
+    ActionChains(driver).move_to_element(expand).click().perform()
+    print("Found hamburger icon.")
+
+    # click "Download Data" from the menu
+    download_option = wait.until(EC.element_to_be_clickable((By.XPATH, "//*[contains(text(), 'Download Data')]")))
+    download_option.click()
+    print("Clicked Download Data")
+
+    # waits for periscope to finish materializing the csv:
+    # first wait for the loader to appear, then for it to disappear
+    loader_css = ".widget-18183666 .loader.materializing"
+    WebDriverWait(driver, 30).until(
+        EC.presence_of_element_located((By.CSS_SELECTOR, loader_css))
+    )
+    print("Materializing...")
+    WebDriverWait(driver, 120).until(
+        EC.invisibility_of_element_located((By.CSS_SELECTOR, loader_css))
+    )
+    print("Materialization complete.")
+
+    # wait for the .csv file to finish downloading
+    timeout = 60
+    start = time.time()
+    while time.time() - start < timeout:
+        files = os.listdir(download_dir)
+        csv_files = [f for f in files if f.endswith(".csv")]
+        tmp_files = [f for f in files if f.endswith(".crdownload") or f.startswith(".com.google.Chrome")]
+        if csv_files and not tmp_files:
+            print("Downloaded:", csv_files)
+            break
+        time.sleep(1)
+    else:
+        print("Timed out waiting for download")
+
+    driver.quit()
+
+if __name__ == "__main__":
+    fetch_list()

src/utils/scan_lists.py

@@ -1,4 +1,5 @@
 """Parse all membership lists into pandas dataframes for display on dashboard."""
+from re import L
 
 import logging
 from io import TextIOWrapper
@@ -145,7 +146,7 @@
     return df
 
 
-def scan_memb_list_from_csv(csv_file_data: TextIOWrapper | IO[bytes]) -> pd.DataFrame:
+def scan_memb_list_from_csv(csv_file_data: str | TextIOWrapper | IO[bytes]) -> pd.DataFrame:
     """Convert the provided csv data into a pandas dataframe."""
     return pd.read_csv(csv_file_data, dtype={"zip": str}, header=0)
 
@@ -155,23 +156,51 @@
     with ZipFile(zip_path) as memb_list_zip, memb_list_zip.open(f"{list_name}.csv", "r") as memb_list_csv:
         return scan_memb_list_from_csv(memb_list_csv)
 
+def date_from_stem(stem: str) -> str:
+    """Extract an ISO date string from a filename stem by trying each underscore-separated segment."""
+    for part in reversed(stem.split("_")):
+        try:
+            parsed = pd.to_datetime(part, format="mixed").date()
+            if parsed.year < 2000 or parsed.year > 2100:
+                continue
+            return parsed.isoformat()
+        except ValueError:
+            continue
+    raise ValueError(f"No parseable date found in filename stem: {stem}")
 
-def scan_all_membership_lists(list_name: str) -> dict[str, pd.DataFrame]:
+
+def scan_all_zip_membership_lists(list_name: str) -> dict[str, pd.DataFrame]:
     """Scan all zip files and call scan_memb_list_from_zip on each, returning the results."""
     memb_lists = {}
     logger.info("Scanning zipped membership lists in %s/.", list_name)
     files = sorted((Path(__file__).parents[2] / list_name).glob("**/*.zip"), reverse=True)
     for zip_file in files:
         filename = Path(zip_file).name
         try:
-            date_from_filename = str(PurePath(filename).stem).split("_")[-1]
-            list_date_iso = pd.to_datetime(date_from_filename, format="%Y%m%d").date().isoformat()
+            list_date_iso = date_from_stem(PurePath(filename).stem)
             memb_lists[list_date_iso] = scan_memb_list_from_zip(str(Path(zip_file).absolute()), list_name)
         except (IndexError, ValueError):
             logger.warning("Could not extract list from %s. Skipping file.", filename)
     logger.info("Found %s zipped membership lists.", len(memb_lists))
     return memb_lists
 
+def scan_all_csv_membership_lists(list_name: str) -> dict[str, pd.DataFrame]:
+    """Scan all csv files and call scan_memb_list_from_csv on each, return results."""
+    memb_lists = {}
+    logger.info("Scanning csv membership lists in %s/.", list_name)
+    files = sorted((Path(__file__).parents[2] / list_name).glob("**/*.csv"), reverse=True)
+    for csv in files:
+        filename = Path(csv).name
+        try:
+            list_date_iso = date_from_stem(PurePath(filename).stem)
+            memb_lists[list_date_iso] = scan_memb_list_from_csv(csv)
+        except (IndexError, ValueError):
+            logger.warning("Could not extract list from %s. Skipping file.", filename)
+    logger.info("Found %s csv membership lists.", len(memb_lists))
+    return memb_lists
+
+def scan_all_membership_lists(list_name: str) -> dict[str, pd.DataFrame]:
+    return scan_all_zip_membership_lists(list_name) | scan_all_csv_membership_lists(list_name)
 
 def branch_name_from_zip_code(zip_code: str, branch_zips: pd.DataFrame) -> str:
     """Check for provided zip_code in provided branch_zips and return relevant branch name if found."""