lint(): run various linting tools to improve code style (#39)

Author: AmitMY

pose_evaluation/analysis/analyze_scores.py

@@ -1,24 +1,22 @@
 import argparse
-from typing import Optional, List, Dict, Tuple
-from collections import defaultdict
-from pathlib import Path
 import json
-import re
-
-import pandas as pd
 import numpy as np
-from tqdm import tqdm
+import pandas as pd
+import re
 import torch
+from collections import defaultdict
+from pathlib import Path
 from torchmetrics.retrieval import RetrievalMAP, RetrievalMRR, RetrievalPrecision, RetrievalRecall
+from tqdm import tqdm
+from typing import Optional, List, Dict, Tuple
 
-from pose_evaluation.evaluation.score_dataframe_format import ScoreDFCol, load_score_csv
 from pose_evaluation.evaluation.index_score_files import ScoresIndexDFCol, index_scores
 from pose_evaluation.evaluation.load_pyarrow_dataset import load_dataset, load_metric_dfs
+from pose_evaluation.evaluation.score_dataframe_format import ScoreDFCol, load_score_csv
 
 _SIGNATURE_RE = re.compile(r"default_distance:([\d.]+)")
 _DEFAULTDIST_RE = re.compile(r"defaultdist([\d.]+)")
 
-
 tqdm.pandas()
 
 
@@ -514,6 +512,5 @@ def load_metric_dfs_from_filenames(scores_folder: Path, file_format: str = "csv"
     else:
         print("No metrics were analyzed.")
 
-
 # conda activate /opt/home/cleong/envs/pose_eval_src && cd /opt/home/cleong/projects/pose-evaluation && python pose_evaluation/evaluation/analyze_scores.py metric_results_1_2_z_combined_818_metrics/scores --file-format parquet
 # conda activate /opt/home/cleong/envs/pose_eval_src && cd /opt/home/cleong/projects/pose-evaluation && python pose_evaluation/evaluation/analyze_scores.py metric_results_round_4/scores/ --file-format parquet

pose_evaluation/analysis/correlate_and_graph_intergloss_scores.py

@@ -1,19 +1,18 @@
-from itertools import combinations
-import math
-from pathlib import Path
-from typing import Optional
+import ast
+import gc
 import hashlib
-from datetime import datetime
+import math
+import matplotlib.pyplot as plt
 import pandas as pd
 import plotly.express as px
 import plotly.graph_objects as go
-from tqdm import tqdm
 import seaborn as sns
-import gc
-import ast
+from datetime import datetime
+from itertools import combinations
+from pathlib import Path
 from scipy.stats import pearsonr, spearmanr, kendalltau, ttest_rel
-import matplotlib.pyplot as plt
-
+from tqdm import tqdm
+from typing import Optional
 
 sns.set_theme()
 

pose_evaluation/analysis/count_files_by_hour.py

@@ -1,9 +1,9 @@
 #!/usr/bin/env python3
 
 import argparse
-from pathlib import Path
 from collections import Counter
 from datetime import datetime, timedelta
+from pathlib import Path
 from tqdm import tqdm
 
 

pose_evaluation/analysis/explore_metric_stats.py

@@ -1,14 +1,14 @@
-from pathlib import Path
-from itertools import combinations
-import re
+import io
 import os
-import torch
-import streamlit as st
 import pandas as pd
 import plotly.express as px
 import plotly.graph_objects as go
 import plotly.io as pio
-import io
+import re
+import streamlit as st
+import torch
+from itertools import combinations
+from pathlib import Path
 from typing import List, Tuple
 
 # pio.templates.default = "plotly"
@@ -260,7 +260,6 @@ def prettify_axis_label(label: str) -> str:
 
 
 def apply_minimal_layout(fig: go.Figure, size: int = 600) -> None:
-
     # fig.update_layout(
     #     title=None,
     #     xaxis=dict(visible=False),
@@ -753,7 +752,7 @@ def match_keywords(text):
                 # f"\n% {interpret_name(metric)}",
                 f"\n% {descriptive_name(metric)}",
                 f"\n% {metric} &\t{metric_row['mean_average_precision']:.2f} &\t{metric_row['precision@10']:.2f}\t\\\\",
-                f"\n{descriptive_name(metric).split()[0]} & {dd} & {fm} & {trim} & {norm} & {seq_align} & {kp} &\t{metric_row['mean_average_precision']*100:.0f}\\% &\t{metric_row['precision@10']*100:.0f}\\%\t\\\\",
+                f"\n{descriptive_name(metric).split()[0]} & {dd} & {fm} & {trim} & {norm} & {seq_align} & {kp} &\t{metric_row['mean_average_precision'] * 100:.0f}\\% &\t{metric_row['precision@10'] * 100:.0f}\\%\t\\\\",
             ]
 
             for mdl in markdown_lines:
@@ -876,11 +875,10 @@ def match_keywords(text):
             st.write(f"**Average on '{sort_col}' without '{kw}':** `{avg_without:.4f}`")
             st.write(f"**Estimated effect on '{sort_col}' of '{kw}':** `{delta:+.4f}`")
             st.write(f"{kw} count within {top_or_bottom} 100 by {sort_col}: {(has_kw['RANK'] <= 100).sum()}")
-            st.write(f"{kw} count within {top_or_bottom} 10 by {sort_col}: {(has_kw['RANK']<= 10).sum()}")
+            st.write(f"{kw} count within {top_or_bottom} 10 by {sort_col}: {(has_kw['RANK'] <= 10).sum()}")
             st.write(f"{kw} count within {top_or_bottom} 5 by {sort_col}: {(has_kw['RANK'] <= 5).sum()}")
 
             if st.checkbox(f"Show distributions for {kw}?"):
-
                 fig = go.Figure()
 
                 fig.add_trace(
@@ -1026,5 +1024,4 @@ def match_keywords(text):
     else:
         plot_pareto_frontier(df)
 
-
 # conda activate /opt/home/cleong/envs/pose_eval_src && streamlit run pose_evaluation/evaluation/explore_metric_stats.py

pose_evaluation/analysis/plot_dtw.py

@@ -1,10 +1,9 @@
-from itertools import product
-from pathlib import Path
-
 import numpy as np
 import plotly.graph_objects as go
 import plotly.io as pio
 from fastdtw import fastdtw  # type: ignore (pylint can't find it but it's there)
+from itertools import product
+from pathlib import Path
 
 
 def pad_seq(seq, val, count):

pose_evaluation/evaluation/add_semantic_and_lookalike_to_gloss_dfs.py

@@ -1,10 +1,10 @@
 """WIP script to try and add relations to dataset_dfs, depends on fixing gloss/vocabulary matching problems"""
 
-from pathlib import Path
 import pandas as pd
+from pathlib import Path
 
-from pose_evaluation.evaluation.load_splits_and_run_metrics import combine_dataset_dfs
 from pose_evaluation.evaluation.combine_semantic_and_lookalike import create_gloss_tuple
+from pose_evaluation.evaluation.load_splits_and_run_metrics import combine_dataset_dfs
 
 # /opt/home/cleong/projects/pose-evaluation/pose_evaluation/evaluation/combine_semantic_and_lookalike.py
 

pose_evaluation/evaluation/analyze_scores_dask.py

@@ -1,29 +1,28 @@
-from pathlib import Path
 import argparse
-
-
+import argparse
+import dask.array as da
+import dask.bag as db
+import dask.dataframe as dd
+import dask.dataframe as dd
 import numpy as np
 import pandas as pd
-
 import pyarrow.dataset as ds
+from dask.distributed import LocalCluster
+from pathlib import Path
 
-import dask.dataframe as dd
-import dask.array as da
-import dask.bag as db
-
+#         client.close()
+#         cluster.close()
+from pathlib import Path
 
 # # https://docs.dask.org/en/latest/dataframe-hive.html#reading-parquet-data-with-hive-partitioning
 # # https://docs.dask.org/en/stable/deploying.html?utm_source=tds&utm_medium=pyarrow-in-pandas-and-dask
-
 # if __name__ == "__main__":
 #     from dask.distributed import LocalCluster
-
 #     cluster = LocalCluster()  # Fully-featured local Dask cluster
 #     client = cluster.get_client()
 #     parser = argparse.ArgumentParser(description="Analyze")
 #     parser.add_argument("dataset_dir", type=Path, help="Pyarrow Dataset of scores")
 #     args = parser.parse_args()
-
 #     # Assuming your data is already in a Dask DataFrame 'ddf'
 #     # as indicated by dd.read_parquet
 #     try:
@@ -34,17 +33,12 @@
 #     except Exception as e:
 #         print(f"Error reading parquet files: {e}")
 #         exit(1)
-
 #     # Filter the Dask DataFrame where "GLOSS_A" is equal to "GLOSS_B"
 #     # filtered_ddf = ddf[ddf["GLOSS_A"] == ddf["GLOSS_B"]]
-
 #     # # Group by "METRIC" and calculate the mean of "SCORE" on the filtered data
 #     # mean_scores = filtered_ddf.groupby("METRIC", observed=True)["SCORE"].mean()
-
 #     mean_scores = ddf.groupby("METRIC", observed=True)["SCORE"].mean()
-
 #     print(cluster.dashboard_link)
-
 #     # Compute the result to get a Pandas Series
 #     try:
 #         result = mean_scores.compute()
@@ -55,13 +49,6 @@
 #         print(f"Error during computation: {e}")
 #     finally:
 
-#         client.close()
-#         cluster.close()
-from pathlib import Path
-import argparse
-import dask.dataframe as dd
-from dask.distributed import LocalCluster
-
 if __name__ == "__main__":
     cluster = LocalCluster()
     client = cluster.get_client()

pose_evaluation/evaluation/analyze_scores_polars.py

@@ -1,16 +1,13 @@
-import polars as pl
-from pathlib import Path
 import argparse
+import os
+import polars as pl
 import polars as pl
-from pathlib import Path
-
 import polars as pl
-
-
-import time
 import psutil
-import os
 import statistics
+import time
+from pathlib import Path
+from pathlib import Path
 
 
 def run_experiments(path: Path, limit_rows: int = 100_000_000, runs: int = 10):
@@ -129,7 +126,6 @@ def show_head(lf: pl.LazyFrame, n=5):
 
 
 def compute_score_by_metric(lf: pl.LazyFrame):
-
     print(lf.schema)
 
     lazy_result = lf.group_by("METRIC").agg(pl.col("SCORE").mean().alias("mean_SCORE"))
@@ -270,7 +266,7 @@ def compute_map_by_metric_safe_chunked(
         for i in range(0, len(gloss_a_paths), batch_size):
             batch_paths = gloss_a_paths[i : i + batch_size]
             print(
-                f"  📁 Processing batch {i//batch_size + 1}/{(len(gloss_a_paths) + batch_size - 1)//batch_size} ({len(batch_paths)} paths)"
+                f"  📁 Processing batch {i // batch_size + 1}/{(len(gloss_a_paths) + batch_size - 1) // batch_size} ({len(batch_paths)} paths)"
             )
 
             batch_lf = metric_lf.filter(pl.col("GLOSS_A_PATH").is_in(batch_paths))
@@ -557,7 +553,7 @@ def compute_map_single_metric_chunked(lf: pl.LazyFrame, metric_name: str, chunk_
     for i in range(0, len(unique_paths), chunk_size):
         chunk_paths = unique_paths[i : i + chunk_size]
         print(
-            f"  📦 Processing chunk {i//chunk_size + 1}/{(len(unique_paths) + chunk_size - 1)//chunk_size} "
+            f"  📦 Processing chunk {i // chunk_size + 1}/{(len(unique_paths) + chunk_size - 1) // chunk_size} "
             f"({len(chunk_paths)} paths)"
         )
 
@@ -645,7 +641,7 @@ def compute_map_ultra_minimal_memory(dataset_path: str, metrics_to_process: list
         # Process each query (GLOSS_A_PATH) individually
         for i, path in enumerate(unique_paths):
             if i % 1000 == 0:
-                print(f"  📍 Processing query {i+1}/{len(unique_paths)}")
+                print(f"  📍 Processing query {i + 1}/{len(unique_paths)}")
 
             # Get data for this specific query
             query_lf = (

pose_evaluation/evaluation/combine_semantic_and_lookalike.py

@@ -1,6 +1,6 @@
-from pathlib import Path
 import pandas as pd
 from io import StringIO
+from pathlib import Path
 
 """Trying to combine Sem-Lex, ASL Citizen, ASL Knowledge Graph and "Known Lookalikes" from 
 """

pose_evaluation/evaluation/compare_two_poses.py

@@ -1,8 +1,9 @@
-from typing import List
-import typer
 import pandas as pd
-from pose_format import Pose
+import typer
 from pathlib import Path
+from pose_format import Pose
+from typing import List
+
 from pose_evaluation.evaluation.create_metrics import get_metrics
 from pose_evaluation.evaluation.load_splits_and_run_metrics import get_filtered_metrics
 
@@ -63,7 +64,6 @@ def compare(
         None, help="Will filter metrics to only those that include none of these"
     ),
 ):
-
     # paths_csvs = Path("/opt/home/cleong/projects/pose-evaluation/debug_zspeed/zspeed_results_from_preliminary/").glob(
     #     "*.csv"
     # )