Merge branch 'gen-178-update-pandas-2' into gen-998-resolve-fill-na

Author: rxu17

Dockerfile

@@ -1,5 +1,5 @@
 # TODO: update this
-FROM ubuntu:focal-20220113
+FROM ubuntu:jammy-20250714
 ENV DEBIAN_FRONTEND=noninteractive
 
 # Must install this because gpg not installed
@@ -24,8 +24,8 @@ RUN apt-get update && apt-get install -y --allow-unauthenticated --no-install-re
 		python3-pip \
 		python3-dev \
 		git \
-		r-base-core=4.3.3-1.2004.0 \
-		r-base-dev=4.3.3-1.2004.0 \
+		r-base-core=4.3.3-1.2204.0 \
+		r-base-dev=4.3.3-1.2204.0 \
 		cmake \
 		curl \
 		# synapser client dependencies
@@ -45,7 +45,7 @@ RUN apt-get update && apt-get install -y --allow-unauthenticated --no-install-re
 		openjdk-11-jre \
 		# This is for reticulate
 		# TODO: update this
-		python3.11-venv && \
+		python3.10-venv && \
 	apt-get clean && \
 	rm -rf /var/lib/apt/lists/*
 

R/install_packages.R

@@ -8,3 +8,5 @@ library(testthat)
 library(VariantAnnotation)
 library(xtable)
 library(ggpubr)
+library(reticulate)
+reticulate::py_install(c("pandas>=1.5,<=2.0.3", "numpy<=1.24.4"))

genie/load.py

@@ -226,16 +226,15 @@ def _reorder_new_dataset(
 def _generate_primary_key(
     dataset: pd.DataFrame, primary_key_cols: List[str], primary_key: str
 ) -> pd.DataFrame:
-    """
-    Generate primary key column a dataframe
+    """Generate primary key column a dataframe
 
     Args:
-        dataset(pd.DataFrame): A dataframe
-        new_dataset: The re-ordered new dataset
-        primary_key_cols (list): Column(s) that make up the primary key
-        primary_key: The column name of the primary_key
+        dataset (pd.DataFrame): A dataframe
+        primary_key_cols (List[str]): Column(s) that make up the primary key
+        primary_key (str): The column name of the primary_key
+
     Returns:
-        The dataframe with primary_key column added
+        pd.DataFrame: The dataframe with primary_key column added
     """
     # replace NAs with emtpy string
     dataset = dataset.fillna("")
@@ -298,21 +297,18 @@ def store_database(
     syn: synapseclient.Synapse,
     database_synid: str,
     col_order: List[str],
-    allupdates: pd.DataFrame,
+    all_updates: pd.DataFrame,
     to_delete_rows: pd.DataFrame,
-):
+) -> None:
     """
     Store changes to the database
 
     Args:
-        syn (synapseclient.Synaps): Synapse object
+        syn (synapseclient.Synapse): Synapse object
         database_synid (str): Synapse Id of the Synapse table
         col_order (List[str]): The ordered column names to be saved
-        allupdates (pd.DataFrame): rows to be appended and/or updated
+        all_updates (pd.DataFrame): rows to be appended and/or updated
         to_deleted_rows (pd.DataFrame): rows to be deleted
-
-    Returns:
-        None
     """
     storedatabase = False
     update_all_file = tempfile.NamedTemporaryFile(
@@ -321,14 +317,14 @@ def store_database(
     with open(update_all_file.name, "w") as updatefile:
         # Must write out the headers in case there are no appends or updates
         updatefile.write(",".join(col_order) + "\n")
-        if not allupdates.empty:
+        if not all_updates.empty:
             """
             This is done because of pandas typing.
             An integer column with one NA/blank value
             will be cast as a double.
             """
             updatefile.write(
-                allupdates[col_order]
+                all_updates[col_order]
                 .to_csv(index=False, header=None)
                 .replace(".0,", ",")
                 .replace(".0\n", "\n")

genie_registry/clinical.py

@@ -583,8 +583,9 @@ def process_steps(
         newClinicalDf.to_csv(newPath, sep="\t", index=False)
         return newPath
 
+    @staticmethod
     def _validate_oncotree_code_mapping(
-        self: "Clinical", clinicaldf: pd.DataFrame, oncotree_mapping: pd.DataFrame
+        clinicaldf: pd.DataFrame, oncotree_mapping: pd.DataFrame
     ) -> pd.Index:
         """Checks that the oncotree codes in the input clinical
         data is a valid oncotree code from the official oncotree site
@@ -609,8 +610,8 @@ def _validate_oncotree_code_mapping(
         ]
         return unmapped_oncotrees.index
 
+    @staticmethod
     def _validate_oncotree_code_mapping_message(
-        self: "Clinical",
         clinicaldf: pd.DataFrame,
         unmapped_oncotree_indices: pd.DataFrame,
     ) -> Tuple[str, str]:

requirements.txt

@@ -6,3 +6,4 @@ pyranges==0.1.4
 # known working version 6.0
 PyYAML>=5.1
 synapseclient[pandas]>=4.0.0, <5.0.0
+opentelemetry-semantic-conventions==0.56b0

setup.cfg

@@ -21,7 +21,6 @@ classifiers =
     Programming Language :: Python :: 3 :: Only
     Programming Language :: Python :: 3.10
     Programming Language :: Python :: 3.11
-    Programming Language :: Python :: 3.12
     Topic :: Scientific/Engineering
 project_urls =
     Bug Tracker = https://github.com/Sage-Bionetworks/Genie/issues

tests/test_input_to_database.py

@@ -670,7 +670,6 @@ def asDataFrame(self):
 
 class TestValidation:
     def setup_method(self):
-        # TODO: Look into 1553428800000, locally it needs to be 1553454000000 to pass
         valid = [
             [
                 sample_clinical_entity.id,