Merge branch 'master' into spikesorting-populator-v1

Author: edeno

CHANGELOG.md

@@ -4,8 +4,8 @@
 
 ### Release Notes
 
-Running draft to be removed immediately prior to release.
-When altering tables, import all foreign key references.
+Running draft to be removed immediately prior to release. When altering tables,
+import all foreign key references.
 
 ```python
 #
@@ -20,23 +20,27 @@ When altering tables, import all foreign key references.
 
 - Set default codecov threshold for test fail, disable patch check #1370, #1372
 - Simplify PR template #1370
+- Allow email send on space check success, clean up maintenance logging #1381
+- Update pynwb pin to >=2.5.0 for `TimeSeries.get_timestamps` #1385
 
-### Behavior
-
-- Add methods for calling moseq visualization functions #1374
-
-### Decoding
-
-- Ensure results directory is created if it doesn't exist #1362
-
-### Position
+### Infrastructure
 
-- Ensure video files are properly added to `DLCProject` #1367
-- DLC parameter handling improvements and default value corrections #1379
+- Auto-load within-Spyglass tables for graph operations #1368
 
-### Spikesorting
+### Pipelines
 
-- Implement short-transaction `SpikeSortingRecording.make` for v0 #1338
+- Behavior
+    - Add methods for calling moseq visualization functions #1374
+- Common
+    - Add tables for storing optogenetic experiment information #1312
+    - Remove wildcard matching in `Nwbfile().get_abs_path` #1382
+- Decoding
+    - Ensure results directory is created if it doesn't exist #1362
+- Position
+    - Ensure video files are properly added to `DLCProject` # 1367
+    - DLC parameter handling improvements and default value corrections #1379
+- Spikesorting
+    - Implement short-transaction `SpikeSortingRecording.make` for v0 #1338
 
 ## [0.5.5] (Aug 6, 2025)
 
@@ -533,3 +537,4 @@ When altering tables, import all foreign key references.
 [0.5.3]: https://github.com/LorenFrankLab/spyglass/releases/tag/0.5.3
 [0.5.4]: https://github.com/LorenFrankLab/spyglass/releases/tag/0.5.4
 [0.5.5]: https://github.com/LorenFrankLab/spyglass/releases/tag/0.5.5
+[0.5.6]: https://github.com/LorenFrankLab/spyglass/releases/tag/0.5.6

maintenance_scripts/README.md

@@ -68,7 +68,9 @@ regularly as cron jobs.
         - `SPACE_LOG`: the path to the log file.
         - `SPACE_EMAIL_SRC`/`SPACE_EMAIL_PASS`: email sender settings for disk
             space notifications.
-        - `SPACE_EMAIL_RECIPENTS`: a space-separated list of email addresses to
+        - `SPACE_EMAIL_ON_PASS`: if `true`, send an email summary even if all
+            drives are above the limit. If `false`, only send emails on failure.
+        - `SPACE_EMAIL_RECIPIENTS`: a space-separated list of email addresses to
             receive disk space notifications.
     - Items for posting to slack:
         - `SLACK_TOKEN`: the token for the slack app.

maintenance_scripts/check_disk_space.sh

@@ -16,7 +16,8 @@ if [[ -z "${SPACE_CHECK_DRIVES}" \
   || -z "${SPACE_EMAIL_RECIPIENTS}" \
   || -z "${SPACE_DRIVE_LIMITS}" \
   || -z "${SPACE_LOG}" \
-  || -z "${SPACE_ROOT_NAME}" ]] ; then
+  || -z "${SPACE_ROOT_NAME}" \
+  || -z "${SPACE_EMAIL_ON_PASS}" ]] ; then
   echo "Error: Missing one or more variables required for check_disk_space.sh"
   exit 1
 fi
@@ -36,7 +37,7 @@ echo "SPACE CHECK: $(date)" >> "$SPACE_LOG"
 EMAIL_TEMPLATE=$(cat <<-EOF
 From: "Spyglass" <$SPACE_EMAIL_SRC>
 To: %s
-Subject: Drive almost full: %s
+Subject: %s
 
 %s
 EOF
@@ -48,7 +49,7 @@ send_email_message() {
   local SUBJECT="$2"
   local BODY="$3"
   EMAIL=$(printf "$EMAIL_TEMPLATE" "$RECIPIENT" "$SUBJECT" "$BODY")
-  curl -s --url "smtps://smtp.gmail.com:465" \
+  curl -sS --url "smtps://smtp.gmail.com:465" \
       --ssl-reqd \
       --user "$SPACE_EMAIL_SRC:$SPACE_EMAIL_PASS" \
       --mail-from "$SPACE_EMAIL_SRC" \
@@ -76,6 +77,9 @@ for DRIVE in $SPACE_CHECK_DRIVES; do
     [[ $LEN -gt $MAX_DRIVE_LEN ]] && MAX_DRIVE_LEN=$LEN
 done
 
+OUTPUT=""
+FOUND_ISSUE=0
+
 # Check each drive
 for i in "${!DRIVE_LIST[@]}"; do
     DRIVE="${DRIVE_LIST[$i]}"
@@ -102,14 +106,19 @@ for i in "${!DRIVE_LIST[@]}"; do
     else
         NAME="${DRIVE:1}" # Assumes first char is `/`
     fi
-    printf "%-*s: %s/%s\n" "$MAX_DRIVE_LEN" "$NAME" \
-        "$FREE_HUMAN" "$TOTAL_HUMAN" >> "$SPACE_LOG"
+    line=$(\
+      printf "%-*s: %s/%s\n" \
+      "$MAX_DRIVE_LEN" "$NAME" "$FREE_HUMAN" "$TOTAL_HUMAN"\
+    )
+    OUTPUT+="$line\n"
 
     # Do nothing if under capacity
     if [[ "$FREE_BYTES" -gt "$SPACE_LIMIT_BYTES" ]]; then
         continue
     fi
 
+    FOUND_ISSUE=1
+
     # Send email alert
     BODY="Low space warning: ${NAME} has ${FREE_HUMAN}/${TOTAL_HUMAN} free"
     SUBJ="${NAME}"
@@ -119,7 +128,15 @@ for i in "${!DRIVE_LIST[@]}"; do
     send_slack_message "$BODY"
 
     for RECIPIENT in $SPACE_EMAIL_RECIPIENTS; do
-        send_email_message "$RECIPIENT" "$SUBJ" "$BODY"
+        send_email_message "$RECIPIENT" "ALMOST FULL: $SUBJ" "$BODY"
     done
 
 done
+
+echo -e "$OUTPUT" >> "$SPACE_LOG"
+
+if [[ "$SPACE_EMAIL_ON_PASS" == "true" ]] && [[ "$FOUND_ISSUE" == "0" ]]; then
+  for RECIPIENT in $SPACE_EMAIL_RECIPIENTS; do
+      send_email_message "$RECIPIENT" "Disk Space OK" "$OUTPUT"
+  done
+fi

maintenance_scripts/example.env

@@ -19,6 +19,7 @@ export SPACE_DRIVE_LIMITS="10T 10G 3G" # space separated, same order as above
 export SPACE_LOG=$SPYGLASS_LOG
 export SPACE_EMAIL_SRC=$SPYGLASS_EMAIL_SRC
 export SPACE_EMAIL_PASS=$SPYGLASS_EMAIL_PASS
+export SPACE_EMAIL_ON_PASS=true # email only on issues
 export SPACE_EMAIL_RECIPIENTS="[email protected] [email protected]" # space separated
 export SLACK_TOKEN="xoxb-not-a-real-token-this-will-not-work" # optional, slack token
 export SLACK_CHANNEL="B056AQM5VFZ" # optional, slack channel

maintenance_scripts/run_jobs.sh

@@ -45,7 +45,7 @@ on_fail() { # $1: error message. Echo message and send as email
     local content
     content=$(printf "$EMAIL_TEMPLATE" "$error_msg")
 
-    curl -o /dev/null --ssl-reqd \
+    curl -sS -o /dev/null --ssl-reqd \
       --url "smtps://smtp.gmail.com:465" \
       --user "${SPYGLASS_EMAIL_SRC}:${SPYGLASS_EMAIL_PASS}" \
       --mail-from "$SPYGLASS_EMAIL_SRC" \
@@ -82,7 +82,7 @@ CONN_TEST="import datajoint as dj; dj.logger.setLevel('ERROR'); dj.conn()"
 conda_run python -c "$CONN_TEST" > /dev/null || \
   { on_fail "Could not connect to the database"; exit 1; }
 
-# Chmod new files in past 2 days
+# Chmod new files in past 2 days, requires sudo
 if $SPYGLASS_CHMOD_FILES; then
   find $SPYGLASS_BASE_PATH -type f -mtime -2 -exec chmod 644 {} \; || \
     { on_fail "Could not chmod new files in $SPYGLASS_BASE_PATH"; exit 1; }

pyproject.toml

@@ -55,7 +55,7 @@ dependencies = [
     "probeinterface<0.3.0", # Bc some probes fail space checks
     "pubnub<6.4.0",  # TODO: remove this when sortingview is updated
     "pydotplus",
-    "pynwb>=2.2.0,<3",
+    "pynwb>=2.5.0,<3",
     "ripple_detection",
     "seaborn",
     "sortingview>=0.11",

src/spyglass/common/__init__.py

@@ -42,6 +42,13 @@
 )
 from spyglass.common.common_lab import Institution, Lab, LabMember, LabTeam
 from spyglass.common.common_nwbfile import AnalysisNwbfile, Nwbfile
+from spyglass.common.common_optogenetics import (
+    OpticalFiberDevice,
+    OpticalFiberImplant,
+    OptogeneticProtocol,
+    Virus,
+    VirusInjection,
+)
 from spyglass.common.common_position import (
     IntervalLinearizationSelection,
     IntervalLinearizedPosition,
@@ -57,6 +64,7 @@
 from spyglass.common.common_session import Session
 from spyglass.common.common_subject import Subject
 from spyglass.common.common_task import Task, TaskEpoch
+from spyglass.common.common_user import UserEnvironment
 from spyglass.common.populate_all_common import populate_all_common
 from spyglass.common.prepopulate import populate_from_yaml, prepopulate_default
 from spyglass.settings import prepopulate
@@ -69,12 +77,16 @@
     get_raw_eseries,
     get_valid_intervals,
 )
-from spyglass.common.common_user import UserEnvironment
 
 __all__ = [
     "LabTeam",
     "LinearizationParameters",
     "Nwbfile",
+    "OpticalFiberDevice",
+    "OpticalFiberImplant",
+    "OptogeneticProtocol",
+    "Virus",
+    "VirusInjection",
     "PositionInfoParameters",
     "PositionIntervalMap",
     "PositionSource",

src/spyglass/common/common_behav.py

@@ -78,7 +78,7 @@ def insert_from_nwbfile(cls, nwb_file_name, skip_duplicates=False) -> None:
         """
         nwbf = get_nwb_file(nwb_file_name)
         all_pos = get_all_spatial_series(nwbf, verbose=True)
-        sess_key = Nwbfile.get_file_key(nwb_file_name)
+        sess_key = {"nwb_file_name": nwb_file_name}
         src_key = dict(**sess_key, source="imported", import_file_name="")
 
         if all_pos is None:

src/spyglass/common/common_nwbfile.py

@@ -84,24 +84,6 @@ def fetch_nwb(self):
             for file in self.fetch("nwb_file_name")
         ]
 
-    @classmethod
-    def _get_file_name(cls, nwb_file_name: str) -> str:
-        """Get valid nwb file name given substring."""
-        query = cls & f'nwb_file_name LIKE "%{nwb_file_name}%"'
-
-        if len(query) == 1:
-            return query.fetch1("nwb_file_name")
-
-        raise ValueError(
-            f"Found {len(query)} matches for {nwb_file_name} in Nwbfile table:"
-            + f" \n{query}"
-        )
-
-    @classmethod
-    def get_file_key(cls, nwb_file_name: str) -> dict:
-        """Return primary key using nwb_file_name substring."""
-        return {"nwb_file_name": cls._get_file_name(nwb_file_name)}
-
     @classmethod
     def get_abs_path(
         cls, nwb_file_name: str, new_file: bool = False, **kwargs
@@ -124,10 +106,17 @@ def get_abs_path(
         nwb_file_abspath : str
             The absolute path for the given file name.
         """
+        file_path = raw_dir + "/" + nwb_file_name
         if new_file:
-            return raw_dir + "/" + nwb_file_name
+            return file_path
+
+        query = cls & {"nwb_file_name": nwb_file_name}
+        if len(query) != 1:
+            raise ValueError(
+                f"Could not find 1 entry for {nwb_file_name}:\n{query}"
+            )
 
-        return raw_dir + "/" + cls._get_file_name(nwb_file_name)
+        return file_path
 
     @staticmethod
     def add_to_lock(nwb_file_name: str) -> None: