Merge pull request #130 from SNEWS2/integrate-snews-data-formats

Integrate snews-data-formats into snews-cs

Author: justinvasel

.github/workflows/tests.yml

@@ -6,7 +6,7 @@ name: tests
 # events but only for the main branch
 on:
   push:
-    branches: 
+    branches:
       - main
       - release_*
   pull_request:
@@ -22,14 +22,7 @@ jobs:
       matrix:
         # Test all supported Python versions under Ubuntu
         os: [ubuntu-latest]
-#        python-version: ['3.9', '3.10', '3.11']
         python-version: ['3.11', '3.12']
-#        # Additionally, test one Python version under MacOS and Windows, to detect OS-specific issues
-#        include:
-#          - os: macos-latest
-#            python-version: '3.12'
-#          - os: windows-latest
-#            python-version: '3.12'
 
     # The type of runner that the job will run on.
     runs-on: ${{ matrix.os }}
@@ -56,7 +49,7 @@ jobs:
       - name: Install hop-client
         run: |
           python -m pip install setuptools wheel
-          python -m pip install hop-client==0.9.0
+          python -m pip install hop-client==0.10.0
           cd /home/runner/work/SNEWS_Coincidence_System/SNEWS_Coincidence_System
 
       - shell: bash

poetry.lock

@@ -31,18 +31,19 @@ snews_cs = "snews_cs.__main__:main"
 [tool.poetry.dependencies]
 python = ">=3.11, <3.13"
 click = "^8.1.7"
-hop-client = "^0.9.0"
+hop-client = "^0.10.0"
 ipython = "<7.33.0"  # Constrained by snews_pt
 numpy = "^1.26.0"
 pandas = "^2.1.1"
-python-dotenv = "0.19.2"  # Constrained by snews_pt
+python-dotenv = "^0.19"  # Constrained by snews_pt
 pymongo = "^4.5.0"
 slack-sdk = "^3.26.2"
 matplotlib = "^3.8.0"
 scipy = "^1.11.3"
 sqlalchemy = "^2.0.32"
 tabulate = "^0.9.0"
 snews-pt = { git = "https://github.com/SNEWS2/SNEWS_Publishing_Tools.git", branch = "main" }
+snews-data-formats = "^1.0.0"
 
 [tool.poetry.group.dev.dependencies]
 autopep8 = "^2.0.4"

pyproject.toml

@@ -1,65 +1,72 @@
 adc-streaming==2.4.0 ; python_version >= "3.11" and python_version < "3.13"
+annotated-types==0.7.0 ; python_version >= "3.11" and python_version < "3.13"
 ansicon==1.89.0 ; python_version >= "3.11" and python_version < "3.13" and platform_system == "Windows"
 appnope==0.1.4 ; python_version >= "3.11" and python_version < "3.13" and sys_platform == "darwin"
-authlib==1.3.1 ; python_version >= "3.11" and python_version < "3.13"
+authlib==1.3.2 ; python_version >= "3.11" and python_version < "3.13"
 backcall==0.2.0 ; python_version >= "3.11" and python_version < "3.13"
 blessed==1.20.0 ; python_version >= "3.11" and python_version < "3.13"
-certifi==2024.7.4 ; python_version >= "3.11" and python_version < "3.13"
-cffi==1.17.0 ; python_version >= "3.11" and python_version < "3.13" and platform_python_implementation != "PyPy"
-charset-normalizer==3.3.2 ; python_version >= "3.11" and python_version < "3.13"
+certifi==2024.8.30 ; python_version >= "3.11" and python_version < "3.13"
+cffi==1.17.1 ; python_version >= "3.11" and python_version < "3.13" and platform_python_implementation != "PyPy"
+charset-normalizer==3.4.0 ; python_version >= "3.11" and python_version < "3.13"
 click==8.1.7 ; python_version >= "3.11" and python_version < "3.13"
 colorama==0.4.6 ; python_version >= "3.11" and python_version < "3.13" and (platform_system == "Windows" or sys_platform == "win32")
-confluent-kafka==2.5.0 ; python_version >= "3.11" and python_version < "3.13"
-contourpy==1.2.1 ; python_version >= "3.11" and python_version < "3.13"
-cryptography==43.0.0 ; python_version >= "3.11" and python_version < "3.13"
+confluent-kafka==2.6.1 ; python_version >= "3.11" and python_version < "3.13"
+contourpy==1.3.1 ; python_version >= "3.11" and python_version < "3.13"
+cryptography==43.0.3 ; python_version >= "3.11" and python_version < "3.13"
 cycler==0.12.1 ; python_version >= "3.11" and python_version < "3.13"
 decorator==5.1.1 ; python_version >= "3.11" and python_version < "3.13"
-dnspython==2.6.1 ; python_version >= "3.11" and python_version < "3.13"
-fastavro==1.9.5 ; python_version >= "3.11" and python_version < "3.13"
-fonttools==4.53.1 ; python_version >= "3.11" and python_version < "3.13"
-greenlet==3.0.3 ; python_version < "3.13" and (platform_machine == "aarch64" or platform_machine == "ppc64le" or platform_machine == "x86_64" or platform_machine == "amd64" or platform_machine == "AMD64" or platform_machine == "win32" or platform_machine == "WIN32") and python_version >= "3.11"
-hop-client==0.9.0 ; python_version >= "3.11" and python_version < "3.13"
-idna==3.7 ; python_version >= "3.11" and python_version < "3.13"
+dnspython==2.7.0 ; python_version >= "3.11" and python_version < "3.13"
+fastavro==1.9.7 ; python_version >= "3.11" and python_version < "3.13"
+fonttools==4.55.0 ; python_version >= "3.11" and python_version < "3.13"
+greenlet==3.1.1 ; python_version < "3.13" and (platform_machine == "win32" or platform_machine == "WIN32" or platform_machine == "AMD64" or platform_machine == "amd64" or platform_machine == "x86_64" or platform_machine == "ppc64le" or platform_machine == "aarch64") and python_version >= "3.11"
+hop-client==0.10.0 ; python_version >= "3.11" and python_version < "3.13"
+idna==3.10 ; python_version >= "3.11" and python_version < "3.13"
 inquirer==2.10.1 ; python_version >= "3.11" and python_version < "3.13"
 ipython==7.32.0 ; python_version >= "3.11" and python_version < "3.13"
-jedi==0.19.1 ; python_version >= "3.11" and python_version < "3.13"
+jedi==0.19.2 ; python_version >= "3.11" and python_version < "3.13"
 jinxed==1.3.0 ; python_version >= "3.11" and python_version < "3.13" and platform_system == "Windows"
-kiwisolver==1.4.5 ; python_version >= "3.11" and python_version < "3.13"
+kiwisolver==1.4.7 ; python_version >= "3.11" and python_version < "3.13"
 matplotlib-inline==0.1.7 ; python_version >= "3.11" and python_version < "3.13"
 matplotlib==3.9.2 ; python_version >= "3.11" and python_version < "3.13"
 numpy==1.26.4 ; python_version >= "3.11" and python_version < "3.13"
-packaging==24.1 ; python_version >= "3.11" and python_version < "3.13"
-pandas==2.2.2 ; python_version >= "3.11" and python_version < "3.13"
+packaging==24.2 ; python_version >= "3.11" and python_version < "3.13"
+pandas==2.2.3 ; python_version >= "3.11" and python_version < "3.13"
 parso==0.8.4 ; python_version >= "3.11" and python_version < "3.13"
 pexpect==4.9.0 ; python_version >= "3.11" and python_version < "3.13" and sys_platform != "win32"
 pickleshare==0.7.5 ; python_version >= "3.11" and python_version < "3.13"
-pillow==10.4.0 ; python_version >= "3.11" and python_version < "3.13"
+pillow==11.0.0 ; python_version >= "3.11" and python_version < "3.13"
 pluggy==1.5.0 ; python_version >= "3.11" and python_version < "3.13"
-prompt-toolkit==3.0.47 ; python_version >= "3.11" and python_version < "3.13"
+prompt-toolkit==3.0.48 ; python_version >= "3.11" and python_version < "3.13"
 ptyprocess==0.7.0 ; python_version >= "3.11" and python_version < "3.13" and sys_platform != "win32"
+pycountry==22.3.5 ; python_version >= "3.11" and python_version < "3.13"
 pycparser==2.22 ; python_version >= "3.11" and python_version < "3.13" and platform_python_implementation != "PyPy"
+pydantic-core==2.23.4 ; python_version >= "3.11" and python_version < "3.13"
+pydantic-extra-types==2.10.0 ; python_version >= "3.11" and python_version < "3.13"
+pydantic==2.9.2 ; python_version >= "3.11" and python_version < "3.13"
 pygments==2.18.0 ; python_version >= "3.11" and python_version < "3.13"
-pymongo==4.8.0 ; python_version >= "3.11" and python_version < "3.13"
-pyparsing==3.1.2 ; python_version >= "3.11" and python_version < "3.13"
+pymongo==4.10.1 ; python_version >= "3.11" and python_version < "3.13"
+pyparsing==3.2.0 ; python_version >= "3.11" and python_version < "3.13"
 python-dateutil==2.9.0.post0 ; python_version >= "3.11" and python_version < "3.13"
 python-dotenv==0.19.2 ; python_version >= "3.11" and python_version < "3.13"
 python-editor==1.0.4 ; python_version >= "3.11" and python_version < "3.13"
-pytz==2024.1 ; python_version >= "3.11" and python_version < "3.13"
-readchar==4.2.0 ; python_version >= "3.11" and python_version < "3.13"
+pytz==2024.2 ; python_version >= "3.11" and python_version < "3.13"
+readchar==4.2.1 ; python_version >= "3.11" and python_version < "3.13"
 requests==2.32.3 ; python_version >= "3.11" and python_version < "3.13"
-scipy==1.14.0 ; python_version >= "3.11" and python_version < "3.13"
+scipy==1.14.1 ; python_version >= "3.11" and python_version < "3.13"
 setuptools==62.1.0 ; python_version >= "3.11" and python_version < "3.13"
+single-version==1.6.0 ; python_version >= "3.11" and python_version < "3.13"
 six==1.16.0 ; python_version >= "3.11" and python_version < "3.13"
-slack-sdk==3.31.0 ; python_version >= "3.11" and python_version < "3.13"
-snews-pt @ git+https://github.com/SNEWS2/SNEWS_Publishing_Tools.git@e513fb7561ad1a037c06489a8e4ecb20ef5efb88 ; python_version >= "3.11" and python_version < "3.13"
-sqlalchemy==2.0.32 ; python_version >= "3.11" and python_version < "3.13"
+slack-sdk==3.33.4 ; python_version >= "3.11" and python_version < "3.13"
+snews-data-formats==1.0.0 ; python_version >= "3.11" and python_version < "3.13"
+snews-pt @ git+https://github.com/SNEWS2/SNEWS_Publishing_Tools.git@main ; python_version >= "3.11" and python_version < "3.13"
+sqlalchemy==2.0.36 ; python_version >= "3.11" and python_version < "3.13"
 tabulate==0.9.0 ; python_version >= "3.11" and python_version < "3.13"
 toml==0.10.2 ; python_version >= "3.11" and python_version < "3.13"
-tqdm==4.66.5 ; python_version >= "3.11" and python_version < "3.13"
+tqdm==4.67.0 ; python_version >= "3.11" and python_version < "3.13"
 traitlets==5.14.3 ; python_version >= "3.11" and python_version < "3.13"
 typing-extensions==4.12.2 ; python_version >= "3.11" and python_version < "3.13"
-tzdata==2024.1 ; python_version >= "3.11" and python_version < "3.13"
-urllib3==2.2.2 ; python_version >= "3.11" and python_version < "3.13"
+tzdata==2024.2 ; python_version >= "3.11" and python_version < "3.13"
+urllib3==2.2.3 ; python_version >= "3.11" and python_version < "3.13"
 wcwidth==0.2.13 ; python_version >= "3.11" and python_version < "3.13"
 wheel==0.34.2 ; python_version >= "3.11" and python_version < "3.13"
-xmltodict==0.13.0 ; python_version >= "3.11" and python_version < "3.13"
+xmltodict==0.14.2 ; python_version >= "3.11" and python_version < "3.13"

requirements.txt

@@ -1,10 +1,4 @@
+from ._version import version as __version__
 from .core.logging import initialize_logging
 
 initialize_logging("debug")
-import pandas as pd
-pd.options.mode.chained_assignment = None
-
-try:
-    from ._version import version as __version__
-except ImportError:
-    pass

snews_cs/__init__.py

@@ -7,13 +7,21 @@
 
 """
 
+import os
+from pathlib import Path
+from socket import gethostname
+
 # https://click.palletsprojects.com/en/8.0.x/utils/
-import click, os
-from . import __version__
-from . import cs_utils
+import click
+
+from . import __version__, cs_utils
 from . import snews_coinc as snews_coinc
-from . heartbeat_feedbacks import FeedBack
-from socket import gethostname
+from .database import Database
+from .heartbeat_feedbacks import FeedBack
+
+# Database initialized before any command is called
+db = Database(db_file_path=Path(__file__).parent.parent / "snews_cs.db")
+db.initialize_database(sql_schema_path=Path(__file__).parent / "db_schema.sql")
 
 
 @click.group(invoke_without_command=True)
@@ -38,17 +46,18 @@ def main(ctx, env):
 @click.option('--email/--no-email', default=True, show_default='True', help='Whether to send emails along with the alert')
 @click.option('--slackbot/--no-slackbot', default=True, show_default='True', help='Whether to send the alert on slack')
 def run_coincidence(firedrill, dropdb, email, slackbot):
-    """ Initiate Coincidence Decider 
+    """ Initiate Coincidence Decider
     """
+
     HOST = gethostname()
     coinc = snews_coinc.CoincidenceDistributor(drop_db=dropdb,
                                                firedrill_mode=firedrill,
                                                server_tag=HOST,
                                                send_email=email,
                                                send_slack=slackbot)
-    try: 
+    try:
         coinc.run_coincidence()
-    except KeyboardInterrupt: 
+    except KeyboardInterrupt:
         pass
     except Exception as e:
         print(e)

snews_cs/__main__.py

@@ -1,6 +1,8 @@
-from .cs_utils import set_env
 from datetime import datetime
 
+from .cs_utils import set_env
+
+
 class CoincidenceTierAlert:
     """ The Message scheme for the alert and observations
 
@@ -20,13 +22,13 @@ def id_format(self, num_detectors):
             time format should always be same for all detectors.
             num_detectors: `int`
                 Number of detectors in the alert. If more than 2,
-                it is an update to an earlier alert, and will be 
+                it is an update to an earlier alert, and will be
                 appended with -UPDATE_ string
 
         Returns
             :`str`
                 The formatted id as a string
-            
+
         """
         date_time = datetime.utcnow().isoformat()
         if num_detectors == 2:
@@ -37,32 +39,30 @@ def id_format(self, num_detectors):
     def get_cs_alert_schema(self, data, is_test=False):
         """ Create a message schema for alert.
             Internally called in hop_pub
-        
+
             Parameters
             ----------
             data : `named tuple`
                 cs_utils data tuple with predefined field.
-            
+
             Returns
             -------
                 :`dict`
-                    message with the correct scheme 
+                    message with the correct scheme
 
         """
         id = self.id_format(len(data['detector_names']))
-        alert_type = "TEST "+ data['alert_type'] if is_test else data['alert_type']
-        try:
-            far = f"Would happen every {data['false_alarm_prob']:.2e} year"
-        except:
-            far = data['false_alarm_prob']
-        return {"_id": id,
-                "alert_type":alert_type,
-                "server_tag": data['server_tag'],
-                "False Alarm Prob": far,
-                "detector_names": data['detector_names'],
-                "sent_time": id.split(' ')[1],
-                "p_values": data['p_vals'],
-                "neutrino_times": data['neutrino_times'],
-                "p_values average": data['p_val_avg'],
-                "sub list number": data['sub_list_num']
-                }
+        alert_type = "TEST " + data['alert_type'] if is_test else data['alert_type']
+
+        return {
+            "id": id,
+            "alert_type": alert_type,
+            "server_tag": data['server_tag'],
+            "False Alarm Prob": data['false_alarm_prob'],
+            "detector_names": data['detector_names'],
+            "sent_time": id.split(' ')[1],
+            "p_values": data['p_vals'],
+            "neutrino_times": data['neutrino_times'],
+            "p_values average": data['p_val_avg'],
+            "sub list number": data['sub_list_num'],
+        }