Merge pull request #240 from macrocosm-os/staging

Staging

Author: mccrindlebrian

README.md

@@ -76,28 +76,36 @@ When the simulations finally converge (ΔE/t < threshold), they produce the form
 ## Requirements 
 Protein folding utilizes an open-source package called [OpenMM](https://openmm.org). To run, you will need:
 1. A Linux-based machine 
-2. At least 1 CUDA-compatible GPU
+2. At least 1 CUDA-compatible GPU. We recommend an RXT 4090. 
 3. Conda Distribution (we recommend [Miniconda](https://docs.anaconda.com/miniconda/)). Using conda is an [OpenMM requirement](http://docs.openmm.org/latest/userguide/application/01_getting_started.html#installing-openmm). 
 
 For more information regarding recommended hardware specifications, look at [min_compute.yml](./min_compute.yml)
 
-## Weights and Biases
+## Installation 
 As a validator, you are **required** to have Weights and Biases (Wandb) active on your machine. We open-source our logging to the community, so this is a necessary component. The repo will not work without Wandb. 
 
-Simply:
+As a miner, this is an optional include. As such, we do not have logic for logging natively in the base miner, but can be easily added. 
+
+This repository requires python3.8 or higher. To install it, simply clone this repository and run the [install.sh](./install.sh) script. Below are all the steps needed to ensure that your machine is running properly:
+
+Firstly, you must install conda: 
+```bash 
+mkdir -p ~/miniconda3
+wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda3/miniconda.sh
+bash ~/miniconda3/miniconda.sh -b -u -p ~/miniconda3
+rm ~/miniconda3/miniconda.sh
+```
+
 ```bash
 pip install wandb 
 wandb login
-```
-
-As a miner, this is an optional include. As such, we do not have logic for logging natively in the base miner, but can be easily added. 
 
-## Installation
-This repository requires python3.8 or higher. To install it, simply clone this repository and run the [install.sh](./install.sh) script.
-```bash
 git clone https://github.com/macrocosm-os/folding.git
 cd folding
 bash install.sh
+
+conda activate folding
+pip install -e .
 ```
 
 This will also create a virtual environment in which the repo can be run inside of.

folding/__init__.py

@@ -1,7 +1,7 @@
 from .protocol import JobSubmissionSynapse
 from .validators.protein import Protein
 
-__version__ = "1.0.5"
+__version__ = "1.0.6"
 version_split = __version__.split(".")
 __spec_version__ = (
     (10000 * int(version_split[0]))

folding/utils/data.py

@@ -2,14 +2,15 @@
 import os
 import sys
 import tqdm
+import signal
 import random
 import shutil
 import requests
+import functools
 import traceback
 import subprocess
 import pickle as pkl
 from typing import Dict, List
-
 import numpy as np
 import pandas as pd
 import parmed as pmd
@@ -19,6 +20,10 @@
 from folding.protocol import JobSubmissionSynapse
 
 
+class TimeoutException(Exception):
+    pass
+
+
 class OpenMMException(Exception):
     """Exception raised for errors in the versioning."""
 
@@ -35,6 +40,33 @@ def __init__(self, message="Version error occurred"):
         super().__init__(self.message)
 
 
+def timeout_handler(seconds, func_name):
+    raise TimeoutException(f"Function '{func_name}' timed out after {seconds} seconds")
+
+
+# Decorator to apply the timeout
+def timeout(seconds):
+    def decorator(func):
+        @functools.wraps(func)  # Retain original function metadata
+        def wrapper(*args, **kwargs):
+            # Set the signal alarm with the function name
+            signal.signal(
+                signal.SIGALRM,
+                lambda signum, frame: timeout_handler(seconds, func.__name__),
+            )
+            signal.alarm(seconds)
+            try:
+                result = func(*args, **kwargs)
+            finally:
+                # Disable the alarm
+                signal.alarm(0)
+            return result
+
+        return wrapper
+
+    return decorator
+
+
 def delete_directory(directory: str):
     """We create a lot of files in the process of tracking pdb files.
     Therefore, we want to delete the directory after we are done with the tests.
@@ -134,31 +166,6 @@ def get_tracebacks():
     bt.logging.warning(" ---------------- End of Traceback ----------------\n")
 
 
-def run_cmd_commands(
-    commands: List[str], suppress_cmd_output: bool = True, verbose: bool = False
-):
-    for cmd in tqdm.tqdm(commands):
-        bt.logging.debug(f"Running command: {cmd}")
-
-        try:
-            result = subprocess.run(
-                cmd,
-                check=True,
-                shell=True,
-                stdout=subprocess.PIPE,
-                stderr=subprocess.PIPE,
-            )
-            if not suppress_cmd_output:
-                bt.logging.info(result.stdout.decode())
-
-        except subprocess.CalledProcessError as e:
-            bt.logging.error(f"❌ Failed to run command ❌: {cmd}")
-            if verbose:
-                bt.logging.error(f"Output: {e.stdout.decode()}")
-                bt.logging.error(f"Error: {e.stderr.decode()}")
-                get_tracebacks()
-            raise
-
 
 def check_and_download_pdbs(
     pdb_directory: str,

folding/utils/ops.py

@@ -17,6 +17,8 @@
 from folding.utils.ops import (
     load_and_sample_random_pdb_ids,
     get_response_info,
+    TimeoutException,
+    OpenMMException,
 )
 
 ROOT_DIR = Path(__file__).resolve().parents[2]
@@ -261,9 +263,18 @@ def try_prepare_challenge(config, pdb_id: str) -> Dict:
                     f"Initial energy is positive: {protein.init_energy}. Simulation failed."
                 )
 
-        except Exception:
+        except TimeoutException as e:
+            bt.logging.info(e)
+            event["validator_search_status"] = False
+            tries = 10
+            
+        except OpenMMException as e:
+            bt.logging.info(f"OpenMMException occurred: init_energy is NaN {e}")
+            event["validator_search_status"] = False
+
+        except Exception as e:
             # full traceback
-            bt.logging.error(traceback.format_exc())
+            bt.logging.info(e)
             event["validator_search_status"] = False
 
         finally:

folding/validators/forward.py

@@ -27,6 +27,7 @@
     write_pkl,
     load_and_sample_random_pdb_ids,
     plot_miner_validator_curves,
+    timeout,
 )
 
 ROOT_DIR = Path(__file__).resolve().parents[2]
@@ -198,6 +199,7 @@ def read_and_return_files(self, filenames: List) -> Dict:
                     continue
         return files_to_return
 
+    @timeout(180)
     def setup_simulation(self):
         """forward method defines the following:
         1. gather the pdb_id and setup the namings.
@@ -578,6 +580,11 @@ def is_run_valid(self):
 
         return True, check_energies.tolist(), miner_energies.tolist()
 
+    def get_ns_computed(self):
+        """Calculate the number of nanoseconds computed by the miner."""
+
+        return (self.cpt_step * self.system_config.time_step_size) / 1e6
+
     def save_pdb(self, output_path: str):
         """Save the pdb file to the output path."""
         positions = self.simulation.context.getState(getPositions=True).getPositions()

folding/validators/protein.py

@@ -29,6 +29,7 @@ def get_energies(
     event["rmsds"] = [0] * len(uids)
     event["process_md_output_time"] = [0] * len(uids)
     event["is_run_valid"] = [0] * len(uids)
+    event["ns_computed"] = [0] * len(uids)
 
     energies = np.zeros(len(uids))
 
@@ -52,7 +53,7 @@ def get_energies(
                     f"uid {uid} responded with status code {resp.dendrite.status_code}"
                 )
                 continue
-
+            ns_computed = protein.get_ns_computed()
             energy = protein.get_energy()
             rmsd = protein.get_rmsd()
 
@@ -70,6 +71,7 @@ def get_energies(
             event["is_valid"][i] = is_valid
             event["reported_energy"][i] = float(energy)
             event["rmsds"][i] = float(rmsd)
+            event["ns_computed"][i] = float(ns_computed)
 
         except Exception as E:
             # If any of the above methods have an error, we will catch here.