Migrate to pyenergyplus (#541)

* update to openstudio 3.8, e+ 24.1 and pyfmi 2.11

* first pass at migration to pyenergyplus

* first version of pyenergyplus migration passing integration tests

* fix formatting, bump min python to 3.11

* fix exception import stack

* fix api tests, import changes, point id changes

* add all the new stuff that didn't get comitted earlier

* remove goaws from simulation ci test

* fix simulation test

* fix arguments for timescale test

* starting fixing mock step job

* implement initialize_simulation

* correct increment of time in mock job

* remove deprecated models from the scaling tests

* reduce scale test models to current set

* fix simulation model testing

* fix influxdb reporting

* add sleep to allow mock step run to fall behind

* fix step duration

* Remove more redundant variables in modelica step_run.py

* bump bake-action to v5

* remove files which are no longer needed

* Cleanup and document StepRunBase

* clean up StepRunProcess and begin cleaning openstudio StepRun

* refactor handling of points for openstudio

* fix unit tests

Author: TShapinsky

.github/workflows/ci.yml

@@ -21,7 +21,7 @@ jobs:
       - name: Install Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.8"
+          python-version: "3.11"
 
       - name: Run pre-commit
         uses: pre-commit/[email protected]
@@ -39,7 +39,7 @@ jobs:
       - name: Install Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.8"
+          python-version: "3.11"
 
       - name: Install poetry
         uses: abatilo/actions-poetry@v3
@@ -76,7 +76,7 @@ jobs:
         uses: docker/setup-buildx-action@v3
 
       - name: Build
-        uses: docker/bake-action@v4
+        uses: docker/bake-action@v5
         with:
           files: docker-compose.yml, docker-compose.dev.yml
           set: |
@@ -92,7 +92,7 @@ jobs:
           GIT_COMMIT: ${{ github.sha }}
         run: |
           printenv
-          docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d worker mongo redis minio mc goaws
+          docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d worker mongo redis minio mc
 
       - name: Dump docker logs before tests
         uses: jwalton/gh-docker-logs@v2
@@ -117,7 +117,7 @@ jobs:
       - name: Install Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.8"
+          python-version: "3.11"
 
       - name: Install poetry
         uses: abatilo/actions-poetry@v3
@@ -128,7 +128,7 @@ jobs:
         uses: docker/setup-buildx-action@v3
 
       - name: Build
-        uses: docker/bake-action@v4
+        uses: docker/bake-action@v5
         with:
           files: docker-compose.yml
           set: |
@@ -196,7 +196,7 @@ jobs:
         uses: docker/setup-buildx-action@v3
 
       - name: Build
-        uses: docker/bake-action@v4
+        uses: docker/bake-action@v5
         with:
           load: true
           files: docker-compose.yml, docker-compose.dev.yml, docker-compose.historian.yml

.pre-commit-config.yaml

@@ -26,8 +26,8 @@ repos:
       - id: requirements-txt-fixer
       - id: mixed-line-ending
         args: ["--fix=auto"]
-  - repo: https://github.com/pre-commit/mirrors-autopep8
-    rev: v2.0.1
+  - repo: https://github.com/hhatto/autopep8
+    rev: v2.3.1
     hooks:
       - id: autopep8
         args:

alfalfa_web/server/api-v2.js

@@ -106,7 +106,7 @@ router.param("pointId", (req, res, next, id) => {
   const error = validate(
     { id },
     {
-      id: "required|uuid"
+      id: "required|string"
     }
   );
   if (error) return res.status(400).json({ message: error });
@@ -164,6 +164,13 @@ router.get("/runs/:runId/time", async (req, res, next) => {
     .catch(next);
 });
 
+router.get("/runs/:runId/log", async (req, res, next) => {
+  api
+    .getRunLog(req.run)
+    .then((log) => res.json({ payload: { log } }))
+    .catch(next);
+});
+
 router.get("/runs/:runId/points", (req, res, next) => {
   api
     .getPointsByRun(req.run)

alfalfa_web/server/api.js

@@ -92,6 +92,11 @@ class AlfalfaAPI {
     return await getHashValue(this.redis, run.ref_id, "sim_time");
   };
 
+  getRunLog = async (run) => {
+    const log_lines = await this.redis.lRange(`run:${run.ref_id}:log`, -100, -1);
+    return log_lines.join("\n");
+  };
+
   getPointsByRun = async (run) => {
     const pointsCursor = this.points.find({ run: run._id });
     return Promise.resolve(pointsCursor.toArray());
@@ -126,7 +131,8 @@ class AlfalfaAPI {
     const pointDict = {
       id: point.ref_id,
       name: point.name,
-      type: point.point_type
+      type: point.point_type,
+      units: point.units
     };
     return pointDict;
   };
@@ -197,7 +203,7 @@ class AlfalfaAPI {
 
     const { startDatetime, endDatetime, timescale, realtime, externalClock } = data;
 
-    const job = `alfalfa_worker.jobs.${sim_type === "MODELICA" ? "modelica" : "openstudio"}.StepRun`;
+    const job = `alfalfa_worker.jobs.${sim_type === "MODELICA" ? "modelica" : "openstudio"}.step_run.StepRun`;
     const params = {
       run_id: run.ref_id,
       start_datetime: startDatetime,
@@ -297,7 +303,9 @@ class AlfalfaAPI {
 
   createRunFromModel = async (model) => {
     const runId = uuidv1();
-    const job = `alfalfa_worker.jobs.${model.model_name.endsWith(".fmu") ? "modelica" : "openstudio"}.CreateRun`;
+    const job = `alfalfa_worker.jobs.${
+      model.model_name.endsWith(".fmu") ? "modelica" : "openstudio"
+    }.create_run.CreateRun`;
     const params = {
       model_id: model.ref_id,
       run_id: runId

alfalfa_worker/Dockerfile

@@ -1,4 +1,4 @@
-FROM ghcr.io/nrel/alfalfa-dependencies:3.1.0 AS base
+FROM ghcr.io/nrel/alfalfa-dependencies:prepare_080 AS base
 
 ENV HOME=/alfalfa
 
@@ -21,10 +21,7 @@ ENV PYTHONPATH="${HOME}:${PYTHONPATH}"
 
 COPY ./alfalfa_worker ${HOME}/alfalfa_worker
 
-RUN pip3.8 install virtualenv \
-  && pip3.8 install \
-  scipy \
-  symfit
+COPY ./alfalfa_worker /alfalfa/alfalfa_worker
 
 COPY ./deploy /alfalfa/deploy
 COPY ./deploy/wait-for-it.sh /usr/local/wait-for-it.sh

alfalfa_worker/__main__.py

@@ -3,14 +3,22 @@
 import os
 import sys
 import traceback
+from logging import StreamHandler, basicConfig
 from pathlib import Path
 
 # Determine which worker to load based on the QUEUE.
 # This may be temporary for now, not sure on how else
 # to determine which worker gets launched
 from alfalfa_worker.dispatcher import Dispatcher
+from alfalfa_worker.lib.constants import DATETIME_FORMAT
 
 if __name__ == '__main__':
+
+    basicConfig(level=os.environ.get("LOGLEVEL", "INFO"),
+                handlers=[StreamHandler(sys.stdout)],
+                format='%(asctime)s - %(name)s - %(levelname)s: %(message)s',
+                datefmt=DATETIME_FORMAT)
+
     try:
         workdir = Path(os.environ.get('RUN_DIR', '/runs'))
         dispatcher = Dispatcher(workdir)

alfalfa_worker/dispatcher.py

@@ -53,7 +53,7 @@ def process_message(self, message):
         """
         try:
             message_body = json.loads(message)
-            self.logger.info(f"Processing message of {message_body}")
+            self.logger.debug(f"Processing message of {message_body}")
             job = message_body.get('job')
             if job:
                 params = message_body.get('params', {})

alfalfa_worker/jobs/modelica/create_run.py

@@ -1,9 +1,4 @@
-import json
-import os
 from pathlib import Path
-from uuid import uuid4
-
-from pyfmi import load_fmu
 
 from alfalfa_worker.lib.enums import RunStatus, SimType
 from alfalfa_worker.lib.job import Job
@@ -19,7 +14,6 @@ def __init__(self, model_id, run_id=None):
         # Define FMU specific attributes
         self.upload_fmu: Path = self.dir / model_name
         self.fmu_path = self.dir / 'model.fmu'
-        self.fmu_json = self.dir / 'tags.json'
         self.model_name = model_name
 
         # Needs to be set after files are uploaded / parsed.
@@ -34,99 +28,11 @@ def exec(self):
         """
         self.logger.info("add_fmu for {}".format(self.run.ref_id))
 
-        # Create the FMU tags (no longer external now that python2 is deprecated)
-        self.create_tags()
-        # insert tags into db
-        self.insert_fmu_tags()
         self.upload_fmu.rename(self.fmu_path)
 
     def validate(self) -> None:
         assert (self.dir / 'model.fmu').exists(), "model file not created"
-        assert (self.dir / 'tags.json').exists(), "tags file not created"
 
     def cleanup(self) -> None:
         super().cleanup()
         self.set_run_status(RunStatus.READY)
-
-    def get_site_ref(self, haystack_json):
-        """
-        Find the site given the haystack JSON file.  Remove 'r:' from string.
-        :param haystack_json: json serialized Haystack document
-        :return: site_ref: id of site
-        """
-        site_ref = ''
-        with open(haystack_json) as json_file:
-            data = json.load(json_file)
-            for entity in data:
-                if 'site' in entity:
-                    if entity['site'] == 'm:':
-                        site_ref = entity['id'].replace('r:', '')
-                        break
-        return site_ref
-
-    def insert_fmu_tags(self):
-        with open(self.fmu_json, 'r') as f:
-            data = f.read()
-        points_json = json.loads(data)
-
-        self.run_manager.add_site_to_mongo(points_json, self.run)
-
-    def create_tags(self):
-        # 1.0 setup the inputs
-        fmu = load_fmu(self.upload_fmu)
-
-        # 2.0 get input/output variables from the FMU
-        #   causality = 1 is parameter, 2 is input, 3 is output
-        input_names = fmu.get_model_variables(causality=2).keys()
-        output_names = fmu.get_model_variables(causality=3).keys()
-
-        # 3.0 add site tagging
-        tags = []
-
-        fmu_upload_name = os.path.basename(self.model_name)  # without directories
-        fmu_upload_name = os.path.splitext(fmu_upload_name)[0]  # without extension
-
-        # TODO: Figure out how to find geo_city
-        sitetag = {
-            "dis": "s:%s" % fmu_upload_name,
-            "id": "r:%s" % self.run.ref_id,
-            "site": "m:",
-            "datetime": "s:",
-            "simStatus": "s:Stopped",
-            "simType": "s:fmu",
-            "siteRef": "r:%s" % self.run.ref_id
-        }
-        tags.append(sitetag)
-
-        # 4.0 add input tagging
-        for var_input in input_names:
-            if not var_input.endswith("_activate"):
-                tag_input = {
-                    "id": "r:%s" % uuid4(),
-                    "dis": "s:%s" % var_input,
-                    "siteRef": "r:%s" % self.run.ref_id,
-                    "point": "m:",
-                    "writable": "m:",
-                    "writeStatus": "s:disabled",
-                    "kind": "s:Number",
-                }
-                tags.append(tag_input)
-                tag_input = {}
-
-        # 5.0 add output tagging
-        for var_output in output_names:
-            tag_output = {
-                "id": "r:%s" % uuid4(),
-                "dis": "s:%s" % var_output,
-                "siteRef": "r:%s" % self.run.ref_id,
-                "point": "m:",
-                "cur": "m:",
-                "curVal": "n:",
-                "curStatus": "s:disabled",
-                "kind": "s:Number",
-            }
-            tags.append(tag_output)
-
-        # 6.0 write tags to the json file
-        with open(self.fmu_json, 'w') as outfile:
-            json.dump(tags, outfile)