Merge pull request #8 from pyobs/performance

Performance enhancement and day pipeline

Author: GermanHydrogen

Readme.md

@@ -12,13 +12,39 @@ This pipeline estimates the limiting magnitude of each pixel based on the visibi
 
 ## Web Service
 
-Route: `/query?ra={Right ascension in degree}&dec={Declination in degree}`
+### Point Query
+Returns the cloudiness value of the analyzed sky position closest to the requested position. <br>
+**Routes**: <br>
+`/query/point?ra={Right ascension in degree}&dec={Declination in degree}` or <br>
+`/query/point?alt={Altitude in degree}&az={Azimuth in degree}`
 
-### Result
-| Field    | Type  | Description                                     |
-|----------|-------|-------------------------------------------------|
+**Example:** <br>
+`/query/point?alt=90.0&az=0.0`
+
+
+**Result:**
+
+| Field    | Type  | Description                                      |
+|----------|-------|--------------------------------------------------|
+| obs_time | float | Observation Unix time of the last analyzed image |
+| value    | bool  | If it is cloudy at the requested point           |
+
+### Area Query
+Returns the cloud fraction within the requested great circle. 
+
+**Routes**: <br>
+`/query/area?ra={Right ascension in degree}&dec={Declination in degree}&radius={Radius in degree}` or <br>
+`/query/area?alt={Altitude in degree}&az={Azimuth in degree}&radius={Radius in degree}`
+
+**Example:** <br>
+`/query/area?alt=90.0&az=0.0&radius=10.0`
+
+**Result:**
+
+| Field    | Type  | Description                                      |
+|----------|-------|--------------------------------------------------|
 | obs_time | float | Observation Unix time of the last analyzed image |
-| value    | float | Limiting magnitude in mag                       |
+| value    | float | Cloud fraction in the requested area in percent  |
 
 
 ## Configuration

data/mask.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3fd9a8cbda94f9a57d23e64da3446978d96a17075829d6ee28c84149ed80c4b3
+size 12888000

data/test_day_image.fits

@@ -0,0 +1,80 @@
+class: pyobs.modules.MultiModule
+
+shared:
+  world:
+    class: pyobs.utils.simulation.SimWorld
+    time: "2020-10-21 12:00:00"
+
+
+modules:
+  cloudcover:
+    class: pyobs_cloudcover.application.Application
+
+    image_sender: "tester"
+
+    server:
+      url: "localhost"
+      port: 8080
+
+    # This is overridden by the test module
+    measurement_log:
+      url: ""
+      bucket: ""
+      org: ""
+      token: ""
+
+    pipelines:
+      day:
+        alt_interval:
+          start: 18
+          end:
+
+        options:
+          world_model:
+            class: pyobs_cloudcover.world_model.SimpleModel
+            a0: 4.81426598e-03
+            F: 2.00000000e+00
+            R: 1.06352627e+03
+            c_x: 7.57115607e+02
+            c_y: 5.11194838e+02
+
+          mask_filepath: "data/mask.npy"
+
+          cloud_map:
+            threshold: 3.5
+
+          coverage_info:
+            zenith_altitude: 80
+
+    comm:
+      class: pyobs.comm.local.LocalComm
+      name: "cloudcover"
+
+  tester:
+    class: pyobs_cloudcover.testing.TestModule
+    image_path: "data/test_day_image.fits"
+    total_fraction: 0.7
+    zenith_fraction: 0.02
+    zenith_value: False
+    zenith_average: 2.3
+
+    comm:
+      class: pyobs.comm.local.LocalComm
+      name: "tester"
+
+vfs:
+  class: pyobs.vfs.VirtualFileSystem
+  roots:
+    cache:
+      class: pyobs.vfs.MemoryFile
+      #root: .
+
+comm:
+  class: pyobs.comm.local.LocalComm
+  name: multi
+
+timezone: Europe/Berlin
+location:
+    longitude: 9.944333
+    latitude: 51.560583
+    elevation: 201.0000000008158

data/test_image.fits data/test_night_image.fitsdata/test_image.fits renamed to data/test_night_image.fits

@@ -3,23 +3,6 @@ class: pyobs_cloudcover.application.Application
 # Camera which images are used for cloud detection
 image_sender: "allskycam"
 
-# Astronomical solution
-# Possible models:
-#   Simple Model
-#     class: pyobs_cloudcover.world_model.SimpleModel
-#     parameters: (a0, F, R, c_x, c_y) see https://ui.adsabs.harvard.edu/abs/2019A&A...626A.105B
-#   WCS Model
-#     class: pyobs_cloudcover.world_model.WCSModel
-#     parameters:
-#       file_path: Path to FITS file containing WCS
-model:
-  class: pyobs_cloudcover.world_model.SimpleModel
-  a0: 4.81426598e-03
-  F: 2.00000000e+00
-  R: 1.06352627e+03
-  c_x: 7.57115607e+02
-  c_y: 5.11194838e+02
-
 # Web Service
 server:
   # Host Name
@@ -37,13 +20,63 @@ measurement_log:
 # Different image analysis pipelines can be used for different solar altitude intervals
 # Currently only the "night" pipeline is available, which works best after the astronomical dawn
 pipelines:
+  day:
+    alt_interval:
+      start: 18
+      end:
+
+    options:
+      # Astronomical solution
+      # Possible models:
+      #   Simple Model
+      #     class: pyobs_cloudcover.world_model.SimpleModel
+      #     parameters: (a0, F, R, c_x, c_y) see https://ui.adsabs.harvard.edu/abs/2019A&A...626A.105B
+      #   WCS Model
+      #     class: pyobs_cloudcover.world_model.WCSModel
+      #     parameters:
+      #       file_path: Path to FITS file containing WCS
+      world_model:
+        class: pyobs_cloudcover.world_model.SimpleModel
+        a0: 4.81426598e-03
+        F: 2.00000000e+00
+        R: 1.06352627e+03
+        c_x: 7.57115607e+02
+        c_y: 5.11194838e+02
+
+      # Filepath to the image mask
+      mask_filepath: "data/mask.npy"
+
+      cloud_map:
+        # Under which color ratio a pixel is considered cloudy (blue/red + blue/green)
+        threshold: 3.5
+
+      coverage_info:
+        # Altitude which is considered to be the lower boundary of the zenith to average for the zenith cloud fraction measurement (in degree)
+        zenith_altitude: 80
   night:
     # Solar altitude interval (in degree), in which this pipeline should be used
     alt_interval:
       start:
       end: -18
 
     options:
+      # Astronomical solution
+      # Possible models:
+      #   Simple Model
+      #     class: pyobs_cloudcover.world_model.SimpleModel
+      #     parameters: (a0, F, R, c_x, c_y) see https://ui.adsabs.harvard.edu/abs/2019A&A...626A.105B
+      #   WCS Model
+      #     class: pyobs_cloudcover.world_model.WCSModel
+      #     parameters:
+      #       file_path: Path to FITS file containing WCS
+      world_model:
+        class: pyobs_cloudcover.world_model.SimpleModel
+        a0: 4.81426598e-03
+        F: 2.00000000e+00
+        R: 1.06352627e+03
+        c_x: 7.57115607e+02
+        c_y: 5.11194838e+02
+
       preprocessor:
         # Filepath to the image mask
         mask_filepath: "tests/integration/mask.npy"
@@ -72,14 +105,23 @@ pipelines:
         # Rectangle around the calculated star position with 2 * window_size + 1 length, in which the star searched for (in px)
         window_size: 6.0
 
-      cloud_map:
+      # Determines at which celestial positions the limiting magnitude is evaluated
+      altaz_grid:
+        # Number of equidistant points on the hole celestial sphere
+        point_number: 100000
+        # Lower boundary for altitude
+        limiting_altitude: 30
+
+      lim_mag_map:
         # Radius in which to use stars for estimating the limiting magnitude (in degree)
         radius: 7.0
         # Number of consecutive images in which a star has to be visible to be considered visible for the limiting magnitude estimation
         integration_length: 1
 
-      coverage_info:
+      cloud_map:
         # Under which limiting magnitude a pixel is considered cloudy (in mag)
-        cloud_threshold: 5.5
-        # Radius around the zenith to average for the zenith cloud fraction measurement (in degree)
-        zenith_radius: 20
+        threshold: 5.5
+
+      coverage_info:
+        # Altitude which is considered to be the lower boundary of the zenith to average for the zenith cloud fraction measurement (in degree)
+        zenith_altitude: 80

day_integration_test.yaml

@@ -12,10 +12,6 @@ modules:
 
     image_sender: "tester"
 
-    model:
-      class: pyobs_cloudcover.world_model.WCSModel
-      file_path: "data/test_wcs.fits"
-
     server:
       url: "localhost"
       port: 8080
@@ -34,6 +30,10 @@ modules:
           end: -18
 
         options:
+          world_model:
+            class: pyobs_cloudcover.world_model.WCSModel
+            file_path: "data/test_wcs.fits"
+
           preprocessor:
             mask_filepath: ""
             bin_size: 2
@@ -48,22 +48,29 @@ modules:
           reverse_matcher:
             sigma_threshold: 4.0
             window_size: 10.0
-
-          cloud_map:
+          altaz_grid:
+            point_number: 100000
+            limiting_altitude: 30
+          lim_mag_map:
             radius: 7.0
             integration_length: 1
+          cloud_map:
+            threshold: 5.5
 
           coverage_info:
-            cloud_threshold: 5.5
-            zenith_radius: 20
+            zenith_altitude: 80
 
     comm:
       class: pyobs.comm.local.LocalComm
       name: "cloudcover"
 
   tester:
     class: pyobs_cloudcover.testing.TestModule
-    image_path: "data/test_image.fits"
+    image_path: "data/test_night_image.fits"
+    total_fraction: 0.1
+    zenith_fraction: 0.0
+    zenith_value: False
+    zenith_average: 0.0
 
     comm:
       class: pyobs.comm.local.LocalComm

example.yaml

@@ -8,16 +8,13 @@
 from pyobs_cloudcover.measurement_log.influx import Influx
 from pyobs_cloudcover.pipeline.pipeline_controller_factory import PipelineControllerFactory
 from pyobs_cloudcover.web_api.server_factory import ServerFactory
-from pyobs_cloudcover.world_model import WorldModel
-from pyobs_cloudcover.world_model.world_model_factory import WorldModelFactory
 
 log = logging.getLogger(__name__)
 
 
 class Application(Module):
     def __init__(self,
                  image_sender: str,
-                 model: Dict[str, Any],
                  server: Dict[str, Any],
                  measurement_log: Dict[str, Any],
                  pipelines: Dict[str, Dict[str, Any]], *args: Any, **kwargs: Any) -> None:
@@ -26,15 +23,12 @@ def __init__(self,
 
         self._image_sender = image_sender
 
-        world_model_factory = WorldModelFactory(self.observer)
-        world_model: WorldModel = world_model_factory(model)
-
-        server_factory = ServerFactory(self.observer, world_model)
+        server_factory = ServerFactory(self.observer)
         self._server = server_factory(server)
 
         self._measurement_log = Influx(**measurement_log)
 
-        pipeline_controller_factory = PipelineControllerFactory(self.observer, world_model)
+        pipeline_controller_factory = PipelineControllerFactory(self.observer)
         self._pipeline_controller = pipeline_controller_factory(pipelines)
 
     async def open(self) -> None:
@@ -58,8 +52,10 @@ async def process_new_image(self, event: Event, sender: str) -> None:
 
         measurement = self._pipeline_controller(image.data, obs_time)
 
-        log.info("Finished measurement!")
-
         if measurement is not None:
+            log.debug(f"{measurement.total_cover}, {measurement.zenith_cover}, {measurement.change}")
+
             self._server.set_measurement(measurement)
             self._measurement_log(measurement)
+
+        log.info("Finished measurement!")

integration_test.yaml night_integration_test.yamlintegration_test.yaml renamed to night_integration_test.yaml

@@ -1,19 +1,14 @@
 import datetime
 from typing import Optional
 
-import numpy as np
-import numpy.typing as npt
+from cloudmap_rs import SkyPixelQuery
 
 
 class CloudCoverageInfo(object):
-    def __init__(self, cloud_cover_map: npt.NDArray[np.float_],
-                 total_cover: float, average: float, std: float, zenith_cover: float, zenith_average: float, zenith_std: float, change: Optional[float], obs_time: datetime.datetime) -> None:
-        self.cloud_cover_map = cloud_cover_map
+    def __init__(self, cloud_cover_query: SkyPixelQuery,
+                 total_cover: float, zenith_cover: Optional[float], change: Optional[float], obs_time: datetime.datetime) -> None:
+        self.cloud_cover_query = cloud_cover_query
         self.total_cover = total_cover
-        self.average = average
-        self.std = std
         self.zenith_cover = zenith_cover
-        self.zenith_average = zenith_average
-        self.zenith_std = zenith_std
         self.change = change
         self.obs_time = obs_time

pyobs_cloudcover/application.py

@@ -0,0 +1,6 @@
+from pyobs_cloudcover.cloud_info_calculator.coverage_info_calculator import CoverageInfoCalculator
+from pyobs_cloudcover.cloud_info_calculator.zenith_cloud_coverage_calculator import ZenithCloudCoverageCalculator
+from pyobs_cloudcover.cloud_info_calculator.cloud_info_calculator_factory import CloudInfoCalculatorFactory
+from pyobs_cloudcover.cloud_info_calculator.cloud_info_calculator_options import CloudInfoCalculatorOptions
+
+__all__ = ["CoverageInfoCalculator", "ZenithCloudCoverageCalculator", "CloudInfoCalculatorFactory", "CloudInfoCalculatorOptions"]