tests: increase shared memory size

Author: nraultwang

.github/workflows/ci.yml

@@ -2,68 +2,68 @@ name: PANOSETI gRPC CI
 
 on:
   push:
-    branches: [ tests ]
+    branches: [ main, tests ]
   pull_request:
-    branches: [ tests ]
+    branches: [ main ]
 
 jobs:
-#  daq-data-build-and-test:
-#    runs-on: ubuntu-latest
-#    steps:
-#      # Checks-out repository
-#      - name: Checkout repository
-#        uses: actions/checkout@v3
-#
-#      # Set up Docker Buildx
-#      # enables advanced features like multi-platform builds and cache management.
-#      - name: Set up Docker Buildx
-#        uses: docker/setup-buildx-action@v3
-#
-#      #Build the Docker image with cache enabled
-#      # This step builds the image and pushes the cache to the GitHub Actions cache.
-#      # The 'load: true' flag makes the image available to subsequent steps in the same job.
-#      - name: Build and cache Docker image
-#        uses: docker/build-push-action@v5
-#        with:
-#          context: .
-#          file: ./tests/daq_data/Dockerfile
-#          load: true # Makes the image available for the next step
-#          tags: panoseti-grpc-test-daq-data:latest
-#          cache-from: type=gha # Pulls cache from GitHub Actions cache
-#          cache-to: type=gha,mode=max # Pushes cache to GitHub Actions cache
-#
-#      # Run tests inside the now-cached Docker container
-#      - name: Run Pytest in container
-#        run: docker run --rm panoseti-grpc-test-daq-data:latest
-#
-#  ublox-control-build-and-test:
-#    runs-on: ubuntu-latest
-#    steps:
-#      # Checks-out repository
-#      - name: Checkout repository
-#        uses: actions/checkout@v3
-#
-#      # Set up Docker Buildx
-#      # enables advanced features like multi-platform builds and cache management.
-#      - name: Set up Docker Buildx
-#        uses: docker/setup-buildx-action@v3
-#
-#      #Build the Docker image with cache enabled
-#      # This step builds the image and pushes the cache to the GitHub Actions cache.
-#      # The 'load: true' flag makes the image available to subsequent steps in the same job.
-#      - name: Build and cache Docker image
-#        uses: docker/build-push-action@v5
-#        with:
-#          context: .
-#          file: ./tests/ublox_control/Dockerfile
-#          load: true # Makes the image available for the next step
-#          tags: panoseti-grpc-test-ublox-control:latest
-#          cache-from: type=gha # Pulls cache from GitHub Actions cache
-#          cache-to: type=gha,mode=max # Pushes cache to GitHub Actions cache
-#
-#      # Run tests inside the now-cached Docker container
-#      - name: Run Pytest in container
-#        run: docker run --rm panoseti-grpc-test-ublox-control:latest
+  daq-data-build-and-test:
+    runs-on: ubuntu-latest
+    steps:
+      # Checks-out repository
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      # Set up Docker Buildx
+      # enables advanced features like multi-platform builds and cache management.
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      #Build the Docker image with cache enabled
+      # This step builds the image and pushes the cache to the GitHub Actions cache.
+      # The 'load: true' flag makes the image available to subsequent steps in the same job.
+      - name: Build and cache Docker image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ./tests/daq_data/Dockerfile
+          load: true # Makes the image available for the next step
+          tags: panoseti-grpc-test-daq-data:latest
+          cache-from: type=gha # Pulls cache from GitHub Actions cache
+          cache-to: type=gha,mode=max # Pushes cache to GitHub Actions cache
+
+      # Run tests inside the now-cached Docker container
+      - name: Run Pytest in container
+        run: docker run --rm panoseti-grpc-test-daq-data:latest
+
+  ublox-control-build-and-test:
+    runs-on: ubuntu-latest
+    steps:
+      # Checks-out repository
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      # Set up Docker Buildx
+      # enables advanced features like multi-platform builds and cache management.
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      #Build the Docker image with cache enabled
+      # This step builds the image and pushes the cache to the GitHub Actions cache.
+      # The 'load: true' flag makes the image available to subsequent steps in the same job.
+      - name: Build and cache Docker image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ./tests/ublox_control/Dockerfile
+          load: true # Makes the image available for the next step
+          tags: panoseti-grpc-test-ublox-control:latest
+          cache-from: type=gha # Pulls cache from GitHub Actions cache
+          cache-to: type=gha,mode=max # Pushes cache to GitHub Actions cache
+
+      # Run tests inside the now-cached Docker container
+      - name: Run Pytest in container
+        run: docker run --rm panoseti-grpc-test-ublox-control:latest
   real-daq-robustness-tests:
     name: Real DAQ Robustness Tests
     runs-on: ubuntu-latest
@@ -93,5 +93,5 @@ jobs:
             --shm-size=2g \
             --env RUN_REAL_DATA_TESTS=${RUN_REAL_DATA_TESTS} \
             panoseti-grpc-test-real-daq:latest \
-            python3 -m pytest -v tests/daq_data_hashpipe/
+            python3 -m pytest -s -v --maxfail=2 tests/daq_data_hashpipe/
 

panoseti_util/stop_daq.py

@@ -9,7 +9,8 @@
 #
 # On success, print OK.  Otherwise print an error message
 
-import os, signal
+import os, signal, sys
+sys.path.append('/app/')
 from panoseti_util import control_utils as util
 
 def main():

tests/daq_data_hashpipe/Makefile

@@ -19,7 +19,7 @@ SOURCES  = net_thread.c \
 
 INCLUDES = databuf.h compute_thread.h process_frame.h snapshot.h
 
-N_INPUT_BLOCKS=4
+N_INPUT_BLOCKS=64
 N_OUTPUT_BLOCKS=1
 
 all: $(TARGET)

tests/daq_data_hashpipe/conftest.py

@@ -15,11 +15,13 @@
 import copy
 from typing import Optional, Tuple
 
+from google.protobuf.struct_pb2 import Struct
+from google.protobuf.json_format import ParseDict
+
 from daq_data.server import serve
 from daq_data.client import AioDaqDataClient, DaqDataClient
 from daq_data.daq_data_pb2 import PanoImage
-from google.protobuf.struct_pb2 import Struct
-from google.protobuf.json_format import ParseDict
+from panoseti_util import control_utils as util
 
 TEST_CFG_DIR = Path("tests/daq_data/config")
 TEST_CFG_DIR.mkdir(exist_ok=True)
@@ -76,7 +78,7 @@ def hashpipe_pcap_runner():
     # Command to loop the pcap file to the loopback interface, simulating network traffic.
     tcpreplay_cmd = [
         "tcpreplay",
-        "--mbps=0.5",
+        "--mbps=1",
         "--loop=0",  # Loop indefinitely
         "--intf1=lo",  # Send to loopback interface
         pcap_file
@@ -107,11 +109,20 @@ def hashpipe_pcap_runner():
     )
 
     # --- 4. Wait for Initialization and Validation ---
-    time.sleep(5)  # Allow time for processes to initialize and sockets to be created.
-    if tcpreplay_proc.poll() is not None:
-        pytest.fail(f"tcpreplay failed to start. Exit code: {tcpreplay_proc.returncode}")
-    if hashpipe_proc.poll() is not None:
-        pytest.fail(f"hashpipe failed to start. Exit code: {hashpipe_proc.returncode}. Check logs in {base_dir}.")
+    num_retries = 20
+    for i in range(num_retries):
+        # Allow time for processes to initialize and sockets to be created.
+        if tcpreplay_proc.poll() is not None:
+            pytest.fail(f"tcpreplay failed to start. Exit code: {tcpreplay_proc.returncode}")
+        elif hashpipe_proc.poll() is not None:
+            pytest.fail(f"hashpipe failed to start. Exit code: {hashpipe_proc.returncode}. Check logs in {base_dir}.")
+        if util.is_hashpipe_running():
+            print(f"hashpipe is running after {i} retries.")
+            break
+        print(f"hashpipe is not running after {i}/{num_retries} retries. Retrying in 1 second.")
+        time.sleep(1)
+    else:
+        pytest.fail(f"hashpipe failed to start after {num_retries} retries. Check logs in {base_dir}.")
 
     # Yield to let the tests run
     yield

tests/daq_data_hashpipe/databuf.h

@@ -21,21 +21,21 @@
 
 // the size and characteristics of the input and output circular buffers
 
-#define CACHE_ALIGNMENT 256
+#define CACHE_ALIGNMENT 16
 // Align the cache within the buffer
 #ifndef N_INPUT_BLOCKS
-    #define N_INPUT_BLOCKS 16
+    #define N_INPUT_BLOCKS 4
 #endif
 // Number of blocks in the input buffer
 #ifndef N_OUTPUT_BLOCKS
-    #define N_OUTPUT_BLOCKS 2
+    #define N_OUTPUT_BLOCKS 1
 #endif
 // Number of blocks in the output buffer
-#define IN_PKT_PER_BLOCK 16
+#define IN_PKT_PER_BLOCK 1
 // Number of input packets stored in each block of the input buffer
-#define OUT_MOD_PER_BLOCK 32
+#define OUT_MOD_PER_BLOCK 1
 // Max Number of Modules stored in each block of the output buffer
-#define OUT_PH_IMG_PER_BLOCK 32
+#define OUT_PH_IMG_PER_BLOCK 1
 // Max # of PH packets stored in each block of the output buffer
 
 // Imaging Data Values and characteristics of modules

tests/daq_data_hashpipe/run_ci_tests.sh

@@ -12,9 +12,9 @@ echo "--- Running Integration Tests ---"
 # The RUN_REAL_DATA_TESTS=1 environment variable enables the fixture.
 # The --rm flag ensures the container is removed after the test run.
 docker run --rm \
-    --shm-size=1g \
+    --shm-size=2g \
     -e RUN_REAL_DATA_TESTS=1 \
     $IMAGE_NAME \
-    python3 -m pytest -v tests/daq_data_hashpipe/
+    python3 -m pytest -v -s --maxfail=2 tests/daq_data_hashpipe/
 
 echo "--- CI Test Run Completed Successfully ---"