Skip to content

Commit 3c797fb

Browse files
nraultwangnraultwang
committed
fix: update Dockerfile and add initial python test code
1 parent 06217c1 commit 3c797fb

14 files changed

+1331
-61
lines changed

‎.github/workflows/build.yml‎

Lines changed: 0 additions & 60 deletions
This file was deleted.

‎.github/workflows/ci.yml‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
name: panoseti_daq ci
2+
3+
on:
4+
push:
5+
branches: [ main, test-build-action ]
6+
pull_request:
7+
branches: [ main ]
8+
9+
jobs:
10+
compile-test:
11+
name: Test compilation and simple functionality
12+
runs-on: ubuntu-latest
13+
steps:
14+
- name: Checkout repository
15+
uses: actions/checkout@v3
16+
17+
- name: Set up Docker Buildx
18+
uses: docker/setup-buildx-action@v3
19+
20+
- name: Build and cache Docker image
21+
uses: docker/build-push-action@v5
22+
with:
23+
context: .
24+
file: ./tests/ci_tests/Dockerfile
25+
load: true
26+
tags: panoseti-daq:latest
27+
cache-from: type=gha
28+
cache-to: type=gha,mode=max
29+
30+
- name: Run Hashpipe in a container
31+
run: |
32+
docker run --rm \
33+
--shm-size=2g \
34+
panoseti-daq:latest \
35+
python3 -m pytest -s -v --maxfail=2 tests/ci_tests/
36+

‎run_ci_tests.sh‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
#!/bin/bash
2+
set -e
3+
4+
# Define the name for the Docker image
5+
IMAGE_NAME="panoseti-daq-ci"
6+
7+
echo "--- Building CI Docker Image: $IMAGE_NAME ---"
8+
docker build -t $IMAGE_NAME -f tests/ci_tests/Dockerfile .
9+
10+
echo "--- Running Integration Tests ---"
11+
# Run the tests inside the container.
12+
# The RUN_REAL_DATA_TESTS=1 environment variable enables the fixture.
13+
# The --rm flag ensures the container is removed after the test run.
14+
docker run --rm \
15+
--shm-size=2g \
16+
$IMAGE_NAME \
17+
python3 -m pytest -v -s --maxfail=2 tests/ci_tests/
18+
19+
echo "--- CI Test Run Completed Successfully ---"

‎snapshot.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -191,13 +191,13 @@ static void uds_connect(uds_connection_t* conn) {
191191

192192
// Gets a connection manager for a given data product.
193193
static uds_connection_t* get_uds_connection(const char* dp_name) {
194-
hashpipe_info(__FUNCTION__, "Getting UDS connection for %s", dp_name);
195194
uds_connection_t* conn;
196195
for (conn = g_uds_connections; conn != NULL; conn = conn->next) {
197196
if (strcmp(conn->dp_name, dp_name) == 0) {
198197
return conn;
199198
}
200199
}
200+
hashpipe_info(__FUNCTION__, "Attempting to create a new UDS client manager for %s", dp_name);
201201
conn = (uds_connection_t*)malloc(sizeof(uds_connection_t));
202202
if (conn == NULL) {
203203
hashpipe_error(__FUNCTION__, "Failed to allocate UDS connection memory");

‎tests/ci_tests/Dockerfile‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
# Stage 1: Build Environment for C/C++ Artifacts
2+
FROM ubuntu:latest AS builder
3+
4+
LABEL stage="builder"
5+
6+
ENV DEBIAN_FRONTEND=noninteractive
7+
8+
# Install build-time dependencies for Hashpipe and Python
9+
RUN apt-get update && \
10+
apt-get install -y --no-install-recommends \
11+
build-essential \
12+
git \
13+
autoconf \
14+
automake \
15+
libtool \
16+
python3 \
17+
python3-pip \
18+
python3-venv \
19+
python3-dev \
20+
g++ \
21+
make && \
22+
rm -rf /var/lib/apt/lists/*
23+
24+
25+
# Build and install Hashpipe
26+
WORKDIR /build
27+
RUN git clone --depth 1 https://github.com/david-macmahon/hashpipe.git hashpipe
28+
WORKDIR /build/hashpipe/src
29+
RUN autoreconf -is && \
30+
./configure --prefix=/usr/local && \
31+
make && \
32+
make install
33+
34+
# Set up the Python Environment
35+
# Activate the venv by adding its bin directory to the PATH.
36+
ENV VENV_PATH=/opt/venv
37+
RUN python3 -m venv $VENV_PATH
38+
ENV PATH="$VENV_PATH/bin:$PATH"
39+
40+
# Install Python dependencies into a python venv
41+
WORKDIR /app
42+
ARG TEST_DIR="tests/ci_tests"
43+
COPY ${TEST_DIR}/requirements.txt .
44+
RUN pip install --no-cache-dir -r requirements.txt
45+
46+
# Build panoseti_daq
47+
# The compiled .so file will be in the workdir /build/panoseti_daq
48+
WORKDIR /build
49+
COPY . panoseti_daq
50+
# RUN git clone --depth 1 --branch uds-to-grpc https://github.com/panoseti/panoseti_daq.git panoseti_daq
51+
WORKDIR /build/panoseti_daq
52+
53+
# Overwrite the Makefile with our Makefile to define small data buffers
54+
# This is necessary for the github actions CI environments which don't give us storage to work with
55+
COPY "${TEST_DIR}/Makefile-TEST" Makefile
56+
COPY "${TEST_DIR}/databuf-TEST.h" databuf.h
57+
58+
RUN make
59+
RUN make install
60+
61+
# Stage 2: Final Runtime and Test Environment
62+
# This stage creates the small final image.
63+
FROM ubuntu:latest
64+
65+
LABEL stage="runtime"
66+
67+
# Copy compiled artifacts from the builder stage
68+
COPY --from=builder /usr/local/bin/hashpipe /usr/local/bin/
69+
COPY --from=builder /usr/local/lib/libhashpipe.so* /usr/local/lib/
70+
COPY --from=builder /usr/local/lib/libhashpipestatus.so* /usr/local/lib/
71+
COPY --from=builder /build/panoseti_daq/hashpipe.so /usr/local/lib/
72+
COPY --from=builder /opt/venv /opt/venv
73+
74+
# Install runtime-only and build-essential dependencies
75+
# build-essential is required for packages that compile C extensions, like netifaces.
76+
RUN apt-get update && \
77+
apt-get install -y --no-install-recommends \
78+
tcpreplay \
79+
python3 \
80+
python3-venv \
81+
build-essential && \
82+
rm -rf /var/lib/apt/lists/*
83+
84+
# Set up the Python Environment
85+
# Activate the venv by adding its bin directory to the PATH.
86+
ENV VENV_PATH=/opt/venv
87+
RUN python3 -m venv $VENV_PATH
88+
ENV PATH="$VENV_PATH/bin:$PATH"
89+
90+
# Update the shared library cache to ensure the system finds libhashpipe.so
91+
RUN ldconfig
92+
RUN apt-get remove build-essential --autoremove -y
93+
94+
# Copy test data
95+
ARG TEST_DIR="tests/ci_tests"
96+
COPY "${TEST_DIR}/PH_onsky_NGC7331_1pixTrig_Anytrig1.5pe__20250801_102213_965_filtered2.pcapng" /app/test_data.pcapng
97+
COPY ./tests/ /app/tests/
98+
99+
# Set a working directory for command execution
100+
WORKDIR /app

‎tests/ci_tests/Makefile-TEST‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
CC = g++
2+
REDIS_LIB_CCFLAGS = -lhiredis
3+
CCFLAGS = -g -O3 -fPIC -shared -lstdc++ \
4+
-I. -I$(CUDA_DIR)/include -I/usr/local/include \
5+
-I ./util \
6+
-L. -L/usr/local/lib \
7+
-lhashpipe -lrt -lm \
8+
-ldl \
9+
-Wl,-rpath
10+
TARGET = hashpipe.so
11+
SOURCES = net_thread.c \
12+
compute_thread.c \
13+
output_thread.c \
14+
process_frame.c \
15+
databuf.c \
16+
snapshot.c \
17+
./util/pff.cpp \
18+
./util/image.cpp
19+
20+
INCLUDES = databuf.h compute_thread.h process_frame.h snapshot.h net_thread.h
21+
22+
N_INPUT_BLOCKS=64
23+
N_OUTPUT_BLOCKS=1
24+
25+
all: $(TARGET)
26+
27+
$(TARGET): $(SOURCES) $(INCLUDES)
28+
$(CC) -o $(TARGET) $(SOURCES) $(CCFLAGS) -DN_INPUT_BLOCKS=$(N_INPUT_BLOCKS) -DN_OUTPUT_BLOCKS=$(N_OUTPUT_BLOCKS)
29+
30+
tags:
31+
ctags -R .
32+
clean:
33+
rm -f $(TARGET) tags
34+
35+
.PHONY: all tags clean install install-lib

‎tests/ci_tests/PH_onsky_NGC7331_1pixTrig_Anytrig1.5pe__20250801_102213_965_filtered2.pcapng‎

604 KB
Binary file not shown.

0 commit comments

Comments
 (0)