Skip to content

Commit d35c2c3

Browse files
AdamGlusteinAdamGlustein
authored
Setup a sanitized build for CSP (#577)
* Setup a sanitized build for CSP Signed-off-by: Adam Glustein <adam.glustein@point72.com>
1 parent 65349af commit d35c2c3

12 files changed

Lines changed: 136 additions & 18 deletions

File tree

‎.github/workflows/build.yml‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -68,9 +68,8 @@ jobs:
6868
runs-on: ubuntu-24.04
6969

7070
outputs:
71-
COMMIT_MESSAGE: ${{ steps.setup.outputs.COMMIT_MSG }}
71+
COMMIT_MESSAGE: ${{ steps.getcommitpush.outputs.COMMIT_MSG || steps.getcommitpr.outputs.COMMIT_MSG }}
7272
FULL_RUN: ${{ steps.setuppush.outputs.FULL_RUN || steps.setuppr.outputs.FULL_RUN || steps.setupmanual.outputs.FULL_RUN || steps.setupschedule.outputs.FULL_RUN }}
73-
7473
steps:
7574
- name: Checkout
7675
uses: actions/checkout@v5
@@ -79,11 +78,13 @@ jobs:
7978
fetch-depth: 2
8079

8180
- name: Get Commit Message
82-
run: echo "COMMIT_MSG=$(git log -1 --pretty=%B HEAD | tr '\n' ' ')" >> $GITHUB_ENV
81+
id: getcommitpush
82+
run: echo "COMMIT_MSG=$(git log -1 --pretty=%B HEAD | tr '\n' ' ')" >> $GITHUB_OUTPUT
8383
if: ${{ github.event_name == 'push' }}
8484

8585
- name: Get Commit Message
86-
run: echo "COMMIT_MSG=$(git log -1 --pretty=%B HEAD^2 | tr '\n' ' ')" >> $GITHUB_ENV
86+
id: getcommitpr
87+
run: echo "COMMIT_MSG=$(git log -1 --pretty=%B HEAD^2 | tr '\n' ' ')" >> $GITHUB_OUTPUT
8788
if: ${{ github.event_name == 'pull_request' }}
8889

8990
- name: Display and Setup Build Args (Push)

‎.github/workflows/conda.yml‎

Lines changed: 43 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
name: Conda End-to-end Test
22

3+
env:
4+
SANITIZER_CI_SCHEDULE: '25 6 * * 5'
5+
36
on:
47
push:
58
branches:
@@ -18,9 +21,16 @@ on:
1821
- README.md
1922
- "docs/**"
2023
workflow_dispatch:
24+
inputs:
25+
sanitizer:
26+
description: "Run sanitized build"
27+
required: false
28+
type: boolean
29+
default: false
2130
schedule:
2231
# Run conda CI on Monday and Thursday at 1:25am EST (06:25 UTC)
23-
- cron: '25 6 * * 1,4'
32+
# Run conda sanitized builds on Fridays at 1:25 am EST (06:25 UTC)
33+
- cron: '25 6 * * 1,4,5'
2434

2535
concurrency:
2636
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
@@ -31,7 +41,32 @@ permissions:
3141
checks: write
3242

3343
jobs:
44+
initialize:
45+
runs-on: ubuntu-24.04
46+
outputs:
47+
SANITIZER: ${{ steps.setupmanual.outputs.SANITIZER || steps.setupschedule.outputs.SANITIZER }}
48+
steps:
49+
- name: Display and Setup Build Args (Manual)
50+
id: setupmanual
51+
run: |
52+
echo "Sanitizer: $SANITIZER"
53+
echo "SANITIZER=$SANITIZER" >> $GITHUB_OUTPUT
54+
env:
55+
SANITIZER: ${{ github.event.inputs.sanitizer }}
56+
if: ${{ github.event_name == 'workflow_dispatch' }}
57+
58+
- name: Display and Setup Build Args (Schedule)
59+
id: setupschedule
60+
run: |
61+
echo "Sanitizer: $SANITIZER"
62+
echo "SANITIZER=$SANITIZER" >> $GITHUB_OUTPUT
63+
env:
64+
SANITIZER: ${{ github.event.schedule == env.SANITIZER_CI_SCHEDULE }}
65+
if: ${{ github.event_name == 'schedule' }}
66+
3467
build:
68+
needs:
69+
- initialize
3570
strategy:
3671
matrix:
3772
os:
@@ -81,7 +116,7 @@ jobs:
81116
if: ${{ runner.os == 'Windows' }}
82117

83118
- name: Python Build Steps
84-
run: make build-conda
119+
run: make build-conda ${{ needs.initialize.outputs.SANITIZER && 'ASAN="ON" UBSAN="ON"' || '' }}
85120
shell: micromamba-shell {0}
86121
if: ${{ runner.os != 'Windows' }}
87122

@@ -95,7 +130,12 @@ jobs:
95130
- name: Python Test Steps
96131
run: make test
97132
shell: micromamba-shell {0}
98-
if: ${{ runner.os != 'Windows' }}
133+
if: ${{ runner.os != 'Windows' && needs.initialize.outputs.SANITIZER == 'false' }}
134+
135+
- name: Python Test Steps (Sanitizer)
136+
run: make test-sanitizer
137+
shell: micromamba-shell {0}
138+
if: ${{ runner.os != 'Windows' && needs.initialize.outputs.SANITIZER == 'true' }}
99139

100140
- name: Python Test Steps ( Windows )
101141
run: make test

‎CMakeLists.txt‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,8 @@ option(CSP_MANYLINUX "Build for python's manylinux setup" OFF)
6767
option(CSP_USE_VCPKG "Build with vcpkg dependencies" ON)
6868
option(CSP_USE_CCACHE "Build with ccache caching" OFF)
6969
option(CSP_USE_LD_CLASSIC_MAC "On macOS, link with ld_classic" OFF)
70+
option(CSP_ENABLE_ASAN "Build with address sanitizer" OFF)
71+
option(CSP_ENABLE_UBSAN "Build with undefined behavior sanitizer" OFF)
7072

7173
# Extension options
7274
option(CSP_BUILD_KAFKA_ADAPTER "Build kafka adapter" ON)
@@ -213,6 +215,18 @@ else()
213215
endif()
214216
endif()
215217

218+
if(CSP_ENABLE_ASAN)
219+
message(STATUS "Enabling Address Sanitizer")
220+
add_compile_options(-fsanitize=address -fno-omit-frame-pointer)
221+
add_link_options(-fsanitize=address -fno-omit-frame-pointer)
222+
endif()
223+
224+
if(CSP_ENABLE_UBSAN)
225+
message(STATUS "Enabling Undefined Behavior Sanitizer")
226+
add_compile_options(-fsanitize=undefined -fno-omit-frame-pointer)
227+
add_link_options(-fsanitize=undefined -fno-omit-frame-pointer)
228+
endif()
229+
216230

217231
###################################################################################################################################################
218232
# Messages #

‎Makefile‎

Lines changed: 23 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,9 @@ EXTRA_ARGS :=
55
#########
66
.PHONY: requirements develop build build-debug build-conda install
77

8+
ASAN :=
9+
UBSAN :=
10+
811
requirements: ## install python dev and runtime dependencies
912
ifeq ($(OS),Windows_NT)
1013
Powershell.exe -executionpolicy bypass -noprofile .\ci\scripts\windows\make_requirements.ps1
@@ -18,13 +21,13 @@ develop: requirements ## install dependencies and build library
1821
python -m pip install -e .[develop]
1922

2023
build: ## build the library
21-
python setup.py build build_ext --inplace
24+
CSP_ENABLE_ASAN=$(ASAN) CSP_ENABLE_UBSAN=$(UBSAN) python setup.py build build_ext --inplace
2225

2326
build-debug: ## build the library ( DEBUG ) - May need a make clean when switching from regular build to build-debug and vice versa
24-
SKBUILD_CONFIGURE_OPTIONS="" DEBUG=1 python setup.py build build_ext --inplace
27+
CSP_ENABLE_ASAN=$(ASAN) CSP_ENABLE_UBSAN=$(UBSAN) SKBUILD_CONFIGURE_OPTIONS="" DEBUG=1 python setup.py build build_ext --inplace
2528

2629
build-conda: ## build the library in Conda
27-
python setup.py build build_ext --csp-no-vcpkg --inplace
30+
CSP_ENABLE_ASAN=$(ASAN) CSP_ENABLE_UBSAN=$(UBSAN) python setup.py build build_ext --csp-no-vcpkg --inplace
2831

2932
install: ## install library
3033
python -m pip install .
@@ -78,12 +81,26 @@ checks: check
7881
#########
7982
# TESTS #
8083
#########
81-
.PHONY: test-py test-cpp coverage-py test tests
84+
.PHONY: test-py test-cpp test-py-sanitizer coverage-py test test-sanitizer tests
8285

8386
TEST_ARGS :=
8487
test-py: ## Clean and Make unit tests
8588
python -m pytest -v csp/tests --junitxml=junit.xml $(TEST_ARGS)
8689

90+
test-py-sanitizer: ## Clean and Make unit tests with sanitizers enabled
91+
@if [ "$$(uname -s)" = "Darwin" ]; then \
92+
ASAN_OPTIONS=detect_leaks=0,detect_stack_use_after_return=true,use_odr_indicator=1,strict_init_order=true,strict_string_checks=true \
93+
DYLD_INSERT_LIBRARIES=$$($(CXX) -print-file-name=libclang_rt.asan_osx_dynamic.dylib) \
94+
python -m pytest -v csp/tests --junitxml=junit.xml $(TEST_ARGS); \
95+
elif [ "$$(uname -s)" = "Linux" ]; then \
96+
ASAN_OPTIONS=detect_leaks=0,detect_stack_use_after_return=true,use_odr_indicator=1,strict_init_order=true,strict_string_checks=true \
97+
LD_PRELOAD=$$($(CXX) -print-file-name=libasan.so) \
98+
python -m pytest -v csp/tests --junitxml=junit.xml $(TEST_ARGS); \
99+
else \
100+
echo "Unsupported platform: $$(uname -s)"; \
101+
exit 1; \
102+
fi
103+
87104
test-cpp: ## Make C++ unit tests
88105
ifneq ($(OS),Windows_NT)
89106
for f in ./csp/tests/bin/*; do $$f; done || (echo "TEST FAILED" && exit 1)
@@ -96,6 +113,8 @@ coverage-py:
96113

97114
test: test-cpp test-py ## run the tests
98115

116+
test-sanitizer: test-cpp test-py-sanitizer ## run the tests
117+
99118
# Alias
100119
tests: test
101120

‎cpp/cmake/modules/Findcsp_autogen.cmake‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,32 @@ function(csp_autogen MODULE_NAME DEST_FILENAME HEADER_NAME_OUTVAR SOURCE_NAME_OU
2626
set(CSP_AUTOGEN_PYTHONPATH ${PROJECT_BINARY_DIR}/lib:${CMAKE_SOURCE_DIR}:$$PYTHONPATH)
2727
endif()
2828

29+
if(CSP_ENABLE_ASAN)
30+
if(CMAKE_CXX_COMPILER_ID MATCHES "Clang")
31+
# Clang - use DYLD_INSERT_LIBRARIES
32+
execute_process(
33+
COMMAND ${CMAKE_CXX_COMPILER} -print-file-name=libclang_rt.asan_osx_dynamic.dylib
34+
OUTPUT_VARIABLE ASAN_LIB_PATH
35+
OUTPUT_STRIP_TRAILING_WHITESPACE
36+
)
37+
set(PRELOAD_CMD "DYLD_INSERT_LIBRARIES=${ASAN_LIB_PATH}")
38+
elseif(CMAKE_CXX_COMPILER_ID MATCHES "GNU")
39+
# GCC - use LD_PRELOAD
40+
execute_process(
41+
COMMAND ${CMAKE_CXX_COMPILER} -print-file-name=libasan.so
42+
OUTPUT_VARIABLE ASAN_LIB_PATH
43+
OUTPUT_STRIP_TRAILING_WHITESPACE
44+
)
45+
set(PRELOAD_CMD "LD_PRELOAD=${ASAN_LIB_PATH}")
46+
endif()
47+
# Turn off leak checks as we are using PyMalloc when we run autogen
48+
set(ASAN_PRELOAD_CMD "ASAN_OPTIONS=detect_leaks=0" ${PRELOAD_CMD})
49+
else()
50+
set(ASAN_PRELOAD_CMD "")
51+
endif()
52+
2953
add_custom_command(OUTPUT "${CSP_AUTOGEN_CPP_OUT}" "${CSP_AUTOGEN_H_OUT}"
30-
COMMAND ${CMAKE_COMMAND} -E env "PYTHONPATH=${CSP_AUTOGEN_PYTHONPATH}" ${Python_EXECUTABLE} ${CSP_AUTOGEN_MODULE_PATH} -m ${MODULE_NAME} -d ${CSP_AUTOGEN_DESTINATION_FOLDER} -o ${DEST_FILENAME} ${CSP_AUTOGEN_EXTRA_ARGS}
54+
COMMAND ${CMAKE_COMMAND} -E env "PYTHONPATH=${CSP_AUTOGEN_PYTHONPATH}" ${ASAN_PRELOAD_CMD} ${Python_EXECUTABLE} ${CSP_AUTOGEN_MODULE_PATH} -m ${MODULE_NAME} -d ${CSP_AUTOGEN_DESTINATION_FOLDER} -o ${DEST_FILENAME} ${CSP_AUTOGEN_EXTRA_ARGS}
3155
COMMENT "generating csp c++ types from module ${MODULE_NAME}"
3256
DEPENDS mkdir_autogen_${MODULE_TARGETNAME}
3357
${CSP_AUTOGEN_MODULE_PATH}

‎cpp/csp/core/DynamicBitSet.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -141,7 +141,8 @@ class DynamicBitSet
141141
{
142142
node_type * old = m_nodes;
143143
m_nodes = new node_type[ newNodes ];
144-
memcpy( m_nodes, old, m_numNodes * sizeof( node_type ) );
144+
if( likely( m_numNodes > 0 ) )
145+
memcpy( m_nodes, old, m_numNodes * sizeof( node_type ) );
145146
memset( m_nodes + m_numNodes, 0, ( newNodes - m_numNodes ) * sizeof( node_type ) );
146147

147148
m_numNodes = newNodes;

‎cpp/csp/core/Exception.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ static void printBacktrace( char ** messages, int size, std::ostream & dest )
3434
{
3535
char *begin_name = 0, *begin_offset = 0;
3636
char tmp[1024];
37-
strncpy( tmp, messages[i], sizeof(tmp) );
37+
strncpy( tmp, messages[i], sizeof(tmp) - 1 );
3838
tmp[ sizeof( tmp ) - 1 ] = 0;
3939

4040
// find parentheses and +address offset surrounding the mangled name:

‎cpp/csp/python/cspbaselibimpl.cpp‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -274,6 +274,14 @@ DECLARE_CPPNODE( exprtk_impl )
274274
csp.make_passive( inputs );
275275
}
276276

277+
virtual ~exprtk_impl()
278+
{
279+
// Need to release the expression before clearing values/symbol table
280+
// https://github.com/ArashPartow/exprtk/blob/cc1b800c2bd1ac3ac260478c915d2aec6f4eb41c/readme.txt#L909
281+
s_expr.release();
282+
s_valuesContainer.clear();
283+
}
284+
277285
INVOKE()
278286
{
279287
if( use_trigger )

‎cpp/csp/python/npstatsimpl.cpp‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1301,12 +1301,12 @@ DECLARE_CPPNODE( _np_arg_min_max )
13011301
PyArray_Descr *descr;
13021302
PyArray_DescrConverter( date_type, &descr );
13031303
Py_XDECREF( date_type );
1304-
DateTime * values = new DateTime[s_elem.size()];
1304+
1305+
PyObject * out = PyArray_NewFromDescr( &PyArray_Type, descr, s_shp.m_dims.size(), &s_shp.m_dims[0], NULL, NULL, 0, NULL );
1306+
DateTime * values = static_cast<DateTime *>( PyArray_DATA( ( PyArrayObject * )out ) );
13051307
for( size_t i = 0; i < s_elem.size(); ++i )
13061308
values[i] = s_elem[i].compute_dt();
1307-
1308-
PyObject * out = PyArray_NewFromDescr( &PyArray_Type, descr, s_shp.m_dims.size(), &s_shp.m_dims[0], NULL, values, 0, NULL );
1309-
PyArray_ENABLEFLAGS( ( PyArrayObject * ) out, NPY_ARRAY_OWNDATA );
1309+
13101310
RETURN( PyObjectPtr::own( out ) );
13111311
}
13121312
}

‎cpp/tests/engine/test_tick_buffer.cpp‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,4 +98,8 @@ TEST( TickBufferTest, test_flatten )
9898
ASSERT_EQ( values_wrap[ i ], i + 3 );
9999
ASSERT_EQ( values_nowrap[ i ], i );
100100
}
101+
102+
free( values_wrap );
103+
free( values_nowrap );
104+
free( values_single );
101105
}

0 commit comments

Comments
 (0)