Skip to content

Commit f2d11f0

Browse files
timkpainetimkpaine
authored
Merge branch 'main' into tkp/rm2
2 parents d93674d + 0f27b36 commit f2d11f0

109 files changed

Lines changed: 1612 additions & 17538 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/build.yml‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -196,7 +196,7 @@ jobs:
196196
- ubuntu-24.04 # https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2404-Readme.md
197197
- macos-13 # https://github.com/actions/runner-images/blob/main/images/macos/macos-13-Readme.md
198198
- macos-14 # https://github.com/actions/runner-images/blob/main/images/macos/macos-14-arm64-Readme.md
199-
- windows-2019 # https://github.com/actions/runner-images/blob/main/images/windows/Windows2019-Readme.md
199+
- windows-2022 # https://github.com/actions/runner-images/blob/main/images/windows/Windows2022-Readme.md
200200
python-version:
201201
- "3.9"
202202
- "3.10"
@@ -250,15 +250,15 @@ jobs:
250250

251251
# Avoid extra resources for windows build
252252
- is-full-run: false
253-
os: windows-2019
253+
os: windows-2022
254254
python-version: "3.9"
255255

256256
- is-full-run: false
257-
os: windows-2019
257+
os: windows-2022
258258
python-version: "3.10"
259259

260260
- is-full-run: false
261-
os: windows-2019
261+
os: windows-2022
262262
python-version: "3.11"
263263

264264
# avoid unnecessary use of mac resources
@@ -328,14 +328,14 @@ jobs:
328328

329329
##########
330330
# Windows
331-
- name: Python Build Steps (Windows 2019)
331+
- name: Python Build Steps (Windows 2022)
332332
run: make dist-py-cibw
333333
env:
334334
CIBW_BUILD: "${{ matrix.cibuildwheel }}-win_amd64"
335-
CSP_GENERATOR: "Visual Studio 16 2019"
335+
CSP_GENERATOR: "Visual Studio 17 2022"
336336
VCPKG_DEFAULT_BINARY_CACHE: C:\\Users\\runneradmin\\AppData\\Local\\vcpkg_cache
337337
VCPKG_DOWNLOADS: C:\\Users\\runneradmin\\AppData\\Local\\vcpkg_download_cache
338-
if: ${{ matrix.os == 'windows-2019' }}
338+
if: ${{ matrix.os == 'windows-2022' }}
339339

340340
##########
341341
# Common
@@ -433,7 +433,7 @@ jobs:
433433
- ubuntu-24.04
434434
- macos-13
435435
- macos-14
436-
- windows-2019
436+
- windows-2022
437437
python-version:
438438
- 3.9
439439
- "3.10"
@@ -451,15 +451,15 @@ jobs:
451451

452452
# Avoid extra resources for windows build
453453
- is-full-run: false
454-
os: windows-2019
454+
os: windows-2022
455455
python-version: "3.9"
456456

457457
- is-full-run: false
458-
os: windows-2019
458+
os: windows-2022
459459
python-version: "3.10"
460460

461461
- is-full-run: false
462-
os: windows-2019
462+
os: windows-2022
463463
python-version: "3.11"
464464

465465
# avoid unnecessary use of mac resources
@@ -650,7 +650,7 @@ jobs:
650650
strategy:
651651
matrix:
652652
os:
653-
- ubuntu-20.04
653+
- ubuntu-24.04
654654
python-version:
655655
- 3.9
656656
package:

‎.github/workflows/conda.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ jobs:
3838
- ubuntu-24.04
3939
- macos-14
4040
- macos-13
41-
- windows-2019
41+
- windows-2022
4242
runs-on: ${{ matrix.os }}
4343
steps:
4444
- name: Checkout
@@ -86,7 +86,7 @@ jobs:
8686

8787
- name: Python Build Steps ( Windows )
8888
env:
89-
CSP_GENERATOR: "Visual Studio 16 2019"
89+
CSP_GENERATOR: "Visual Studio 17 2022"
9090
run: make build-conda
9191
shell: cmd /C call {0}
9292
if: ${{ runner.os == 'Windows' }}

‎CMakeLists.txt‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
# Project Configuration #
33
#########################
44
cmake_minimum_required(VERSION 3.20.0)
5-
project(csp VERSION "0.0.11")
5+
project(csp VERSION "0.10.1")
66
set(CMAKE_CXX_STANDARD 20)
77

88
###################################################################################################################################################
@@ -70,6 +70,7 @@ option(CSP_USE_LD_CLASSIC_MAC "On macOS, link with ld_classic" OFF)
7070

7171
# Extension options
7272
option(CSP_BUILD_KAFKA_ADAPTER "Build kafka adapter" ON)
73+
option(CSP_BUILD_ARROW_ADAPTER "Build arrow adapter" ON)
7374
option(CSP_BUILD_PARQUET_ADAPTER "Build parquet adapter" ON)
7475
option(CSP_BUILD_WS_CLIENT_ADAPTER "Build ws client adapter" ON)
7576

@@ -238,6 +239,10 @@ find_package(csp_autogen REQUIRED)
238239
find_package(DepsBase REQUIRED)
239240

240241
# Adapter dependencies
242+
if(CSP_BUILD_ARROW_ADAPTER)
243+
find_package(DepsArrowAdapter REQUIRED)
244+
endif()
245+
241246
if(CSP_BUILD_KAFKA_ADAPTER)
242247
find_package(DepsKafkaAdapter REQUIRED)
243248
endif()

‎Makefile‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -194,16 +194,17 @@ dependencies-mac: ## install dependencies for mac
194194
brew unlink bison flex && brew link --force bison flex
195195

196196
dependencies-debian: ## install dependencies for linux
197-
apt-get install -y autoconf autoconf-archive automake bison cmake curl flex libtool ninja-build pkg-config tar unzip zip
197+
apt-get install -y autoconf autoconf-archive automake bison cmake curl flex libtool ninja-build pkg-config tar unzip
198198

199199
dependencies-fedora: ## install dependencies for linux
200-
yum install -y autoconf autoconf-archive automake bison ccache cmake curl flex libtool perl-IPC-Cmd pkg-config tar unzip zip
200+
yum install -y autoconf autoconf-archive automake bison ccache cmake curl flex libtool perl-IPC-Cmd pkg-config tar unzip
201201

202202
dependencies-vcpkg: ## install dependencies via vcpkg
203203
cd vcpkg && ./bootstrap-vcpkg.sh && ./vcpkg install
204204

205205
dependencies-win: ## install dependencies via windows
206-
choco install cmake curl winflexbison ninja unzip zip --no-progress -y
206+
choco install cmake --version=3.31.6 --allow-downgrade
207+
choco install curl winflexbison ninja unzip --no-progress -y
207208

208209
############################################################################################
209210
# Thanks to Francoise at marmelab.com for this

‎conda/dev-environment-unix.yml‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ channels:
33
- conda-forge
44
- nodefaults
55
dependencies:
6+
- astor
67
- bison
78
- brotli
89
- bump-my-version
@@ -16,7 +17,7 @@ dependencies:
1617
- graphviz
1718
- gtest
1819
- httpx>=0.20,<1
19-
- libarrow=16
20+
- libarrow<20
2021
- libboost>=1.80.0
2122
- libboost-headers>=1.80.0
2223
- librdkafka
@@ -27,11 +28,12 @@ dependencies:
2728
- mdformat-tables>=1,<1.1
2829
- ninja
2930
- numpy<2
30-
- pandas
31+
- pandas<2.3 # [py<310]
32+
- pandas # [py>=310]
3133
- pillow
3234
- polars
3335
- psutil
34-
- pyarrow>=15,<19
36+
- pyarrow>=15,<20
3537
- pydantic>=2
3638
- pytest
3739
- pytest-asyncio

‎conda/dev-environment-win.yml‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ channels:
44
- nodefaults
55
dependencies:
66
# - bison # not available on windows
7+
- astor
78
- brotli
89
- bump-my-version
910
- cmake
@@ -16,7 +17,7 @@ dependencies:
1617
- graphviz
1718
- gtest
1819
- httpx>=0.20,<1
19-
- libarrow=16
20+
- libarrow<20
2021
- libboost>=1.80.0
2122
- libboost-headers>=1.80.0
2223
- librdkafka
@@ -27,11 +28,12 @@ dependencies:
2728
- mdformat-tables>=1,<1.1
2829
- ninja
2930
- numpy<2
30-
- pandas
31+
- pandas<2.3 # [py<310]
32+
- pandas # [py>=310]
3133
- pillow
3234
- polars
3335
- psutil
34-
- pyarrow>=7,<19
36+
- pyarrow>=15,<20
3537
- pydantic>=2
3638
- pytest
3739
- pytest-asyncio

‎cpp/cmake/modules/FindDepsArrowAdapter.cmake‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
cmake_minimum_required(VERSION 3.7.2)
2+
3+
# ARROW
4+
find_package(Arrow REQUIRED)
5+
include_directories(${ARROW_INCLUDE_DIR})

‎cpp/csp/adapters/parquet/ParquetFileWriterWrapper.cpp‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,9 @@ void ParquetFileWriterWrapper::openImpl( const std::string &fileName, const std:
2121
::parquet::ArrowWriterProperties::Builder arrowBuilder;
2222
arrowBuilder.store_schema();
2323

24-
STATUS_OK_OR_THROW_RUNTIME(
25-
::parquet::arrow::FileWriter::Open( *getSchema(), arrow::default_memory_pool(), m_outputStream, builder.build(), arrowBuilder.build(),
26-
&m_fileWriter ),
27-
"Failed to open parquet file writer" );
24+
auto res = ::parquet::arrow::FileWriter::Open( *getSchema(), arrow::default_memory_pool(), m_outputStream, builder.build(), arrowBuilder.build() );
25+
STATUS_OK_OR_THROW_RUNTIME( res.status(), "Failed to open parquet file writer" );
26+
m_fileWriter = res.MoveValueUnsafe();
2827
}
2928

3029
void ParquetFileWriterWrapper::close()

‎cpp/csp/adapters/utils/JSONMessageStructConverter.cpp‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ using SupportedCspTypeSwitch = PartialSwitchCspType<csp::CspType::Type::BOOL,
1111
csp::CspType::Type::INT32,
1212
csp::CspType::Type::INT64,
1313
csp::CspType::Type::DOUBLE,
14+
csp::CspType::Type::DATE,
1415
csp::CspType::Type::DATETIME,
1516
csp::CspType::Type::STRING,
1617
csp::CspType::Type::ENUM,
@@ -73,6 +74,15 @@ std::string JSONMessageStructConverter::convertJSON( const char * fieldname, con
7374
CSP_THROW( TypeError, "expected STRING type for json field " << fieldname );
7475
}
7576

77+
template<>
78+
Date JSONMessageStructConverter::convertJSON( const char * fieldname, const rapidjson::Value & jValue, Date * )
79+
{
80+
if( jValue.IsString() )
81+
return Date::fromYYYYMMDD( jValue.GetString() );
82+
else
83+
CSP_THROW( TypeError, "expected STRING type for json DATE field " << fieldname );
84+
}
85+
7686
template<>
7787
DateTime JSONMessageStructConverter::convertJSON( const char * fieldname, const rapidjson::Value & jValue, DateTime * )
7888
{

‎cpp/csp/adapters/utils/JSONMessageWriter.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ class JSONMessageWriter : public MessageWriter
2020
csp::CspType::Type::INT32,
2121
csp::CspType::Type::INT64,
2222
csp::CspType::Type::DOUBLE,
23+
csp::CspType::Type::DATE,
2324
csp::CspType::Type::DATETIME,
2425
csp::CspType::Type::ENUM,
2526
csp::CspType::Type::STRING,

0 commit comments

Comments
 (0)