Skip to content

Commit bc5d73e

Browse files
langmmlangmm
authored
Merge pull request #8 from cropsinsilico/bug_fixes
Bug fixes
2 parents cec3310 + d116742 commit bc5d73e

8 files changed

Lines changed: 133 additions & 29 deletions

File tree

‎.github/workflows/test-suite.yml‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@ jobs:
7878
run: |
7979
mkdir build
8080
cd build
81-
cmake .. -DCMAKE_BUILD_TYPE=${{ matrix.configuration }} -DDISABLE_YGGDRASIL_RAPIDJSON=ON
81+
cmake .. -DCMAKE_BUILD_TYPE=${{ matrix.configuration }} -DDISABLE_YGGDRASIL_RAPIDJSON=ON -DYGGDRASIL_RAPIDJSON_ENABLE_INSTRUMENTATION_OPT=ON
8282
- name: (WINDOWS) Configure w/ yggdrasil
8383
if: matrix.os == 'windows-latest' && matrix.with-yggdrasil
8484
run: |
@@ -90,7 +90,7 @@ jobs:
9090
run: |
9191
mkdir build
9292
cd build
93-
cmake .. -DCMAKE_BUILD_TYPE=${{ matrix.configuration }} -DPython3_ROOT_DIR=${pythonLocation} -DYGGDRASIL_RAPIDJSON_ENABLE_INSTRUMENTATION_OPT=OFF -DYGGDRASIL_RAPIDJSON_SCHEMA_TESTS=ON -DYGGDRASIL_RAPIDJSON_BUILD_THIRDPARTY_GTEST=ON
93+
cmake .. -DCMAKE_BUILD_TYPE=${{ matrix.configuration }} -DPython3_ROOT_DIR=${pythonLocation} -DYGGDRASIL_RAPIDJSON_SCHEMA_TESTS=ON -DYGGDRASIL_RAPIDJSON_BUILD_THIRDPARTY_GTEST=ON
9494
- name: Build
9595
run: |
9696
cd build
@@ -155,19 +155,19 @@ jobs:
155155
run: |
156156
mkdir build
157157
cd build
158-
cmake .. -DCMAKE_BUILD_TYPE=${{ matrix.configuration }} -DDISABLE_YGGDRASIL_RAPIDJSON=ON
158+
cmake .. -DCMAKE_BUILD_TYPE=${{ matrix.configuration }} -DDISABLE_YGGDRASIL_RAPIDJSON=ON -DYGGDRASIL_RAPIDJSON_ENABLE_INSTRUMENTATION_OPT=ON
159159
- name: (WINDOWS) Configure w/ yggdrasil
160160
if: matrix.os == 'windows-latest' && matrix.with-yggdrasil
161161
run: |
162162
mkdir build
163163
cd build
164-
cmake .. -DCMAKE_BUILD_TYPE=${{ matrix.configuration }} -DYGGDRASIL_RAPIDJSON_ENABLE_COVERAGE=ON
164+
cmake .. -DCMAKE_BUILD_TYPE=${{ matrix.configuration }} -DYGGDRASIL_RAPIDJSON_ENABLE_COVERAGE=ON -DYGGDRASIL_RAPIDJSON_ENABLE_INSTRUMENTATION_OPT=ON
165165
- name: (UNIX) Configure w/ yggdrasil
166166
if: matrix.os != 'windows-latest' && matrix.with-yggdrasil
167167
run: |
168168
mkdir build
169169
cd build
170-
cmake .. -DCMAKE_BUILD_TYPE=${{ matrix.configuration }} -DYGGDRASIL_RAPIDJSON_ENABLE_COVERAGE=ON -DYGGDRASIL_RAPIDJSON_SCHEMA_TESTS=ON -DYGGDRASIL_RAPIDJSON_CREATE_METASCHEMA=ON
170+
cmake .. -DCMAKE_BUILD_TYPE=${{ matrix.configuration }} -DYGGDRASIL_RAPIDJSON_ENABLE_COVERAGE=ON -DYGGDRASIL_RAPIDJSON_SCHEMA_TESTS=ON -DYGGDRASIL_RAPIDJSON_CREATE_METASCHEMA=ON -DYGGDRASIL_RAPIDJSON_ENABLE_INSTRUMENTATION_OPT=ON
171171
- name: Build
172172
run: |
173173
cd build

‎CHANGELOG.md‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,14 @@ All notable changes to this project will be documented in this file.
33
This project adheres to [Semantic Versioning](http://semver.org/).
44
Because YggdrasilRapidJSON continues to merge updates from RapidJSON, the RapidJSON version is used with the addition of a EXTEN version after the PATCH version (i.e. MAJOR.MINOR.PATCH.EXTEN) to indicate the YggdrasilRapidJSON version. The EXTEN version will restart at 0 when/if RapidJSON increments their version.
55

6-
## X.X.X.X - XXXX-XX-XX
6+
## 1.1.0.5 - 2026-07-08
7+
8+
### Bug fixes
9+
* Fix Python3 components when CMakeLists.txt is included directly from a Python extension
10+
* Fix SetPythonObjectRaw for Pandas 3.0.0 (String now stored as "str" instead of NPY_OBJECT which has different representations on different operating systems by default)
11+
12+
### Features
13+
* NPY_TYPE2STRING utility for getting a numpy type enumerator name during debugging
714

815
## 1.1.0.4 - 2026-07-06
916

‎CMakeLists.txt‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -134,11 +134,15 @@ if(NOT (YGGDRASIL_DISABLE_PYTHON_C_API OR DISABLE_YGGDRASIL_RAPIDJSON))
134134
if(Python3_ROOT_DIR)
135135
message(STATUS "Python root directory is ${Python3_ROOT_DIR}")
136136
endif()
137-
find_package(Python3 COMPONENTS Interpreter Development NumPy REQUIRED)
138-
139-
# include_directories(
140-
# SYSTEM ${Python3_INCLUDE_DIRS} ${Python3_NumPy_INCLUDE_DIRS}
141-
# )
137+
if(YGGDRASIL_RAPIDJSON_PYTHON_WRAPPER)
138+
find_package(
139+
Python3 COMPONENTS Interpreter Development.Module NumPy REQUIRED
140+
)
141+
else()
142+
find_package(
143+
Python3 COMPONENTS Interpreter Development NumPy REQUIRED
144+
)
145+
endif()
142146

143147
# Set environment variables required for embedding Python
144148
set(YGGDRASIL_RAPIDJSON_PYTHON_ENV "DATADIR=${CMAKE_SOURCE_DIR}/test;YGG_PYTHON_EXEC=${Python3_EXECUTABLE}")

‎YggdrasilRapidJSONMacros.cmake‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ macro(yggdrasil_rapidjson_options_create)
2424
option(YGGDRASIL_RAPIDJSON_PYTHON_WRAPPER "Set flags to use YggdrasilRapidJSON as part of a Python C extension" OFF)
2525
option(YGGDRASIL_RAPIDJSON_BUILD_ASAN "Build yggdrasil_rapidjson with address sanitizer (gcc/clang)" OFF)
2626
option(YGGDRASIL_RAPIDJSON_BUILD_UBSAN "Build yggdrasil_rapidjson with undefined behavior sanitizer (gcc/clang)" OFF)
27-
option(YGGDRASIL_RAPIDJSON_ENABLE_INSTRUMENTATION_OPT "Build yggdrasil_rapidjson with -march or -mcpu options" ON)
27+
option(YGGDRASIL_RAPIDJSON_ENABLE_INSTRUMENTATION_OPT "Build yggdrasil_rapidjson with -march or -mcpu options" OFF)
2828
endmacro()
2929

3030
function(yggdrasil_rapidjson_config_vars PREFIX)

‎conda.recipe/recipe.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
context:
2-
version: "1.1.0.4"
2+
version: "1.1.0.5"
33

44
package:
55
name: yggdrasil-rapidjson

‎include/yggdrasil_rapidjson/document.h‎

Lines changed: 53 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -5047,6 +5047,10 @@ class GenericValue {
50475047
}
50485048
} else {
50495049
ValueType pyField(field, allocator);
5050+
if (pyField.IsNull()) {
5051+
out = false;
5052+
goto cleanup;
5053+
}
50505054
if (!skipTitle) {
50515055
ValueType field_name(kw_keyS,
50525056
static_cast<SizeType>(kw_keyS_len),
@@ -5105,12 +5109,14 @@ class GenericValue {
51055109
AddSchemaMember(GetEncodingString(), encoding);
51065110
out = true;
51075111
goto cleanup;
5108-
} else if (PyObject_IsInstanceString(x, "pandas.core.frame.DataFrame")) {
5112+
} else if (PyObject_IsInstanceString(x, "pandas.core.frame.DataFrame") ||
5113+
PyObject_IsInstanceString(x, "pandas.DataFrame")) {
51095114
bool error = false;
51105115
PyObject *column_dtypes = NULL, *columns = NULL,
51115116
*itype = NULL, *skipTitleObject = NULL, *vtype = NULL, *vv = NULL,
51125117
*method = NULL, *args = NULL, *kwargs = NULL, *arr = NULL,
5113-
*new_itype = NULL, *pandas_api = NULL, *is_integer_dtype = NULL;
5118+
*new_itype = NULL, *pandas_api = NULL, *is_integer_dtype = NULL,
5119+
*is_string_dtype = NULL, *columnIsString = NULL;
51145120
std::string new_itype_str;
51155121
column_dtypes = PyDict_New();
51165122
if (column_dtypes == NULL) {
@@ -5128,6 +5134,7 @@ class GenericValue {
51285134
goto cleanup_array;
51295135
}
51305136
is_integer_dtype = PyObject_GetAttrString(pandas_api, "is_integer_dtype");
5137+
is_string_dtype = PyObject_GetAttrString(pandas_api, "is_string_dtype");
51315138
Py_CLEAR(pandas_api);
51325139
if (is_integer_dtype == NULL) {
51335140
error = true;
@@ -5159,38 +5166,67 @@ class GenericValue {
51595166
error = true;
51605167
goto cleanup_array;
51615168
}
5162-
if (((PyArray_Descr*)itype)->type_num == NPY_OBJECT) {
5169+
bool is_in_bytes = false;
5170+
if ((PyObject_IsInstanceString(itype, "pandas.StringDtype")) &&
5171+
is_string_dtype != NULL) {
5172+
is_in_bytes = true;
5173+
}
5174+
if (((PyArray_Descr*)itype)->type_num == NPY_OBJECT ||
5175+
is_in_bytes) {
51635176
ival = PyObject_GetItem(x, ikey);
51645177
if (ival == NULL) {
51655178
error = true;
51665179
goto cleanup_array;
51675180
}
5181+
if (is_in_bytes) {
5182+
columnIsString = PyObject_CallFunction(is_string_dtype, "(O)", ival);
5183+
is_in_bytes = (columnIsString == Py_True);
5184+
Py_CLEAR(columnIsString);
5185+
}
5186+
}
5187+
// std::cerr << i << ": " << NPY_TYPE2STRING(((PyArray_Descr*)itype)->type_num) << ", is_in_bytes = " << is_in_bytes << std::endl;
5188+
if (((PyArray_Descr*)itype)->type_num == NPY_OBJECT ||
5189+
is_in_bytes) {
51685190
Py_ssize_t max_len = 0;
51695191
if (PyObject_Size(ival) == 0) {
5170-
// For an empty field with OBJECT type, information about
5171-
// which string type (unicode or bytes) cannot be determined
5172-
// so we just assume bytes
5173-
vtype = (PyObject*)(&PyBytes_Type);
5174-
Py_INCREF(vtype);
5192+
if (is_in_bytes) {
5193+
vtype = (PyObject*)(&PyUnicode_Type);
5194+
Py_INCREF(vtype);
5195+
} else {
5196+
// For an empty field with OBJECT type, information about
5197+
// which string type (unicode or bytes) cannot be determined
5198+
// so we just assume bytes
5199+
vtype = (PyObject*)(&PyBytes_Type);
5200+
Py_INCREF(vtype);
5201+
}
51755202
} else {
51765203
for (Py_ssize_t j = 0; j < PyObject_Size(ival); j++) {
51775204
vv = PySequence_GetItem(ival, j);
51785205
if (vv == NULL) {
51795206
error = true;
51805207
goto cleanup_array;
51815208
}
5182-
if (vtype == NULL) {
5209+
if (is_in_bytes) {
5210+
if (vtype == NULL) {
5211+
vtype = (PyObject*)(&PyUnicode_Type);
5212+
Py_INCREF(vtype);
5213+
}
5214+
} else if (vtype == NULL) {
51835215
vtype = PyObject_Type(vv);
5184-
std::string vtype_str;
5185-
PyObject* vtype_repr = PyObject_Str(vtype);
5186-
vtype_str.assign(PyUnicode_AsUTF8(vtype_repr));
5187-
Py_CLEAR(vtype_repr);
5216+
// This is just for debugging
5217+
// std::string vtype_str;
5218+
// PyObject* vtype_repr = PyObject_Str(vtype);
5219+
// vtype_str.assign(PyUnicode_AsUTF8(vtype_repr));
5220+
// Py_CLEAR(vtype_repr);
5221+
// std::cerr << i << ": " << vtype_str << std::endl;
51885222
} else if (!PyObject_IsInstance(vv, vtype)) {
51895223
error = true;
51905224
goto cleanup_array;
51915225
}
51925226
Py_ssize_t new_len = 0;
5193-
if (vtype == (PyObject*)(&PyUnicode_Type)) {
5227+
if (is_in_bytes) {
5228+
new_len = PyObject_Size(vv);
5229+
} else if (vtype == (PyObject*)(&PyUnicode_Type)) {
51945230
new_len = PyUnicode_GET_LENGTH(vv);
51955231
} else {
51965232
new_len = PyObject_Size(vv);
@@ -5223,8 +5259,8 @@ class GenericValue {
52235259
}
52245260
Py_CLEAR(new_itype);
52255261
Py_CLEAR(vtype);
5226-
Py_CLEAR(ival);
52275262
}
5263+
Py_CLEAR(ival);
52285264
Py_CLEAR(ikey);
52295265
Py_CLEAR(itype);
52305266
}
@@ -5272,6 +5308,7 @@ class GenericValue {
52725308
Py_CLEAR(ival);
52735309
Py_CLEAR(pandas_api);
52745310
Py_CLEAR(is_integer_dtype);
5311+
Py_CLEAR(is_string_dtype);
52755312
out = (!error);
52765313
goto cleanup;
52775314
#endif // YGGDRASIL_RAPIDJSON_DONT_IMPORT_NUMPY
@@ -5629,6 +5666,7 @@ class GenericValue {
56295666
}
56305667
if (!(PyDataType_ISNUMBER(desc))) {
56315668
std::cerr << "NumpyType2SubType: Non-number numpy element (itemsize = " << itemsize << ")" << std::endl;
5669+
std::cerr << "NumpyType2SubType: " << NPY_TYPE2STRING(desc->type_num) << std::endl;
56325670
return false;
56335671
}
56345672
precision = (SizeType)(PyDataType_ELSIZE(desc));

‎include/yggdrasil_rapidjson/pyrj.h‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1028,6 +1028,61 @@ bool PyObject_IsInstanceString(PyObject* x, std::string class_name) {
10281028
out = (check == result);
10291029
PYTHON_ERROR_CLEANUP_NOTHROW_CLEAR_(inst_class, inst_class_str);
10301030
}
1031+
#ifdef YGGDRASIL_RAPIDJSON_DONT_IMPORT_NUMPY
1032+
inline
1033+
std::string NPY_TYPE2STRING(int) {
1034+
return "";
1035+
}
1036+
#else // YGGDRASIL_RAPIDJSON_DONT_IMPORT_NUMPY
1037+
inline
1038+
std::string NPY_TYPE2STRING(int x) {
1039+
YGGDRASIL_RAPIDJSON_ASSERT(isPythonInitialized());
1040+
if (!isPythonInitialized())
1041+
return "";
1042+
#define CASE_TYPE(name) \
1043+
case (name): { \
1044+
return #name; \
1045+
}
1046+
switch(x) {
1047+
CASE_TYPE(NPY_BOOL);
1048+
CASE_TYPE(NPY_BYTE);
1049+
CASE_TYPE(NPY_UBYTE);
1050+
CASE_TYPE(NPY_SHORT);
1051+
CASE_TYPE(NPY_USHORT);
1052+
CASE_TYPE(NPY_INT);
1053+
CASE_TYPE(NPY_UINT);
1054+
CASE_TYPE(NPY_LONG);
1055+
CASE_TYPE(NPY_ULONG);
1056+
CASE_TYPE(NPY_LONGLONG);
1057+
CASE_TYPE(NPY_ULONGLONG);
1058+
CASE_TYPE(NPY_FLOAT);
1059+
CASE_TYPE(NPY_DOUBLE);
1060+
CASE_TYPE(NPY_LONGDOUBLE);
1061+
CASE_TYPE(NPY_CFLOAT); // Complex
1062+
CASE_TYPE(NPY_CDOUBLE);
1063+
CASE_TYPE(NPY_CLONGDOUBLE);
1064+
CASE_TYPE(NPY_OBJECT);
1065+
CASE_TYPE(NPY_STRING);
1066+
CASE_TYPE(NPY_UNICODE);
1067+
CASE_TYPE(NPY_VOID);
1068+
CASE_TYPE(NPY_DATETIME);
1069+
CASE_TYPE(NPY_TIMEDELTA);
1070+
CASE_TYPE(NPY_HALF);
1071+
CASE_TYPE(NPY_CHAR); // Deprecated, but check anyway
1072+
// CASE_TYPE(NPY_NTYPES_LEGACY);
1073+
CASE_TYPE(NPY_NOTYPE);
1074+
CASE_TYPE(NPY_USERDEF);
1075+
CASE_TYPE(NPY_VSTRING);
1076+
default: {
1077+
if (x >= NPY_USERDEF) {
1078+
return "user-defined";
1079+
}
1080+
return std::to_string(x);
1081+
}
1082+
}
1083+
#undef CASE_TYPE
1084+
}
1085+
#endif // YGGDRASIL_RAPIDJSON_DONT_IMPORT_NUMPY
10311086
inline
10321087
bool IsStructuredArray(PyObject* x) {
10331088
YGGDRASIL_RAPIDJSON_ASSERT(isPythonInitialized());

‎include/yggdrasil_rapidjson/yggdrasil_rapidjson_version.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@
3636
#define YGGDRASIL_RAPIDJSON_MAJOR_VERSION 1
3737
#define YGGDRASIL_RAPIDJSON_MINOR_VERSION 1
3838
#define YGGDRASIL_RAPIDJSON_PATCH_VERSION 0
39-
#define YGGDRASIL_RAPIDJSON_EXTEN_VERSION 4
39+
#define YGGDRASIL_RAPIDJSON_EXTEN_VERSION 5
4040
#define YGGDRASIL_RAPIDJSON_VERSION_STRING \
4141
YGGDRASIL_RAPIDJSON_STRINGIFY(YGGDRASIL_RAPIDJSON_MAJOR_VERSION.YGGDRASIL_RAPIDJSON_MINOR_VERSION.YGGDRASIL_RAPIDJSON_PATCH_VERSION.YGGDRASIL_RAPIDJSON_EXTEN_VERSION)
4242

0 commit comments

Comments
 (0)