Fixed carquet repetition option bugs - v0.1.3

Author: Vitruves

CMakeLists.txt

@@ -1,5 +1,5 @@
 cmake_minimum_required(VERSION 3.16)
-project(carquet VERSION 0.1.2 LANGUAGES C)
+project(carquet VERSION 0.1.3 LANGUAGES C)
 
 set(CMAKE_C_STANDARD 11)
 set(CMAKE_C_STANDARD_REQUIRED ON)

examples/nullable_columns.c

@@ -188,16 +188,18 @@ static int write_nullable_data(const char* filename, carquet_schema_t* schema) {
     if (status != CARQUET_OK) goto error;
 
     /* Write optional columns WITH definition levels */
-    /* Note: Only non-null values are in the values array! */
-    status = carquet_writer_write_batch(writer, 2, ages, age_value_count,
+    /* Note: Only non-null values are in the values array, but num_values
+     * is the total number of logical rows (including NULLs). The def_levels
+     * array has one entry per logical row indicating null vs present. */
+    status = carquet_writer_write_batch(writer, 2, ages, NUM_ROWS,
                                         age_def_levels, NULL);
     if (status != CARQUET_OK) goto error;
 
-    status = carquet_writer_write_batch(writer, 3, scores, score_value_count,
+    status = carquet_writer_write_batch(writer, 3, scores, NUM_ROWS,
                                         score_def_levels, NULL);
     if (status != CARQUET_OK) goto error;
 
-    status = carquet_writer_write_batch(writer, 4, emails, email_value_count,
+    status = carquet_writer_write_batch(writer, 4, emails, NUM_ROWS,
                                         email_def_levels, NULL);
     if (status != CARQUET_OK) goto error;
 

include/carquet/carquet.h

@@ -1,7 +1,7 @@
 /**
  * @file carquet.h
  * @brief Carquet - High-Performance Pure C Parquet Library
- * @version 0.1.2
+ * @version 0.1.3
  *
  * @copyright Copyright (c) 2025. All rights reserved.
  * @license MIT License
@@ -196,10 +196,10 @@ extern "C" {
 #define CARQUET_VERSION_MINOR 1
 
 /** @brief Patch version number */
-#define CARQUET_VERSION_PATCH 2
+#define CARQUET_VERSION_PATCH 3
 
 /** @brief Version string in "MAJOR.MINOR.PATCH" format */
-#define CARQUET_VERSION_STRING "0.1.2"
+#define CARQUET_VERSION_STRING "0.1.3"
 
 /** @brief Numeric version for compile-time comparisons: (MAJOR * 10000 + MINOR * 100 + PATCH) */
 #define CARQUET_VERSION_NUMBER (CARQUET_VERSION_MAJOR * 10000 + CARQUET_VERSION_MINOR * 100 + CARQUET_VERSION_PATCH)

src/writer/page_writer.c

@@ -307,10 +307,26 @@ carquet_status_t carquet_page_writer_add_values(
         writer->num_nulls += (num_values - num_non_null);
     }
 
-    /* Encode definition levels */
-    if (writer->max_def_level > 0 && def_levels) {
-        encode_levels(def_levels, num_values, writer->max_def_level,
-                      &writer->def_levels_buffer);
+    /* Encode definition levels.
+     * If def_levels is NULL for an OPTIONAL column, generate all-present levels
+     * since Parquet requires definition levels for non-REQUIRED columns. */
+    if (writer->max_def_level > 0) {
+        if (def_levels) {
+            encode_levels(def_levels, num_values, writer->max_def_level,
+                          &writer->def_levels_buffer);
+        } else {
+            /* Auto-generate all-present definition levels */
+            int16_t* auto_def = malloc(num_values * sizeof(int16_t));
+            if (!auto_def) {
+                return CARQUET_ERROR_OUT_OF_MEMORY;
+            }
+            for (int64_t i = 0; i < num_values; i++) {
+                auto_def[i] = writer->max_def_level;
+            }
+            encode_levels(auto_def, num_values, writer->max_def_level,
+                          &writer->def_levels_buffer);
+            free(auto_def);
+        }
     }
 
     /* Encode repetition levels */