feat: convert to csv (#6)

* feat: convert to csv

* update docs

Author: zachspar

Cargo.lock

@@ -1,6 +1,6 @@
 [package]
 name = "parquet-py"
-version = "0.0.5-alpha"
+version = "0.1.0-beta"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
@@ -12,3 +12,4 @@ crate-type = ["cdylib"]
 parquet = { version = "52.0.0", features = ["json"] }
 pyo3 = "0.22.0"
 serde_json = "1.0.120"
+csv = "1.3.0"

Cargo.toml

@@ -2,15 +2,16 @@
 
 # Parquet-Py
 
-Parquet-Py is a simple Python API & CLI designed to facilitate the interaction with Parquet files. It allows users to convert Parquet files into JSON strings, lists, or iterators for easy manipulation and access in Python applications.
+Parquet-Py is a simple command-line interface & Python API designed to facilitate the interaction with Parquet files. It allows users to convert Parquet files into CSV, JSON, lists, and iterators for easy manipulation and access in Python applications.
 
-Using Rust bindings under the hood, Parquet-Py provides a fast and efficient way to work with Parquet files, making it ideal for processing large datasets.
+Using Rust bindings under the hood, Parquet-Py provides a fast and efficient way to work with Parquet files, making it ideal for converting or processing large datasets.
 
 ## Features
 
-- **Convert Parquet to JSON String**: Easily convert your Parquet files into a JSON string format for quick inspection or processing.
-- **Convert Parquet to Python List**: Transform your Parquet files into Python lists, where each row is represented as a dictionary within the list.
+- **Convert Parquet to CSV**: Convert your Parquet files into CSV format for easy viewing and processing in spreadsheet applications.
+- **Convert Parquet to JSON**: Easily convert your Parquet files into a JSON string format for quick inspection or processing.
 - **Iterable Parquet Rows**: Access Parquet file rows through an iterator, allowing for efficient row-by-row processing without loading the entire file into memory.
+- **Convert Parquet to Python List**: Transform your Parquet files into Python lists, where each row is represented as a dictionary within the list.
 
 ## Installation
 
@@ -20,6 +21,14 @@ Using Rust bindings under the hood, Parquet-Py provides a fast and efficient way
 ## Usage
 ### Command-Line Interface
 
+#### Converting Parquet to CSV
+
+To convert a Parquet file into a CSV file, use the `parq convert` command.
+
+```bash
+parq convert --input path/to/your/file.parquet --format csv --output example.csv
+```
+
 #### Converting Parquet to JSON
 
 To convert a Parquet file into a JSON string, use the `parq convert` command.
@@ -28,11 +37,42 @@ To convert a Parquet file into a JSON string, use the `parq convert` command.
 parq convert --input path/to/your/file.parquet --format json --output example.json
 ```
 
+
 ### Python
 
+#### Iterating Over Parquet Rows
+
+To iterate over the rows of a Parquet file, use the `iter_rows` function. This allows for efficient row-by-row processing without loading the entire file into memory.
+
+```python
+from parq import to_iter
+
+# Path to your Parquet file
+file_path = "path/to/your/file.parquet"
+
+# Iterate over Parquet rows
+for row in to_iter(file_path):
+    print(row)
+```
+
+#### Converting Parquet to CSV String
+
+To convert a Parquet file into a CSV string, use the `to_csv_str` function.
+
+```python
+from parq import to_csv_str
+
+# Path to your Parquet file
+file_path = "path/to/your/file.parquet"
+
+# Convert to CSV string
+csv_str = to_csv_str(file_path)
+print(csv_str)
+```
+
 #### Converting Parquet to JSON String
 
-To convert a Parquet file into a JSON string, use the `to_json_str` function. This is useful for quick inspection or processing of the data.
+To convert a Parquet file into a JSON string, use the `to_json_str` function.
 
 ```python
 from parq import to_json_str
@@ -59,18 +99,3 @@ file_path = "path/to/your/file.parquet"
 data_list = to_list(file_path)
 print(len(data_list))
 ```
-
-#### Iterating Over Parquet Rows
-
-To iterate over the rows of a Parquet file, use the `iter_rows` function. This allows for efficient row-by-row processing without loading the entire file into memory.
-
-```python
-from parq import to_iter
-
-# Path to your Parquet file
-file_path = "path/to/your/file.parquet"
-
-# Iterate over Parquet rows
-for row in to_iter(file_path):
-    print(row)
-```

README.md

@@ -1 +1 @@
-from .lib import to_json_str, to_list, to_iter
+from .lib import to_json_str, to_csv_str, to_list, to_iter

parq/__init__.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 import click
-from parq import to_json_str
+from parq import to_json_str, to_csv_str
 
 
 @click.group()
@@ -19,6 +19,7 @@ def parq_cli():
     type=click.Choice(
         [
             "json",
+            "csv",
         ],
         case_sensitive=False,
     ),
@@ -28,12 +29,17 @@ def parq_cli():
 @click.option("--output", "-o", "output_file_path", help="Output file", type=click.Path(), required=False)
 def convert(parquet_file_path, output_format, output_file_path):
     if output_format == "json":
-        json_str = to_json_str(str(parquet_file_path))
-        if output_file_path:
-            with open(output_file_path, "w") as f:
-                f.write(json_str)
-        else:
-            click.echo(json_str)
+        output_string = to_json_str(str(parquet_file_path))
+    elif output_format == "csv":
+        output_string = to_csv_str(str(parquet_file_path))
+    else:
+        raise ValueError(f"Unsupported output format {output_format}")
+
+    if output_file_path:
+        with open(output_file_path, "w") as f:
+            f.write(output_string)
+    else:
+        click.echo(output_string)
 
 
 if __name__ == "__main__":

parq/cli/__main__.py

@@ -1,6 +1,6 @@
 use parquet::file::reader::{FileReader, SerializedFileReader};
 use parquet::record::reader::RowIter;
-use pyo3::exceptions::PyStopIteration;
+use pyo3::exceptions::{PyIOError, PyStopIteration, PyValueError};
 use pyo3::prelude::*;
 use pyo3::types::{PyBool, PyDict, PyList};
 use serde_json::Value;
@@ -51,14 +51,46 @@ fn value_to_py_object(py: Python, value: &Value) -> PyResult<PyObject> {
     }
 }
 
-// convert parquet file to json string
+/// to_csv_str(path: str) -> str
+/// --
+///
+/// Read parquet file and convert to csv string.
+#[pyfunction]
+fn to_csv_str(path: &str) -> PyResult<String> {
+    let file_path = Path::new(path);
+    let file = File::open(&file_path).map_err(|e| PyIOError::new_err(e.to_string()))?;
+    let reader = SerializedFileReader::new(file).map_err(|e| PyValueError::new_err(e.to_string()))?;
+    let metadata = reader.metadata();
+    let schema = metadata.file_metadata().schema();
+
+    let mut wtr = csv::Writer::from_writer(vec![]);
+    let fields = schema.get_fields();
+    let headers: Vec<String> = fields.iter().map(|f| f.name().to_string()).collect();
+    wtr.write_record(&headers).map_err(|e| PyValueError::new_err(e.to_string()))?;
+
+    let row_iter = reader.get_row_iter(None).map_err(|e| PyValueError::new_err(e.to_string()))?;
+    for row_result in row_iter {
+        let row = row_result.map_err(|e| PyValueError::new_err(e.to_string()))?;
+        let csv_record: Vec<String> = row.get_column_iter().map(|(_col_idx, col)| col.to_string()).collect();
+        wtr.write_record(&csv_record).map_err(|e| PyValueError::new_err(e.to_string()))?;
+    }
+
+    wtr.flush().map_err(|e| PyValueError::new_err(e.to_string()))?;
+    let csv_data = String::from_utf8(wtr.into_inner().map_err(|e| PyValueError::new_err(e.to_string()))?).map_err(|e| PyValueError::new_err(e.to_string()))?;
+    Ok(csv_data)
+}
+
+/// to_json_str(path: str) -> str
+/// --
+///
+/// Read parquet file and convert to JSON string.
 #[pyfunction]
 fn to_json_str(path: &str) -> PyResult<String> {
     let file_path = Path::new(path);
     let file =
-        File::open(&file_path).map_err(|e| pyo3::exceptions::PyIOError::new_err(e.to_string()))?;
+        File::open(&file_path).map_err(|e| PyIOError::new_err(e.to_string()))?;
     let reader = SerializedFileReader::new(file)
-        .map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))?;
+        .map_err(|e| PyValueError::new_err(e.to_string()))?;
 
     // iterate through reader and add to json list string
     let mut json_str = "[".to_string();
@@ -74,6 +106,10 @@ fn to_json_str(path: &str) -> PyResult<String> {
     return Ok(json_str);
 }
 
+/// ParquetRowIterator
+/// --
+///
+/// Iterator over rows in parquet file.
 #[pyclass]
 struct ParquetRowIterator {
     iter: RowIter<'static>,
@@ -85,9 +121,9 @@ impl ParquetRowIterator {
     fn new(path: &str) -> PyResult<Self> {
         let file_path = Path::new(path);
         let file = File::open(&file_path)
-            .map_err(|e| pyo3::exceptions::PyIOError::new_err(e.to_string()))?;
+            .map_err(|e| PyIOError::new_err(e.to_string()))?;
         let reader = SerializedFileReader::new(file)
-            .map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))?;
+            .map_err(|e| PyValueError::new_err(e.to_string()))?;
 
         Ok(Self {
             iter: RowIter::from_file_into(Box::new(reader)),
@@ -112,26 +148,34 @@ impl ParquetRowIterator {
     }
 }
 
+/// to_iter(path: str) -> ParquetRowIterator
+/// --
+///
+/// Return iterator over rows in parquet file.
 #[pyfunction]
 fn to_iter(path: &str) -> PyResult<ParquetRowIterator> {
     let file_path = Path::new(path);
     let file =
-        File::open(&file_path).map_err(|e| pyo3::exceptions::PyIOError::new_err(e.to_string()))?;
+        File::open(&file_path).map_err(|e| PyIOError::new_err(e.to_string()))?;
     let reader = SerializedFileReader::new(file)
-        .map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))?;
+        .map_err(|e| PyValueError::new_err(e.to_string()))?;
 
     Ok(ParquetRowIterator {
         iter: RowIter::from_file_into(Box::new(reader)),
     })
 }
 
+/// to_list(path: str) -> List[Dict[str, Any]]
+/// --
+///
+/// Read parquet file and convert to list of dictionaries.
 #[pyfunction]
 fn to_list(path: &str, py: Python) -> PyResult<PyObject> {
     let file_path = Path::new(path);
     let file =
-        File::open(&file_path).map_err(|e| pyo3::exceptions::PyIOError::new_err(e.to_string()))?;
+        File::open(&file_path).map_err(|e| PyIOError::new_err(e.to_string()))?;
     let reader = SerializedFileReader::new(file)
-        .map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))?;
+        .map_err(|e| PyValueError::new_err(e.to_string()))?;
     let list = PyList::empty_bound(py);
     for row in reader.get_row_iter(None).unwrap() {
         let row_dict = row.unwrap().to_json_value();
@@ -145,10 +189,11 @@ fn to_list(path: &str, py: Python) -> PyResult<PyObject> {
     Ok(list.into())
 }
 
-// python module
+/// A Parquet file reader and converter, written in Rust.
 #[pymodule]
 fn lib(m: &Bound<'_, PyModule>) -> PyResult<()> {
     m.add_function(wrap_pyfunction!(to_json_str, m)?)?;
+    m.add_function(wrap_pyfunction!(to_csv_str, m)?)?;
     m.add_function(wrap_pyfunction!(to_list, m)?)?;
     m.add_function(wrap_pyfunction!(to_iter, m)?)?;
     m.add_class::<ParquetRowIterator>()?;