Add a new Python module example with parquet files

test/library/packages/Python/examples/parquet/.gitignore

@@ -0,0 +1,2 @@
+data.parquet
+readParquet.good

test/library/packages/Python/examples/parquet/CLEANFILES

@@ -0,0 +1 @@
+../../CLEANFILES

test/library/packages/Python/examples/parquet/COMPOPTS

@@ -0,0 +1 @@
+../../COMPOPTS

test/library/packages/Python/examples/parquet/EXECENV

@@ -0,0 +1 @@
+../../EXECENV

test/library/packages/Python/examples/parquet/NUMLOCALES

@@ -0,0 +1 @@
+2

test/library/packages/Python/examples/parquet/PRETEST

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+FILE_DIR=$(cd $(dirname ${BASH_SOURCE[0]}) ; pwd)
+$FILE_DIR/../../checkAndInstallPackage.sh $FILE_DIR pyarrow pandas numpy

test/library/packages/Python/examples/parquet/create_file.py

@@ -0,0 +1,13 @@
+import pandas as pd
+import numpy as np
+
+n = 1000
+data = {
+    'Integers': np.random.randint(1, 100, size=n),
+    'Floats1': np.random.uniform(1.0, 100.0, size=n),
+    'Floats2': np.random.uniform(1.0, 100.0, size=n)
+}
+df = pd.DataFrame(data)
+df.to_parquet("data.parquet", index=False, row_group_size=100)
+for c in data.keys():
+    print(f"Column: {c} Sum: {round(df[c].sum(), 1)}")

test/library/packages/Python/examples/parquet/readParquet.chpl

@@ -0,0 +1,55 @@
+use Python, List;
+
+config const filename = "data.parquet";
+
+
+proc getArray(type eltType, ref data_chunks, num_rows) {
+  var arr: [0..<num_rows] eltType;
+  var i = 0;
+  for chunk in data_chunks {
+    var chunk_arr =
+      chunk!.call(
+        owned PyArray(eltType, 1),
+        "to_numpy", kwargs=["zero_copy_only"=>false, "writable"=>true]);
+    arr[i..#chunk_arr.size] = chunk_arr.array();
+    i += chunk_arr.size;
+  }
+  return arr;
+}
+
+proc main() {
+
+  var interp = new Interpreter();
+
+  var pa = interp.importModule("pyarrow");
+  var pq = interp.importModule("pyarrow.parquet");
+
+  var parquet_file = pq.call("ParquetFile", filename);
+  var columns = parquet_file.get("schema").get("names"):list(string);
+  var num_rows = parquet_file.get("metadata").get("num_rows"):int;
+  var data_chunks: [0..<columns.size] list(owned Value?);
+
+  for batch in parquet_file.call("iter_batches", kwargs=["batch_size"=>300]) {
+    for (col, idx) in zip(columns, 0..) {
+      data_chunks[idx].pushBack(batch.call("__getitem__", col));
+    }
+  }
+
+  var schema_arrow = parquet_file.get('schema_arrow');
+  for (col, idx) in zip(columns, 0..) {
+    write("Column: ", col);
+
+    var rowType = schema_arrow.call('field', col).get('type');
+    if pa.call("int64") == rowType {
+      var arr = getArray(int(64), data_chunks[idx], num_rows);
+      writeln(" Sum: ", + reduce arr);
+    } else if pa.call("float64") == rowType {
+      var arr = getArray(real(64), data_chunks[idx], num_rows);
+      writeln(" Sum: ", + reduce arr);
+    } else {
+      writeln("Unknown type");
+    }
+  }
+}
+
+

test/library/packages/Python/examples/parquet/readParquet.cleanfiles

@@ -0,0 +1,2 @@
+data.parquet
+readParquet.good

test/library/packages/Python/examples/parquet/readParquet.precomp

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+# respect CHPL_TEST_VENV_DIR if it is set and not none
+if [ -n "$CHPL_TEST_VENV_DIR" ] && [ "$CHPL_TEST_VENV_DIR" != "none" ]; then
+  chpl_python=$CHPL_TEST_VENV_DIR/bin/python3
+else
+  chpl_python=$($CHPL_HOME/util/config/find-python.sh)
+fi
+
+# make sure to add the libs dir to the PYTHONPATH if it exists
+FILE_DIR=$(cd $(dirname ${BASH_SOURCE[0]}) ; pwd)
+MY_LIB_DIR=$FILE_DIR/python_libs
+if [ -d "$MY_LIB_DIR" ]; then
+  export PYTHONPATH=$MY_LIB_DIR:$PYTHONPATH
+fi
+
+$chpl_python create_file.py > $1.good