Open
Description
Describe the bug
I am getting file not found '' when I follow the VectorUDF example in Synapse Notebook.
To Reproduce
//Cell 2
var names = new List<string> { "john", "20" };
var df = spark.CreateDataFrame(
new List<GenericRow>{new GenericRow(names.ToArray())},
new StructType(
new List<StructField>()
{
new StructField("name", new StringType()),
new StructField("age", new StringType())
}));
//Cell 3
private static FxDataFrame CountCharacters(FxDataFrame dataFrame)
{
int characterCount = 0;
var characterCountColumn = new Int32DataFrameColumn("nameCharCount");
var ageColumn = new Int32DataFrameColumn("age");
ArrowStringDataFrameColumn nameColumn = dataFrame.Columns.GetArrowStringColumn("name");
for (long i = 0; i < dataFrame.Rows.Count; ++i)
{
characterCount += nameColumn[i].Length;
}
if (dataFrame.Rows.Count > 0)
{
characterCountColumn.Append(characterCount);
ageColumn.Append(dataFrame.Columns.GetInt32Column("age")[0]);
}
return new FxDataFrame(ageColumn, characterCountColumn);
}
//Cell 4
df.GroupBy("age")
.Apply(
new StructType(new[]
{
new StructField("age", new IntegerType()),
new StructField("nameCharCount", new IntegerType())
}),
r => CountCharacters(r))
.Show();
Expected behavior
Get a new dataframe with counts
Desktop (please complete the following information):
- OS: Synapse Notebook
- Browser Edge
- Version
Additional context
I have not tried this in any other environment