Skip to content

Commit e0f4324

Browse files
NiharikaduttaNiharikadutta
authored
Add 3.2.0 Function APIs (#1013)
1 parent 7bc016f commit e0f4324

File tree

12 files changed

+415
-70
lines changed

12 files changed

+415
-70
lines changed

‎azure-pipelines-e2e-tests-template.yml

Lines changed: 54 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,18 @@ stages:
2626
demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open
2727

2828
steps:
29+
- task: PowerShell@2
30+
condition: eq( variables['Agent.OS'], 'Linux')
31+
displayName: Install Maven for Linux agent
32+
inputs:
33+
workingDirectory: $(Build.BinariesDirectory)
34+
pwsh: true
35+
targetType: inline
36+
script: |
37+
sudo apt update
38+
sudo apt -y install maven
39+
mvn -version
40+
2941
- task: UseDotNet@2
3042
displayName: 'Use .NET Core sdk'
3143
inputs:
@@ -107,45 +119,49 @@ stages:
107119
SPARK_HOME: $(Build.BinariesDirectory)$(PATH_SEPARATOR)spark-${{ test.version }}-bin-hadoop2.7
108120
DOTNET_WORKER_DIR: $(CURRENT_DOTNET_WORKER_DIR)
109121

110-
# - pwsh: |
111-
# echo "Downloading ${env:BACKWARD_COMPATIBLE_WORKER_URL}"
112-
# curl -k -L -o Microsoft.Spark.Worker-${{ parameters.backwardCompatibleRelease }}.zip ${env:BACKWARD_COMPATIBLE_WORKER_URL}
113-
# unzip Microsoft.Spark.Worker-${{ parameters.backwardCompatibleRelease }}.zip -d $([System.IO.Directory]::GetParent($env:BACKWARD_COMPATIBLE_DOTNET_WORKER_DIR).FullName)
114-
115-
# if ($env:AGENT_OS -eq 'Linux') {
116-
# chmod +x "${env:BACKWARD_COMPATIBLE_DOTNET_WORKER_DIR}${env:PATH_SEPARATOR}Microsoft.Spark.Worker"
117-
# }
118-
# displayName: 'Setup Backward Compatible Microsoft Spark Worker ${{ parameters.backwardCompatibleRelease }}'
119-
# workingDirectory: $(Build.BinariesDirectory)
120-
121-
# - task: DotNetCoreCLI@2
122-
# displayName: 'E2E Backward Compatibility Tests'
123-
# inputs:
124-
# command: test
125-
# projects: '**/Microsoft.Spark*.E2ETest/*.csproj'
126-
# arguments: '--configuration $(buildConfiguration) ${{ option.backwardCompatibleTestOptions }}'
127-
# workingDirectory: $(Build.SourcesDirectory)$(PATH_SEPARATOR)dotnet-spark
128-
# env:
129-
# HADOOP_HOME: $(Build.BinariesDirectory)$(PATH_SEPARATOR)hadoop
130-
# SPARK_HOME: $(Build.BinariesDirectory)$(PATH_SEPARATOR)spark-${{ test.version }}-bin-hadoop2.7
131-
# DOTNET_WORKER_DIR: $(BACKWARD_COMPATIBLE_DOTNET_WORKER_DIR)
122+
- pwsh: |
123+
echo "Downloading ${env:BACKWARD_COMPATIBLE_WORKER_URL}"
124+
curl -k -L -o Microsoft.Spark.Worker-${{ parameters.backwardCompatibleRelease }}.zip ${env:BACKWARD_COMPATIBLE_WORKER_URL}
125+
unzip Microsoft.Spark.Worker-${{ parameters.backwardCompatibleRelease }}.zip -d $([System.IO.Directory]::GetParent($env:BACKWARD_COMPATIBLE_DOTNET_WORKER_DIR).FullName)
126+
127+
if ($env:AGENT_OS -eq 'Linux') {
128+
chmod +x "${env:BACKWARD_COMPATIBLE_DOTNET_WORKER_DIR}${env:PATH_SEPARATOR}Microsoft.Spark.Worker"
129+
}
130+
condition: ${{ test.enableBackwardCompatibleTests }}
131+
displayName: 'Setup Backward Compatible Microsoft Spark Worker ${{ parameters.backwardCompatibleRelease }}'
132+
workingDirectory: $(Build.BinariesDirectory)
133+
134+
- task: DotNetCoreCLI@2
135+
displayName: 'E2E Backward Compatibility Tests'
136+
condition: ${{ test.enableBackwardCompatibleTests }}
137+
inputs:
138+
command: test
139+
projects: '**/Microsoft.Spark*.E2ETest/*.csproj'
140+
arguments: '--configuration $(buildConfiguration) ${{ option.backwardCompatibleTestOptions }}'
141+
workingDirectory: $(Build.SourcesDirectory)$(PATH_SEPARATOR)dotnet-spark
142+
env:
143+
HADOOP_HOME: $(Build.BinariesDirectory)$(PATH_SEPARATOR)hadoop
144+
SPARK_HOME: $(Build.BinariesDirectory)$(PATH_SEPARATOR)spark-${{ test.version }}-bin-hadoop2.7
145+
DOTNET_WORKER_DIR: $(BACKWARD_COMPATIBLE_DOTNET_WORKER_DIR)
132146

133147
- checkout: forwardCompatibleRelease
134148
path: s$(PATH_SEPARATOR)dotnet-spark-${{ parameters.forwardCompatibleRelease }}
135149

136-
# - task: Maven@3
137-
# displayName: 'Maven build src for forward compatible release v${{ parameters.forwardCompatibleRelease }}'
138-
# inputs:
139-
# mavenPomFile: $(Build.SourcesDirectory)$(PATH_SEPARATOR)dotnet-spark-${{ parameters.forwardCompatibleRelease }}$(PATH_SEPARATOR)src$(PATH_SEPARATOR)scala$(PATH_SEPARATOR)pom.xml
140-
141-
# - task: DotNetCoreCLI@2
142-
# displayName: 'E2E Forward Compatibility Tests'
143-
# inputs:
144-
# command: test
145-
# projects: '**/Microsoft.Spark*.E2ETest/*.csproj'
146-
# arguments: '--configuration $(buildConfiguration) ${{ option.forwardCompatibleTestOptions }}'
147-
# workingDirectory: $(Build.SourcesDirectory)$(PATH_SEPARATOR)dotnet-spark-${{ parameters.forwardCompatibleRelease }}
148-
# env:
149-
# HADOOP_HOME: $(Build.BinariesDirectory)$(PATH_SEPARATOR)hadoop
150-
# SPARK_HOME: $(Build.BinariesDirectory)$(PATH_SEPARATOR)spark-${{ test.version }}-bin-hadoop2.7
151-
# DOTNET_WORKER_DIR: $(CURRENT_DOTNET_WORKER_DIR)
150+
- task: Maven@3
151+
displayName: 'Maven build src for forward compatible release v${{ parameters.forwardCompatibleRelease }}'
152+
condition: ${{ test.enableForwardCompatibleTests }}
153+
inputs:
154+
mavenPomFile: $(Build.SourcesDirectory)$(PATH_SEPARATOR)dotnet-spark-${{ parameters.forwardCompatibleRelease }}$(PATH_SEPARATOR)src$(PATH_SEPARATOR)scala$(PATH_SEPARATOR)pom.xml
155+
156+
- task: DotNetCoreCLI@2
157+
displayName: 'E2E Forward Compatibility Tests'
158+
condition: ${{ test.enableForwardCompatibleTests }}
159+
inputs:
160+
command: test
161+
projects: '**/Microsoft.Spark*.E2ETest/*.csproj'
162+
arguments: '--configuration $(buildConfiguration) ${{ option.forwardCompatibleTestOptions }}'
163+
workingDirectory: $(Build.SourcesDirectory)$(PATH_SEPARATOR)dotnet-spark-${{ parameters.forwardCompatibleRelease }}
164+
env:
165+
HADOOP_HOME: $(Build.BinariesDirectory)$(PATH_SEPARATOR)hadoop
166+
SPARK_HOME: $(Build.BinariesDirectory)$(PATH_SEPARATOR)spark-${{ test.version }}-bin-hadoop2.7
167+
DOTNET_WORKER_DIR: $(CURRENT_DOTNET_WORKER_DIR)

‎azure-pipelines.yml

Lines changed: 60 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,8 @@ variables:
1313
MSBUILDSINGLELOADCONTEXT: 1
1414
ArtifactPath: '$(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries'
1515

16-
backwardCompatibleRelease: '1.0.0'
17-
forwardCompatibleRelease: '1.0.0'
16+
backwardCompatibleRelease: '2.0.0'
17+
forwardCompatibleRelease: '2.0.0'
1818

1919
backwardCompatibleTestOptions_Windows_2_4: ""
2020
forwardCompatibleTestOptions_Windows_2_4: ""
@@ -31,6 +31,12 @@ variables:
3131
backwardCompatibleTestOptions_Linux_3_1: ""
3232
forwardCompatibleTestOptions_Linux_3_1: ""
3333

34+
# Skip all forward/backward compatibility tests since Spark 3.2 is not supported before this release.
35+
backwardCompatibleTestOptions_Windows_3_2: "--filter FullyQualifiedName=NONE"
36+
forwardCompatibleTestOptions_Windows_3_2: $(backwardCompatibleTestOptions_Windows_3_2)
37+
backwardCompatibleTestOptions_Linux_3_2: $(backwardCompatibleTestOptions_Windows_3_2)
38+
forwardCompatibleTestOptions_Linux_3_2: $(backwardCompatibleTestOptions_Windows_3_2)
39+
3440
# Azure DevOps variables are transformed into environment variables, with these variables we
3541
# avoid the first time experience and telemetry to speed up the build.
3642
DOTNET_CLI_TELEMETRY_OPTOUT: 1
@@ -202,6 +208,8 @@ stages:
202208
forwardCompatibleRelease: $(forwardCompatibleRelease)
203209
tests:
204210
- version: '2.4.0'
211+
enableForwardCompatibleTests: true
212+
enableBackwardCompatibleTests: true
205213
jobOptions:
206214
- pool: 'Windows'
207215
testOptions: ""
@@ -212,6 +220,8 @@ stages:
212220
backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_2_4)
213221
forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_2_4)
214222
- version: '2.4.1'
223+
enableForwardCompatibleTests: true
224+
enableBackwardCompatibleTests: true
215225
jobOptions:
216226
- pool: 'Windows'
217227
testOptions: ""
@@ -222,6 +232,8 @@ stages:
222232
backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_2_4)
223233
forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_2_4)
224234
- version: '2.4.3'
235+
enableForwardCompatibleTests: true
236+
enableBackwardCompatibleTests: true
225237
jobOptions:
226238
- pool: 'Windows'
227239
testOptions: ""
@@ -232,6 +244,8 @@ stages:
232244
backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_2_4)
233245
forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_2_4)
234246
- version: '2.4.4'
247+
enableForwardCompatibleTests: true
248+
enableBackwardCompatibleTests: true
235249
jobOptions:
236250
- pool: 'Windows'
237251
testOptions: ""
@@ -242,6 +256,8 @@ stages:
242256
backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_2_4)
243257
forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_2_4)
244258
- version: '2.4.5'
259+
enableForwardCompatibleTests: true
260+
enableBackwardCompatibleTests: true
245261
jobOptions:
246262
- pool: 'Windows'
247263
testOptions: ""
@@ -252,6 +268,8 @@ stages:
252268
backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_2_4)
253269
forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_2_4)
254270
- version: '2.4.6'
271+
enableForwardCompatibleTests: true
272+
enableBackwardCompatibleTests: true
255273
jobOptions:
256274
- pool: 'Windows'
257275
testOptions: ""
@@ -262,6 +280,8 @@ stages:
262280
backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_2_4)
263281
forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_2_4)
264282
- version: '2.4.7'
283+
enableForwardCompatibleTests: true
284+
enableBackwardCompatibleTests: true
265285
jobOptions:
266286
- pool: 'Windows'
267287
testOptions: ""
@@ -272,6 +292,8 @@ stages:
272292
backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_2_4)
273293
forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_2_4)
274294
- version: '2.4.8'
295+
enableForwardCompatibleTests: true
296+
enableBackwardCompatibleTests: true
275297
jobOptions:
276298
- pool: 'Windows'
277299
testOptions: ""
@@ -282,6 +304,8 @@ stages:
282304
backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_2_4)
283305
forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_2_4)
284306
- version: '3.0.0'
307+
enableForwardCompatibleTests: true
308+
enableBackwardCompatibleTests: true
285309
jobOptions:
286310
- pool: 'Windows'
287311
testOptions: ""
@@ -292,6 +316,8 @@ stages:
292316
backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_3_0)
293317
forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_3_0)
294318
- version: '3.0.1'
319+
enableForwardCompatibleTests: true
320+
enableBackwardCompatibleTests: true
295321
jobOptions:
296322
- pool: 'Windows'
297323
testOptions: ""
@@ -302,6 +328,8 @@ stages:
302328
backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_3_0)
303329
forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_3_0)
304330
- version: '3.0.2'
331+
enableForwardCompatibleTests: true
332+
enableBackwardCompatibleTests: true
305333
jobOptions:
306334
- pool: 'Windows'
307335
testOptions: ""
@@ -312,22 +340,50 @@ stages:
312340
backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_3_0)
313341
forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_3_0)
314342
- version: '3.1.1'
343+
enableForwardCompatibleTests: true
344+
enableBackwardCompatibleTests: true
315345
jobOptions:
316346
- pool: 'Windows'
317347
testOptions: ""
318348
backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_3_1)
319-
forwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_3_1)
349+
forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_3_1)
320350
- pool: 'Linux'
321351
testOptions: ""
322352
backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_3_1)
323353
forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_3_1)
324354
- version: '3.1.2'
355+
enableForwardCompatibleTests: true
356+
enableBackwardCompatibleTests: true
325357
jobOptions:
326358
- pool: 'Windows'
327359
testOptions: ""
328360
backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_3_1)
329-
forwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_3_1)
361+
forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_3_1)
330362
- pool: 'Linux'
331363
testOptions: ""
332364
backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_3_1)
333365
forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_3_1)
366+
- version: '3.2.0'
367+
enableForwardCompatibleTests: false
368+
enableBackwardCompatibleTests: false
369+
jobOptions:
370+
- pool: 'Windows'
371+
testOptions: ""
372+
backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_3_2)
373+
forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_3_2)
374+
- pool: 'Linux'
375+
testOptions: ""
376+
backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_3_2)
377+
forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_3_2)
378+
- version: '3.2.1'
379+
enableForwardCompatibleTests: false
380+
enableBackwardCompatibleTests: false
381+
jobOptions:
382+
- pool: 'Windows'
383+
testOptions: ""
384+
backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_3_2)
385+
forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_3_2)
386+
- pool: 'Linux'
387+
testOptions: ""
388+
backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_3_2)
389+
forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_3_2)

‎src/csharp/Extensions/Microsoft.Spark.Extensions.Delta.E2ETest/DeltaFixture.cs

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,12 @@ public class DeltaFixture
1616
public DeltaFixture()
1717
{
1818
Version sparkVersion = SparkSettings.Version;
19-
string deltaVersion = sparkVersion.Major switch
19+
string deltaVersion = (sparkVersion.Major, sparkVersion.Minor) switch
2020
{
21-
2 => "delta-core_2.11:0.6.1",
22-
3 => "delta-core_2.12:0.8.0",
21+
(2, _) => "delta-core_2.11:0.6.1",
22+
(3, 0) => "delta-core_2.12:0.8.0",
23+
(3, 1) => "delta-core_2.12:1.0.0",
24+
(3, 2) => "delta-core_2.12:1.1.0",
2325
_ => throw new NotSupportedException($"Spark {sparkVersion} not supported.")
2426
};
2527

‎src/csharp/Microsoft.Spark.E2ETest/IpcTests/Sql/DataFrameWriterTests.cs

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,10 @@ public void TestSignaturesV2_4_X()
8181

8282
dfw.Csv($"{tempDir.Path}TestCsvPath");
8383

84-
dfw.Option("path", tempDir.Path).SaveAsTable("TestTable");
84+
// In Spark 3.2.0+ cannot create table with location to a non-empty directory.
85+
// To allow overwriting the existing non-empty directory, set
86+
// 'spark.sql.legacy.allowNonEmptyLocationInCTAS' to true.
87+
dfw.Option("path", $"{tempDir.Path}EmptyDir").SaveAsTable("TestTable");
8588

8689
dfw.InsertInto("TestTable");
8790

‎src/csharp/Microsoft.Spark.E2ETest/IpcTests/Sql/FunctionsTests.cs

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -805,5 +805,42 @@ public void TestSignaturesV3_1_X()
805805

806806
Assert.IsType<Column>(Slice(col, col, col));
807807
}
808+
809+
/// <summary>
810+
/// Test signatures for APIs introduced in Spark 3.2.*.
811+
/// </summary>
812+
[SkipIfSparkVersionIsLessThan(Versions.V3_2_0)]
813+
public void TestSignaturesV3_2_X()
814+
{
815+
Column col = Column("col");
816+
817+
Assert.IsType<Column>(Count_Distinct(col, col, col));
818+
819+
Assert.IsType<Column>(Product(col));
820+
821+
Assert.IsType<Column>(Sum_Distinct(col));
822+
823+
Assert.IsType<Column>(Lag(col, 2, null, true));
824+
825+
Assert.IsType<Column>(Lead(col, 2, null, true));
826+
827+
Assert.IsType<Column>(Bitwise_Not(col));
828+
829+
Assert.IsType<Column>(Shiftleft(col, 2));
830+
831+
Assert.IsType<Column>(Shiftright(col, 2));
832+
833+
Assert.IsType<Column>(Shiftrightunsigned(col, 2));
834+
835+
Assert.IsType<Column>(Sentences(col, col, col));
836+
Assert.IsType<Column>(Sentences(col));
837+
838+
Assert.IsType<Column>(NextDay(col, col));
839+
840+
Assert.IsType<Column>(Session_Window(col, "5 seconds"));
841+
Assert.IsType<Column>(Session_Window(col, col));
842+
843+
Assert.IsType<Column>(Call_UDF("name", col, col));
844+
}
808845
}
809846
}

‎src/csharp/Microsoft.Spark.Worker.UnitTest/PayloadWriter.cs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -304,7 +304,8 @@ internal PayloadWriter Create(Version version = null)
304304
new TaskContextWriterV2_4_X(),
305305
new BroadcastVariableWriterV2_4_X(),
306306
new CommandWriterV2_4_X());
307-
case Versions.V3_0_0:
307+
case Versions.V3_0_0:
308+
case Versions.V3_2_0:
308309
return new PayloadWriter(
309310
version,
310311
new TaskContextWriterV3_0_X(),

‎src/csharp/Microsoft.Spark.Worker.UnitTest/SimpleWorkerTests.cs

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
using System.Collections.Generic;
33
using System.Net;
44
using System.Threading.Tasks;
5+
using Microsoft.Spark.Interop.Ipc;
56
using Microsoft.Spark.Network;
67
using Xunit;
78

@@ -25,7 +26,15 @@ public void TestsSimpleWorkerTaskRunners(string version)
2526
Task clientTask = Task.Run(() => simpleWorker.Run(ipEndpoint.Port));
2627

2728
PayloadWriter payloadWriter = new PayloadWriterFactory().Create(typedVersion);
28-
TaskRunnerTests.TestTaskRunnerReadWrite(serverListener, payloadWriter);
29+
using (ISocketWrapper serverSocket = serverListener.Accept())
30+
{
31+
if ((typedVersion.Major == 3 && typedVersion.Minor >= 2) || typedVersion.Major > 3)
32+
{
33+
int pid = SerDe.ReadInt32(serverSocket.InputStream);
34+
}
35+
36+
TaskRunnerTests.TestTaskRunnerReadWrite(serverSocket, payloadWriter);
37+
}
2938

3039
Assert.True(clientTask.Wait(5000));
3140
}

0 commit comments

Comments
 (0)