Skip to content

Commit 86d4904

Browse files
vara-bonthuvara-bonthu
authored
feat: TPCDS Benchmarks for "Spark + Gluten +Velox" Vs Native Spark (#903)
Signed-off-by: vara-bonthu <vara.bonthu@gmail.com> Signed-off-by: Vara Bonthu <vara.bonthu@gmail.com>
1 parent 12ab9a7 commit 86d4904

28 files changed

Lines changed: 5009 additions & 1091 deletions

‎.github/workflows/website-deploy.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ jobs:
1919
- uses: actions/checkout@v2
2020
- uses: actions/setup-node@v3
2121
with:
22-
node-version: 18
22+
node-version: 20
2323
cache: npm
2424
cache-dependency-path: website/package-lock.json
2525
- name: Install dependencies

‎.github/workflows/website-test-deploy.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ jobs:
1717
- uses: actions/checkout@v2
1818
- uses: actions/setup-node@v3
1919
with:
20-
node-version: 18
20+
node-version: 20
2121
cache: npm
2222
cache-dependency-path: website/package-lock.json
2323

‎analytics/terraform/spark-k8s-operator/examples/benchmark/tpcds-benchmark-spark-gluten-velox/Dockerfile-tpcds-gluten-velox‎

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
# TPC-DS Benchmark with Gluten+Velox Support
2+
# Based on your existing TPC-DS benchmark Dockerfile but with Gluten+Velox added
3+
# docker buildx build \
4+
# --platform linux/amd64 \
5+
# -f Dockerfile-tpcds-gluten-velox \
6+
# -t "${FULL_IMAGE}" \
7+
# --load .
8+
9+
# Ubuntu 22.04 base (matching your Gluten base)
10+
FROM ubuntu:22.04
11+
12+
# ---- Versions / args ----
13+
ARG HADOOP_VERSION=3.4.1
14+
ARG AWS_SDK_VERSION=2.29.0 # AWS SDK v2 bundle for Hadoop 3.4.x
15+
ARG SPARK_VERSION=3.5.2 # Gluten supports up to 3.5.2
16+
ARG SCALA_BINARY=2.12
17+
ARG SPARK_UID=185
18+
ARG SPARK_GID=185
19+
ARG TARGETARCH # provided by buildx: amd64 | arm64
20+
21+
ENV DEBIAN_FRONTEND=noninteractive
22+
ENV SPARK_HOME=/opt/spark
23+
ENV PATH=${SPARK_HOME}/bin:${SPARK_HOME}/sbin:$PATH
24+
# Map OpenJDK path for Debian/Ubuntu multi-arch
25+
ENV JAVA_HOME=/usr/lib/jvm/java-17-openjdk-${TARGETARCH}
26+
27+
# ---- Base OS deps (matches your list) ----
28+
RUN apt-get update && apt-get install -y --no-install-recommends \
29+
openjdk-17-jdk-headless \
30+
python3 python3-pip \
31+
curl wget \
32+
ca-certificates tzdata \
33+
libgomp1 numactl \
34+
tini \
35+
gcc \
36+
make \
37+
flex \
38+
bison \
39+
git \
40+
libc6-dev \
41+
build-essential \
42+
&& rm -rf /var/lib/apt/lists/*
43+
44+
# CA bundle path some libs expect (RHEL-style)
45+
RUN mkdir -p /etc/pki/tls/certs && \
46+
ln -sf /etc/ssl/certs/ca-certificates.crt /etc/pki/tls/certs/ca-bundle.crt
47+
48+
# ---- Install Apache Spark 3.5.2 (Hadoop3 build) ----
49+
RUN set -eux; \
50+
wget -q https://mirror.lyrahosting.com/apache/spark/spark-3.5.2/spark-3.5.2-bin-hadoop3.tgz -O /tmp/spark.tgz; \
51+
tar -xzf /tmp/spark.tgz -C /opt; \
52+
ln -s /opt/spark-${SPARK_VERSION}-bin-hadoop3 ${SPARK_HOME}; \
53+
rm -f /tmp/spark.tgz
54+
55+
# ---- Create spark user ----
56+
RUN groupadd -g ${SPARK_GID} spark && \
57+
useradd -m -u ${SPARK_UID} -g ${SPARK_GID} -s /bin/bash spark && \
58+
chown -R spark:spark /opt/spark-${SPARK_VERSION}-bin-hadoop3 && \
59+
chown -R spark:spark ${SPARK_HOME}
60+
61+
# ---- Align Hadoop client + S3A (remove v1 AWS jars, add Hadoop 3.4.1 + AWS SDK v2) ----
62+
RUN set -eux; cd ${SPARK_HOME}/jars; \
63+
# Purge prebundled Hadoop & AWS SDK v1 to avoid mixing
64+
find . -maxdepth 1 -type f -name 'hadoop-*.jar' -delete || true; \
65+
find . -maxdepth 1 -type f -name 'aws-java-sdk-*.jar' -delete || true; \
66+
# Core Hadoop jars
67+
wget -q https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-common/${HADOOP_VERSION}/hadoop-common-${HADOOP_VERSION}.jar; \
68+
wget -q https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-auth/${HADOOP_VERSION}/hadoop-auth-${HADOOP_VERSION}.jar; \
69+
wget -q https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-client-api/${HADOOP_VERSION}/hadoop-client-api-${HADOOP_VERSION}.jar; \
70+
wget -q https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-client-runtime/${HADOOP_VERSION}/hadoop-client-runtime-${HADOOP_VERSION}.jar; \
71+
# S3A + AWS SDK v2 (bundle)
72+
wget -q https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/${HADOOP_VERSION}/hadoop-aws-${HADOOP_VERSION}.jar; \
73+
wget -q https://repo1.maven.org/maven2/software/amazon/awssdk/bundle/${AWS_SDK_VERSION}/bundle-${AWS_SDK_VERSION}.jar; \
74+
# Hadoop 3.4.x needs commons-configuration2 on classpath
75+
wget -q https://repo1.maven.org/maven2/org/apache/commons/commons-configuration2/2.9.0/commons-configuration2-2.9.0.jar; \
76+
# Spark cloud committers (match Spark version)
77+
wget -q https://repo1.maven.org/maven2/org/apache/spark/spark-hadoop-cloud_${SCALA_BINARY}/${SPARK_VERSION}/spark-hadoop-cloud_${SCALA_BINARY}-${SPARK_VERSION}.jar; \
78+
# XML deps to satisfy jackson-dataformat-xml
79+
wget -q https://repo1.maven.org/maven2/com/fasterxml/woodstox/woodstox-core/6.5.1/woodstox-core-6.5.1.jar; \
80+
wget -q https://repo1.maven.org/maven2/org/codehaus/woodstox/stax2-api/4.2.2/stax2-api-4.2.2.jar
81+
82+
# ---- Add Gluten bundle for Spark 3.5.2 ----
83+
RUN set -eux; cd ${SPARK_HOME}/jars; \
84+
wget -q https://dlcdn.apache.org/incubator/gluten/1.4.0-incubating/apache-gluten-1.4.0-incubating-bin-spark35.tar.gz; \
85+
tar -xzf apache-gluten-1.4.0-incubating-bin-spark35.tar.gz; \
86+
cp apache-gluten-1.4.0-incubating-bin-spark35/jar/gluten-*.jar .; \
87+
cp apache-gluten-1.4.0-incubating-bin-spark35/jar/velox-*.jar .; \
88+
rm -rf apache-gluten-1.4.0-incubating-bin-spark35*
89+
90+
# ---- Install sbt 0.13.18 for benchmark compilation ----
91+
RUN wget https://github.com/sbt/sbt/releases/download/v0.13.18/sbt-0.13.18.tgz && \
92+
tar -xzf sbt-0.13.18.tgz -C /usr/local && \
93+
ln -s /usr/local/sbt/bin/sbt /usr/local/bin/sbt && \
94+
rm sbt-0.13.18.tgz
95+
96+
# ---- Clone and compile TPC-DS toolkit ----
97+
WORKDIR /opt
98+
RUN git clone https://github.com/databricks/tpcds-kit.git && \
99+
cd tpcds-kit/tools && \
100+
make OS=LINUX && \
101+
chmod +x dsdgen dsqgen
102+
103+
# ---- Clone the SQL perf library and related files ----
104+
RUN git clone -b delta https://github.com/aws-samples/emr-on-eks-benchmark.git /tmp/emr-on-eks-benchmark
105+
106+
# ---- Build the Databricks SQL perf library ----
107+
RUN cd /tmp/emr-on-eks-benchmark/spark-sql-perf && sbt +package
108+
109+
# ---- Use the compiled Databricks SQL perf library to build benchmark utility ----
110+
RUN cd /tmp/emr-on-eks-benchmark/ && \
111+
mkdir -p /tmp/emr-on-eks-benchmark/benchmark/libs && \
112+
cp /tmp/emr-on-eks-benchmark/spark-sql-perf/target/scala-2.12/*.jar /tmp/emr-on-eks-benchmark/benchmark/libs && \
113+
cd /tmp/emr-on-eks-benchmark/benchmark && sbt assembly
114+
115+
# ---- Create directory for TPC-DS data and set permissions ----
116+
RUN mkdir -p /opt/tpcds-data && \
117+
chown -R ${SPARK_UID}:${SPARK_UID} /opt/tpcds-data
118+
119+
# ---- Copy the built JARs to Spark's examples directory ----
120+
RUN mkdir -p ${SPARK_HOME}/examples/jars/ && \
121+
cp /tmp/emr-on-eks-benchmark/benchmark/target/scala-2.12/*jar ${SPARK_HOME}/examples/jars/ && \
122+
chown -R ${SPARK_UID}:${SPARK_UID} ${SPARK_HOME}/examples
123+
124+
# ---- Clean up build artifacts ----
125+
RUN rm -rf /tmp/emr-on-eks-benchmark
126+
127+
# ---- Spark defaults: JDK17 module opens for Gluten+Velox ----
128+
RUN mkdir -p $SPARK_HOME/conf && cat >> $SPARK_HOME/conf/spark-defaults.conf <<'CONF'
129+
spark.driver.extraJavaOptions=--add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/sun.misc=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-exports=java.base/sun.misc=ALL-UNNAMED --add-exports=java.base/sun.nio.ch=ALL-UNNAMED -Dio.netty.tryReflectionSetAccessible=true
130+
spark.executor.extraJavaOptions=--add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/sun.misc=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-exports=java.base/sun.misc=ALL-UNNAMED --add-exports=java.base/sun.nio.ch=ALL-UNNAMED -Dio.netty.tryReflectionSetAccessible=true
131+
CONF
132+
133+
# Ensure ownership for the non-root user
134+
RUN chown -R spark:spark ${SPARK_HOME}
135+
136+
USER ${SPARK_UID}
137+
WORKDIR ${SPARK_HOME}
138+
139+
# Use standard Spark Kubernetes entrypoint for Spark Operator compatibility
140+
ENTRYPOINT ["/opt/spark/kubernetes/dockerfiles/spark/entrypoint.sh"]

‎analytics/terraform/spark-k8s-operator/examples/benchmark/tpcds-benchmark-spark-gluten-velox/README.md‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
# TPC-DS Benchmark: Spark vs Gluten+Velox
2+
3+
Performance benchmark comparing Native Apache Spark with Gluten+Velox acceleration using 1TB TPC-DS dataset.
4+
5+
## 🚀 Quick Results
6+
7+
- **Overall Performance**: Gluten+Velox **1.72× faster** than native Spark SQL
8+
- **Query Coverage**: 103/104 queries completed (99% compatibility)
9+
- **Peak Speedup**: Up to **5.48× faster** on aggregation-heavy queries
10+
- **Runtime**: 20.9 min (Gluten) vs 33.5 min (Native)
11+
12+
## 📁 Files
13+
14+
| File | Purpose |
15+
|------|---------|
16+
| `tpcds-benchmark-native-c5d.yaml` | Native Spark benchmark job |
17+
| `tpcds-benchmark-gluten-c5d.yaml` | Gluten+Velox benchmark job |
18+
| `Dockerfile-tpcds-gluten-velox` | Docker image with Gluten+Velox |
19+
| `build-tpcds-gluten-image.sh` | Build script for Gluten image |
20+
| `results/` | Benchmark results and comparison data |
21+
22+
## ⚡ Quick Start
23+
24+
1. **Build Gluten image**:
25+
```bash
26+
./build-tpcds-gluten-image.sh v1.0.0
27+
```
28+
29+
2. **Run benchmarks**:
30+
```bash
31+
export S3_BUCKET=your-bucket-name
32+
33+
# Native Spark
34+
envsubst < tpcds-benchmark-native-c5d.yaml | kubectl apply -f -
35+
36+
# Gluten+Velox
37+
envsubst < tpcds-benchmark-gluten-c5d.yaml | kubectl apply -f -
38+
```
39+
40+
3. **Monitor progress**:
41+
```bash
42+
kubectl get sparkapplications -n spark-team-a
43+
```
44+
45+
## 📖 Complete Documentation
46+
47+
For detailed architecture, configuration, and analysis, see:
48+
**[Apache Spark with Gluten+Velox Benchmark Guide](../../../../../../website/docs/benchmarks/spark-operator-benchmark/spark-gluten-velox-benchmark.md)**
49+
50+
## 🔧 Requirements
51+
52+
- **EKS Cluster** with Spark Operator
53+
- **Instance Types**: c5d.12xlarge (8 nodes)
54+
- **Storage**: S3 bucket for data and results
55+
- **Image Registry**: Docker Hub account for custom images

‎analytics/terraform/spark-k8s-operator/examples/benchmark/tpcds-benchmark-spark-gluten-velox/build-tpcds-gluten-image.sh‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
#!/bin/bash
2+
3+
# Build and push TPC-DS Gluten+Velox Docker image
4+
# Usage: ./build-tpcds-gluten-image.sh [version]
5+
6+
set -euo pipefail
7+
8+
# Default version
9+
VERSION=${1:-"v1.0.0"}
10+
IMAGE_NAME="<dockerhubuser>/spark-tpcds-gluten-velox"
11+
FULL_IMAGE="${IMAGE_NAME}:${VERSION}"
12+
13+
echo "Building TPC-DS Gluten+Velox Docker image: ${FULL_IMAGE}"
14+
15+
# Navigate to the spark-jobs directory
16+
cd "$(dirname "$0")/.."
17+
18+
# Build the Docker image (ensure AMD64 for EKS compatibility)
19+
echo "Building Docker image for AMD64 platform..."
20+
docker buildx build \
21+
--platform linux/amd64 \
22+
-f tpcds-benchmark-spark-gluten-velox/Dockerfile-tpcds-gluten-velox \
23+
-t "${FULL_IMAGE}" \
24+
--load .
25+
26+
# Verify the image was built
27+
echo "Verifying image build..."
28+
docker images | grep "${IMAGE_NAME}" | head -1
29+
30+
# Tag as latest
31+
docker tag "${FULL_IMAGE}" "${IMAGE_NAME}:latest"
32+
33+
# Push to Docker Hub
34+
echo "Pushing ${FULL_IMAGE} to Docker Hub..."
35+
docker push "${FULL_IMAGE}"
36+
docker push "${IMAGE_NAME}:latest"
37+
38+
echo "✅ Successfully built and pushed:"
39+
echo " ${FULL_IMAGE}"
40+
echo " ${IMAGE_NAME}:latest"
41+
echo ""
42+
echo "Image is ready for TPC-DS Gluten+Velox benchmarks!"
43+
echo "Update the image reference in tpcds-benchmark-gluten-c5d.yaml if needed."

‎analytics/terraform/spark-k8s-operator/examples/benchmark/tpcds-benchmark-spark-gluten-velox/results/spark-TPCDS-benchamark-result.csv‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
q1-v2.4,5.94626084,5.659786627000001,15.802872012
2+
q10-v2.4,7.785202179,7.21681481,10.612266927999999
3+
q11-v2.4,24.360113586,23.624084258,38.51371252
4+
q12-v2.4,2.207271293,2.026684241,2.896425813
5+
q13-v2.4,8.739074823000001,8.725879449,12.502888361
6+
q14a-v2.4,61.010627583,60.847721965,82.991309778
7+
q14b-v2.4,54.538167861000005,53.836341786999995,57.41624686
8+
q15-v2.4,6.691632064,6.52784757,7.073098474
9+
q16-v2.4,31.310521618,31.13781921,37.846789376000004
10+
q17-v2.4,6.565989421,6.493043734,7.263143183
11+
q18-v2.4,9.798672157999999,9.201197448999999,10.566477897999999
12+
q19-v2.4,3.707617141,3.657258468,4.149846525
13+
q2-v2.4,17.651374781999998,17.569390294,39.891847741999996
14+
q20-v2.4,2.1580954820000002,2.053724967,2.378421913
15+
q21-v2.4,1.760386188,1.601910097,1.833102231
16+
q22-v2.4,7.750308714,7.719488933,8.597467728
17+
q23a-v2.4,113.96456687199999,112.24181698299999,119.327494299
18+
q23b-v2.4,146.067997469,145.956947403,149.478953829
19+
q24a-v2.4,76.541518733,74.460606736,82.89471763
20+
q24b-v2.4,71.58643944299999,71.266330549,74.44054069900001
21+
q25-v2.4,5.478208592,5.28935088,5.691166965000001
22+
q26-v2.4,4.790580857,4.479870693,4.9208097529999995
23+
q27-v2.4,6.538443271,6.489804941,7.449839526
24+
q28-v2.4,56.826696404,55.729690393,57.105219396
25+
q29-v2.4,17.233819574,17.061604428,17.679751378000002
26+
q3-v2.4,4.330919975,4.141845284,6.1803359559999995
27+
q30-v2.4,15.683365889000001,15.3663024,18.36470318
28+
q31-v2.4,10.747915034,10.436555844,11.240571156
29+
q32-v2.4,2.008987504,1.801960059,2.6314307429999997
30+
q33-v2.4,3.681019536,3.63297235,3.937263876
31+
q34-v2.4,6.425746022,6.3199003970000005,8.194717962
32+
q35-v2.4,15.186294680000001,15.128760948,15.473272489000001
33+
q36-v2.4,5.715885706,5.575214997,5.718460686
34+
q37-v2.4,7.845310493,7.541962274,7.942951332
35+
q38-v2.4,16.792041047999998,16.612403141999998,17.015626718
36+
q39a-v2.4,5.519249407,5.3414041800000005,5.541598667000001
37+
q39b-v2.4,4.920771315,4.686805926,4.963379837000001
38+
q4-v2.4,52.983533734999995,51.57699195,90.131217193
39+
q40-v2.4,15.584617286999999,13.922560982999999,16.003383363
40+
q41-v2.4,1.158515054,1.1257108720000002,1.162993533
41+
q42-v2.4,1.4626256530000001,1.450435421,1.622814533
42+
q43-v2.4,4.377313459,4.304515255999999,5.422134705
43+
q44-v2.4,22.738412903,22.21861899,23.733121074
44+
q45-v2.4,6.386203303,6.242595642,6.546131038
45+
q46-v2.4,12.607582845000001,11.776613172,12.664956999
46+
q47-v2.4,9.113369761,8.605047993,9.244677061
47+
q48-v2.4,7.640361367000001,7.389772094,8.106714349
48+
q49-v2.4,25.893803815,24.666732896000003,26.469213939
49+
q5-v2.4,19.136202502,19.047259125,31.367564628
50+
q50-v2.4,38.449515392,38.220949495,39.037645420000004
51+
q51-v2.4,11.955113020999999,11.850860288,12.379503586999999
52+
q52-v2.4,1.538365164,1.518201403,1.655990265
53+
q53-v2.4,4.764723418,4.749154349,4.799754913
54+
q54-v2.4,5.833125367,5.7276672500000005,5.920191417
55+
q55-v2.4,1.520315015,1.376934433,1.57835255
56+
q56-v2.4,3.388743524,3.2894568939999997,3.54169243
57+
q57-v2.4,6.512053496,6.078336805,6.630835337000001
58+
q58-v2.4,3.8244755660000003,3.810702102,4.271772668
59+
q59-v2.4,17.683703385,17.247569358,17.773919567
60+
q6-v2.4,9.788026209,9.485055146,10.337533318
61+
q60-v2.4,3.65224822,3.510164653,4.422574330000001
62+
q61-v2.4,4.67638026,4.308950298,4.744657076
63+
q62-v2.4,9.430475892,9.071840415,9.724780208
64+
q63-v2.4,4.638885255,4.515213095,5.094661889
65+
q64-v2.4,62.065105147000004,61.556414126,62.596965011
66+
q65-v2.4,17.535384221,17.414213904,17.567658597
67+
q66-v2.4,6.48112928,6.388568939,6.7509098540000005
68+
q67-v2.4,72.84689828100001,72.393780624,73.450265899
69+
q68-v2.4,6.469640580999999,6.369155694,6.518682285
70+
q69-v2.4,6.942510388,6.8095949330000005,7.193923431999999
71+
q7-v2.4,7.233630193,6.804396832,11.30614958
72+
q70-v2.4,8.573218545,8.475844862,8.623801283
73+
q71-v2.4,2.688361728,2.484996292,3.0133267040000002
74+
q72-v2.4,20.894096345999998,20.482196322,21.045875932999998
75+
q73-v2.4,3.693322711,3.63634419,3.827752448
76+
q74-v2.4,20.931781098,20.687873156000002,21.45232352
77+
q75-v2.4,40.650773389,40.453087667,41.382013243
78+
q76-v2.4,25.894196119,20.87342284,25.922988199
79+
q77-v2.4,2.050490979,1.9600629360000001,2.111888582
80+
q78-v2.4,63.851592657,63.443333355,64.298522057
81+
q79-v2.4,4.234859658,4.205485125,4.416884526
82+
q8-v2.4,6.114229035,5.725227982000001,6.330723141
83+
q80-v2.4,23.955926378,23.690872736000003,24.655474737
84+
q81-v2.4,18.344921306,18.00811751,18.970320582
85+
q82-v2.4,11.482953277,11.401857905,11.553987182
86+
q83-v2.4,1.761805534,1.711199573,2.137564167
87+
q84-v2.4,7.947454632,7.831167682,8.191072596
88+
q85-v2.4,16.057549975,15.608865776,16.769602543
89+
q86-v2.4,3.420942289,3.2684540109999998,3.478910715
90+
q87-v2.4,16.901443208,16.393007666000003,16.953879082
91+
q88-v2.4,50.648096805,50.610638087,51.44157913
92+
q89-v2.4,5.1335403699999995,4.854170627999999,5.621945603
93+
q9-v2.4,48.077736126999994,42.839557381,91.779986623
94+
q90-v2.4,11.960054941000001,11.674960765,12.522986462999999
95+
q91-v2.4,3.188492804,3.014570649,3.364358154
96+
q92-v2.4,1.5872336379999998,1.5434971020000001,1.59949665
97+
q93-v2.4,80.04224533,79.842919392,80.64532613
98+
q94-v2.4,22.096856194,21.057359108,22.788797229
99+
q95-v2.4,50.008450214,49.467758696,51.264759675
100+
q96-v2.4,9.108196266,8.813864805,9.661397757000001
101+
q97-v2.4,18.739727573,18.388118591999998,18.908132332
102+
q98-v2.4,2.671823464,2.633760132,2.753976506
103+
q99-v2.4,9.376862037,9.27744431,10.459242664
104+
ss_max-v2.4,11.495081793999999,11.439207726000001,11.620844539

0 commit comments

Comments
 (0)