Skip to content

Commit 36f85c6

Browse files
thalesmgthalesmg
committed
feat: add spark
1 parent a1c321a commit 36f85c6

18 files changed

Lines changed: 4437 additions & 0 deletions

‎spark/.pyiceberg.yaml‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.
18+
#
19+
catalog:
20+
default:
21+
uri: http://rest:8181
22+
s3.endpoint: http://minio:9000
23+
s3.access-key-id: admin
24+
s3.secret-access-key: password

‎spark/Dockerfile‎

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one or more
2+
# contributor license agreements. See the NOTICE file distributed with
3+
# this work for additional information regarding copyright ownership.
4+
# The ASF licenses this file to You under the Apache License, Version 2.0
5+
# (the "License"); you may not use this file except in compliance with
6+
# the License. You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
#
16+
# syntax=docker/dockerfile:1
17+
FROM python:3.10-bullseye
18+
19+
RUN apt-get update && \
20+
apt-get install -y --no-install-recommends \
21+
sudo \
22+
curl \
23+
vim \
24+
unzip \
25+
openjdk-17-jdk \
26+
build-essential \
27+
software-properties-common \
28+
ssh && \
29+
apt-get clean && \
30+
rm -rf /var/lib/apt/lists/*
31+
32+
# Install Jupyter and other python deps
33+
COPY requirements.txt .
34+
RUN pip3 install -r requirements.txt
35+
36+
# Add scala kernel via spylon-kernel
37+
RUN python3 -m spylon_kernel install
38+
39+
# Download and install IJava jupyter kernel
40+
RUN curl https://github.com/SpencerPark/IJava/releases/download/v1.3.0/ijava-1.3.0.zip -Lo ijava-1.3.0.zip \
41+
&& unzip ijava-1.3.0.zip \
42+
&& python3 install.py --sys-prefix \
43+
&& rm ijava-1.3.0.zip
44+
45+
# Optional env variables
46+
ENV SPARK_HOME=${SPARK_HOME:-"/opt/spark"}
47+
ENV PYTHONPATH=$SPARK_HOME/python:$SPARK_HOME/python/lib/py4j-0.10.9.7-src.zip:$PYTHONPATH
48+
49+
WORKDIR ${SPARK_HOME}
50+
51+
ENV SPARK_VERSION=3.5.5
52+
ENV SPARK_MAJOR_VERSION=3.5
53+
ENV ICEBERG_VERSION=1.8.1
54+
55+
# Download spark
56+
RUN mkdir -p ${SPARK_HOME} \
57+
&& curl https://dlcdn.apache.org/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop3.tgz -o spark-${SPARK_VERSION}-bin-hadoop3.tgz \
58+
&& tar xvzf spark-${SPARK_VERSION}-bin-hadoop3.tgz --directory /opt/spark --strip-components 1 \
59+
&& rm -rf spark-${SPARK_VERSION}-bin-hadoop3.tgz
60+
61+
# Download iceberg spark runtime
62+
RUN curl https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-spark-runtime-${SPARK_MAJOR_VERSION}_2.12/${ICEBERG_VERSION}/iceberg-spark-runtime-${SPARK_MAJOR_VERSION}_2.12-${ICEBERG_VERSION}.jar -Lo /opt/spark/jars/iceberg-spark-runtime-${SPARK_MAJOR_VERSION}_2.12-${ICEBERG_VERSION}.jar
63+
64+
# Download AWS bundle
65+
RUN curl -s https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-aws-bundle/${ICEBERG_VERSION}/iceberg-aws-bundle-${ICEBERG_VERSION}.jar -Lo /opt/spark/jars/iceberg-aws-bundle-${ICEBERG_VERSION}.jar
66+
67+
# Download GCP bundle
68+
RUN curl -s https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-gcp-bundle/${ICEBERG_VERSION}/iceberg-gcp-bundle-${ICEBERG_VERSION}.jar -Lo /opt/spark/jars/iceberg-gcp-bundle-${ICEBERG_VERSION}.jar
69+
70+
# Download Azure bundle
71+
RUN curl -s https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-azure-bundle/${ICEBERG_VERSION}/iceberg-azure-bundle-${ICEBERG_VERSION}.jar -Lo /opt/spark/jars/iceberg-azure-bundle-${ICEBERG_VERSION}.jar
72+
73+
# Install AWS CLI
74+
RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" \
75+
&& unzip awscliv2.zip \
76+
&& sudo ./aws/install \
77+
&& rm awscliv2.zip \
78+
&& rm -rf aws/
79+
80+
# Add iceberg spark runtime jar to IJava classpath
81+
ENV IJAVA_CLASSPATH=/opt/spark/jars/*
82+
83+
RUN mkdir -p /home/iceberg/data \
84+
&& curl https://data.cityofnewyork.us/resource/tg4x-b46p.json > /home/iceberg/data/nyc_film_permits.json \
85+
&& curl https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2022-04.parquet -o /home/iceberg/data/yellow_tripdata_2022-04.parquet \
86+
&& curl https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2022-03.parquet -o /home/iceberg/data/yellow_tripdata_2022-03.parquet \
87+
&& curl https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2022-02.parquet -o /home/iceberg/data/yellow_tripdata_2022-02.parquet \
88+
&& curl https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2022-01.parquet -o /home/iceberg/data/yellow_tripdata_2022-01.parquet \
89+
&& curl https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2021-12.parquet -o /home/iceberg/data/yellow_tripdata_2021-12.parquet \
90+
&& curl https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2021-11.parquet -o /home/iceberg/data/yellow_tripdata_2021-11.parquet \
91+
&& curl https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2021-10.parquet -o /home/iceberg/data/yellow_tripdata_2021-10.parquet \
92+
&& curl https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2021-09.parquet -o /home/iceberg/data/yellow_tripdata_2021-09.parquet \
93+
&& curl https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2021-08.parquet -o /home/iceberg/data/yellow_tripdata_2021-08.parquet \
94+
&& curl https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2021-07.parquet -o /home/iceberg/data/yellow_tripdata_2021-07.parquet \
95+
&& curl https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2021-06.parquet -o /home/iceberg/data/yellow_tripdata_2021-06.parquet \
96+
&& curl https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2021-05.parquet -o /home/iceberg/data/yellow_tripdata_2021-05.parquet \
97+
&& curl https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2021-04.parquet -o /home/iceberg/data/yellow_tripdata_2021-04.parquet
98+
99+
RUN mkdir -p /home/iceberg/localwarehouse /home/iceberg/notebooks /home/iceberg/warehouse /home/iceberg/spark-events /home/iceberg
100+
COPY notebooks/ /home/iceberg/notebooks
101+
102+
# Add a notebook command
103+
RUN echo '#! /bin/sh' >> /bin/notebook \
104+
&& echo 'export PYSPARK_DRIVER_PYTHON=jupyter-notebook' >> /bin/notebook \
105+
&& echo "export PYSPARK_DRIVER_PYTHON_OPTS=\"--notebook-dir=/home/iceberg/notebooks --ip='*' --NotebookApp.token='' --NotebookApp.password='' --port=8888 --no-browser --allow-root\"" >> /bin/notebook \
106+
&& echo "pyspark" >> /bin/notebook \
107+
&& chmod u+x /bin/notebook
108+
109+
# Add a pyspark-notebook command (alias for notebook command for backwards-compatibility)
110+
RUN echo '#! /bin/sh' >> /bin/pyspark-notebook \
111+
&& echo 'export PYSPARK_DRIVER_PYTHON=jupyter-notebook' >> /bin/pyspark-notebook \
112+
&& echo "export PYSPARK_DRIVER_PYTHON_OPTS=\"--notebook-dir=/home/iceberg/notebooks --ip='*' --NotebookApp.token='' --NotebookApp.password='' --port=8888 --no-browser --allow-root\"" >> /bin/pyspark-notebook \
113+
&& echo "pyspark" >> /bin/pyspark-notebook \
114+
&& chmod u+x /bin/pyspark-notebook
115+
116+
RUN mkdir -p /root/.ipython/profile_default/startup
117+
COPY ipython/startup/00-prettytables.py /root/.ipython/profile_default/startup
118+
COPY ipython/startup/README /root/.ipython/profile_default/startup
119+
120+
COPY spark-defaults.conf /opt/spark/conf
121+
ENV PATH="/opt/spark/sbin:/opt/spark/bin:${PATH}"
122+
123+
RUN chmod u+x /opt/spark/sbin/* && \
124+
chmod u+x /opt/spark/bin/*
125+
126+
COPY .pyiceberg.yaml /root/.pyiceberg.yaml
127+
128+
COPY entrypoint.sh .
129+
130+
ENTRYPOINT ["./entrypoint.sh"]
131+
CMD ["notebook"]

‎spark/entrypoint.sh‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
#!/bin/bash
2+
#
3+
# Licensed to the Apache Software Foundation (ASF) under one
4+
# or more contributor license agreements. See the NOTICE file
5+
# distributed with this work for additional information
6+
# regarding copyright ownership. The ASF licenses this file
7+
# to you under the Apache License, Version 2.0 (the
8+
# "License"); you may not use this file except in compliance
9+
# with the License. You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing,
14+
# software distributed under the License is distributed on an
15+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16+
# KIND, either express or implied. See the License for the
17+
# specific language governing permissions and limitations
18+
# under the License.
19+
20+
start-master.sh -p 7077
21+
start-worker.sh spark://spark-iceberg:7077
22+
start-history-server.sh
23+
start-thriftserver.sh --driver-java-options "-Dderby.system.home=/tmp/derby"
24+
25+
# Entrypoint, for example notebook, pyspark or spark-sql
26+
if [[ $# -gt 0 ]] ; then
27+
eval "$1"
28+
fi

‎spark/ipython/startup/00-prettytables.py‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.
18+
19+
from prettytable import PrettyTable
20+
from IPython.core.magic import register_line_cell_magic
21+
22+
class DFTable(PrettyTable):
23+
def __repr__(self):
24+
return self.get_string()
25+
26+
def _repr_html_(self):
27+
return self.get_html_string()
28+
29+
def _row_as_table(df):
30+
cols = df.columns
31+
32+
t = DFTable()
33+
t.field_names = ["Column", "Value"]
34+
t.align = "r"
35+
row = df.limit(1).collect()[0].asDict()
36+
for col in cols:
37+
t.add_row([ col, row[col] ])
38+
39+
return t
40+
41+
def _to_table(df, num_rows=100):
42+
cols = df.columns
43+
44+
t = DFTable()
45+
t.field_names = cols
46+
t.align = "r"
47+
for row in df.limit(num_rows).collect():
48+
d = row.asDict()
49+
t.add_row([ d[col] for col in cols ])
50+
51+
return t
52+
53+
import re
54+
import sys
55+
from argparse import ArgumentParser
56+
parser = ArgumentParser()
57+
parser.add_argument("--limit", help="Number of lines to return", type=int, default=100)
58+
parser.add_argument("--var", help="Variable name to hold the dataframe", type=str)
59+
60+
@register_line_cell_magic
61+
def sql(line, cell=None):
62+
"""Spark SQL magic
63+
"""
64+
from pyspark.sql import SparkSession
65+
spark = SparkSession.builder.appName("Jupyter").getOrCreate()
66+
if cell is None:
67+
return _to_table(spark.sql(line))
68+
elif line:
69+
df = spark.sql(cell)
70+
71+
(args, others) = parser.parse_known_args([ arg for arg in re.split("\s+", line) if arg ])
72+
73+
if args.var:
74+
setattr(sys.modules[__name__], args.var, df)
75+
76+
if args.limit == 1:
77+
return _row_as_table(df)
78+
else:
79+
return _to_table(df, num_rows=args.limit)
80+
else:
81+
return _to_table(spark.sql(cell))

‎spark/ipython/startup/README‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
This is the IPython startup directory
2+
3+
.py and .ipy files in this directory will be run *prior* to any code or files specified
4+
via the exec_lines or exec_files configurables whenever you load this profile.
5+
6+
Files will be run in lexicographical order, so you can control the execution order of files
7+
with a prefix, e.g.::
8+
9+
00-first.py
10+
50-middle.py
11+
99-last.ipy

0 commit comments

Comments
 (0)