Skip to content

Commit f8866af

Browse files
AdeelHAdeelH
authored
Replace setup.pys with pyproject.tomls (#2317)
1 parent b2b85db commit f8866af

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

49 files changed

+1636
-751
lines changed

‎.readthedocs.yml

+3-19
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,9 @@ build:
1010
os: ubuntu-22.04
1111
tools:
1212
python: "3.11"
13+
jobs:
14+
post_install:
15+
- scripts/setup_dev_env --rtd
1316

1417
# Build documentation in the docs/ directory with Sphinx
1518
sphinx:
@@ -24,25 +27,6 @@ sphinx:
2427
# formats:
2528
# - pdf
2629

27-
# Optionally set the version of Python and requirements required to build your docs
28-
python:
29-
install:
30-
- requirements: docs/requirements.txt
31-
- method: pip
32-
path: rastervision_pipeline/
33-
- method: pip
34-
path: rastervision_aws_s3/
35-
- method: pip
36-
path: rastervision_aws_batch/
37-
- method: pip
38-
path: rastervision_core/
39-
- method: pip
40-
path: rastervision_pytorch_learner/
41-
- method: pip
42-
path: rastervision_pytorch_backend/
43-
- method: pip
44-
path: rastervision_aws_sagemaker/
45-
4630
# https://docs.readthedocs.io/en/stable/config-file/v2.html#search
4731
search:
4832
ranking:

‎Dockerfile

+61-95
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ ARG UBUNTU_VERSION
44

55
########################################################################
66

7-
FROM nvidia/cuda:${CUDA_VERSION}-cudnn8-runtime-ubuntu${UBUNTU_VERSION} as thinbuild
7+
FROM nvidia/cuda:${CUDA_VERSION}-cudnn8-runtime-ubuntu${UBUNTU_VERSION} AS thinbuild
88

99
ARG PYTHON_VERSION=3.11
1010

@@ -22,12 +22,12 @@ RUN --mount=type=cache,target=/var/cache/apt apt update && \
2222

2323
########################################################################
2424

25-
FROM nvidia/cuda:${CUDA_VERSION}-cudnn8-runtime-ubuntu${UBUNTU_VERSION} as fullbuild
25+
FROM nvidia/cuda:${CUDA_VERSION}-cudnn8-runtime-ubuntu${UBUNTU_VERSION} AS fullbuild
2626

2727
ARG TARGETPLATFORM
2828
ARG PYTHON_VERSION=3.11
2929

30-
# wget: needed below to install conda
30+
# wget: needed below to install mamba
3131
# build-essential: installs gcc which is needed to install some deps like rasterio
3232
# libGL1: needed to avoid following error when using cv2
3333
# ImportError: libGL.so.1: cannot open shared object file: No such file or directory
@@ -45,27 +45,32 @@ RUN case ${TARGETPLATFORM} in \
4545
RUN curl -fsSL https://deb.nodesource.com/node_16.x | bash - && \
4646
apt-get install -y nodejs
4747

48-
# Install Python and conda/mamba (mamba installs conda as well)
48+
# Install Python and mamba
4949
RUN wget -q -O ~/micromamba.sh https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-$(cat /root/linux_arch).sh && \
5050
chmod +x ~/micromamba.sh && \
5151
bash ~/micromamba.sh -b -p /opt/conda && \
5252
rm ~/micromamba.sh
53-
ENV PATH /opt/conda/bin:$PATH
54-
ENV LD_LIBRARY_PATH /opt/conda/lib/:$LD_LIBRARY_PATH
55-
RUN mamba init
56-
RUN mamba install -y python=${PYTHON_VERSION}
57-
RUN python -m pip install --upgrade pip
53+
ENV PATH=/opt/conda/bin:/opt/conda/condabin:$PATH
54+
ENV LD_LIBRARY_PATH=/opt/conda/lib/:$LD_LIBRARY_PATH
55+
56+
RUN mamba init bash
57+
RUN mamba create -n rv python=${PYTHON_VERSION} -y
58+
RUN echo "mamba activate rv" >> ~/.bashrc
59+
60+
# Make RUN commands use the new mamba environment:
61+
SHELL ["mamba", "run", "-n", "rv", "bash", "-l", "-c"]
5862

5963
# env variable required by uv
60-
ENV CONDA_PREFIX=/opt/conda
64+
ENV CONDA_PREFIX=/opt/conda/envs/rv/bin/
65+
RUN python -m pip install --upgrade pip
6166
RUN pip install uv
6267

6368
# We need to install GDAL first to install Rasterio on non-AMD64 architectures.
6469
# The Rasterio wheels contain GDAL in them, but they are only built for AMD64 now.
65-
RUN mamba update mamba -y && mamba install -y -c conda-forge gdal=3.6.3
70+
RUN mamba install -y -c conda-forge gdal=3.6.3
6671
ENV GDAL_DATA=/opt/conda/lib/python${PYTHON_VERSION}/site-packages/rasterio/gdal_data/
6772
# Needed for GDAL 3.0
68-
ENV PROJ_LIB /opt/conda/share/proj/
73+
ENV PROJ_LIB=/opt/conda/share/proj/
6974

7075
# This is to prevent the following error when starting the container.
7176
# bash: /opt/conda/lib/libtinfo.so.6: no version information available (required by bash)
@@ -86,99 +91,23 @@ FROM ${BUILD_TYPE:-fullbuild} AS final_stage
8691

8792
ARG TARGETARCH
8893

89-
ENV LC_ALL C.UTF-8
90-
ENV LANG C.UTF-8
94+
ENV LC_ALL=C.UTF-8
95+
ENV LANG=C.UTF-8
9196

9297
WORKDIR /opt/src/
9398

9499
#------------------------------------------------------------------------
95100

96-
# Ideally we'd just pip install each package, but if we do that, then
97-
# a lot of the image will have to be re-built each time we make a
98-
# change to the code. So, we split the install into installing all the
99-
# requirements in bunches (filtering out any prefixed with
100-
# rastervision_*), and then copy over the source code. The
101-
# dependencies are installed in bunches rather than package-by-package
102-
# or on a per-RV component basis to reduce the build time, the number
103-
# of layers, and the overall image size, and to reduce churn
104-
# (installing and uninstalling of Python packages during the build).
105-
#
106-
# The bunches are heuristic and are meant to keep the heaviest and/or
107-
# least-frequently-changing dependencies before the more variable
108-
# ones. At time of writing, the amount of image size attributable to
109-
# PyTorch (and the amount of image size overall) is heavily dominated
110-
# by PyTorch, so it is first.
111-
112-
# Install requirements.
113-
# -E "^\s*$|^#|rastervision_*" means exclude blank lines, comment lines,
114-
# and rastervision plugins.
115-
116-
COPY ./rastervision_pytorch_learner/requirements.txt /opt/src/pytorch-requirements.txt
117-
RUN --mount=type=cache,target=/root/.cache/pip cat pytorch-requirements.txt | sort | uniq > all-requirements.txt && \
118-
uv pip install $(grep -ivE "^\s*$|^#|rastervision_*" all-requirements.txt) && \
119-
rm all-requirements.txt
120-
121-
COPY ./rastervision_aws_batch/requirements.txt /opt/src/batch-requirements.txt
122-
COPY ./rastervision_aws_s3/requirements.txt /opt/src/s3-requirements.txt
123-
COPY ./rastervision_core/requirements.txt /opt/src/core-requirements.txt
124-
125-
# Pip wheels for triangle are missing for ARM64 architectures and building
126-
# from source fails, so we skip it.
127-
RUN if [ "${TARGETARCH}" = "arm64" ]; \
128-
then sed -i '/^triangle.*$/d' /opt/src/core-requirements.txt; fi
129-
130-
COPY ./rastervision_gdal_vsi/requirements.txt /opt/src/gdal-requirements.txt
131-
COPY ./rastervision_pipeline/requirements.txt /opt/src/pipeline-requirements.txt
132-
COPY ./rastervision_aws_sagemaker/requirements.txt /opt/src/sagemaker-requirements.txt
133-
COPY ./requirements-dev.txt /opt/src/requirements-dev.txt
134-
RUN --mount=type=cache,target=/root/.cache/pip cat \
135-
/opt/src/batch-requirements.txt \
136-
/opt/src/s3-requirements.txt \
137-
/opt/src/core-requirements.txt \
138-
/opt/src/gdal-requirements.txt \
139-
/opt/src/pipeline-requirements.txt \
140-
/opt/src/sagemaker-requirements.txt \
141-
/opt/src/requirements-dev.txt \
142-
| sort | uniq > all-requirements.txt && \
143-
uv pip install $(grep -ivE "^\s*$|^#|rastervision_*" all-requirements.txt) && \
144-
rm all-requirements.txt
145-
146-
#########################
147-
# Docs
148-
#########################
149-
# Install docs/requirements.txt
150-
COPY ./docs/requirements.txt /opt/src/docs/pandoc-requirements.txt
151-
152101
# Install pandoc, needed for rendering notebooks
153102
# Get latest release link from here: https://github.com/jgm/pandoc/releases
154-
RUN --mount=type=cache,target=/root/.cache/pip uv pip install -r docs/pandoc-requirements.txt && \
155-
wget https://github.com/jgm/pandoc/releases/download/3.1.12.2/pandoc-3.1.12.2-1-${TARGETARCH}.deb && \
156-
dpkg -i pandoc-3.1.12.2-1-${TARGETARCH}.deb && rm pandoc-3.1.12.2-1-${TARGETARCH}.deb
103+
RUN --mount=type=cache,target=/root/.cache/pip wget \
104+
https://github.com/jgm/pandoc/releases/download/3.1.12.2/pandoc-3.1.12.2-1-${TARGETARCH}.deb && \
105+
dpkg -i pandoc-3.1.12.2-1-${TARGETARCH}.deb && \
106+
rm pandoc-3.1.12.2-1-${TARGETARCH}.deb
157107

158108
#------------------------------------------------------------------------
159109

160-
# needed for this image to be used by the AWS SageMaker PyTorch Estimator
161-
RUN uv pip install sagemaker_pytorch_training==2.8.1
162-
ENV SAGEMAKER_TRAINING_MODULE=sagemaker_pytorch_container.training:main
163-
164-
# Install a onnxruntime-gpu version compatible with CUDA 12. Specifying
165-
# --extra-index-url in requirements.txt seems to cause problems with the
166-
# RTD build.
167-
RUN if [ "${TARGETARCH}" != "arm64" ]; then \
168-
uv pip install onnxruntime-gpu==1.19; fi
169-
170-
#------------------------------------------------------------------------
171-
172-
ENV PYTHONPATH=/opt/src:$PYTHONPATH
173-
ENV PYTHONPATH=/opt/src/rastervision_aws_batch/:$PYTHONPATH
174-
ENV PYTHONPATH=/opt/src/rastervision_aws_s3/:$PYTHONPATH
175-
ENV PYTHONPATH=/opt/src/rastervision_core/:$PYTHONPATH
176-
ENV PYTHONPATH=/opt/src/rastervision_gdal_vsi/:$PYTHONPATH
177-
ENV PYTHONPATH=/opt/src/rastervision_pipeline/:$PYTHONPATH
178-
ENV PYTHONPATH=/opt/src/rastervision_aws_sagemaker/:$PYTHONPATH
179-
ENV PYTHONPATH=/opt/src/rastervision_pytorch_backend/:$PYTHONPATH
180-
ENV PYTHONPATH=/opt/src/rastervision_pytorch_learner/:$PYTHONPATH
181-
110+
COPY ./requirements.txt /opt/src/requirements.txt
182111
COPY scripts /opt/src/scripts/
183112
COPY scripts/rastervision /usr/local/bin/rastervision
184113
COPY tests /opt/src/tests/
@@ -195,4 +124,41 @@ COPY ./rastervision_aws_sagemaker/ /opt/src/rastervision_aws_sagemaker/
195124
COPY ./rastervision_pytorch_backend/ /opt/src/rastervision_pytorch_backend/
196125
COPY ./rastervision_pytorch_learner/ /opt/src/rastervision_pytorch_learner/
197126

127+
# remove rastervision dependencies
128+
RUN sed -i '/^rastervision/d' requirements.txt
129+
130+
# Pip wheels for triangle are missing for ARM64 architectures and building
131+
# from source fails, so we skip it.
132+
RUN if [ "${TARGETARCH}" = "arm64" ]; \
133+
then sed -i '/^triangle.*$/d' /opt/src/requirements.txt; fi
134+
135+
RUN --mount=type=cache,target=/root/.cache/pip uv pip sync /opt/src/requirements.txt
136+
137+
RUN uv pip install -e /opt/src/rastervision_pipeline/ --no-deps
138+
RUN uv pip install -e /opt/src/rastervision_aws_s3/ --no-deps
139+
RUN uv pip install -e /opt/src/rastervision_aws_batch/ --no-deps
140+
RUN uv pip install -e /opt/src/rastervision_core/ --no-deps
141+
RUN uv pip install -e /opt/src/rastervision_pytorch_learner/ --no-deps
142+
RUN uv pip install -e /opt/src/rastervision_pytorch_backend/ --no-deps
143+
RUN uv pip install -e /opt/src/rastervision_aws_sagemaker/ --no-deps
144+
RUN uv pip install -e /opt/src/rastervision_gdal_vsi/ --no-deps
145+
146+
147+
#------------------------------------------------------------------------
148+
149+
# needed for this image to be used by the AWS SageMaker PyTorch Estimator
150+
RUN uv pip install sagemaker_pytorch_training==2.8.1
151+
ENV SAGEMAKER_TRAINING_MODULE=sagemaker_pytorch_container.training:main
152+
153+
#------------------------------------------------------------------------
154+
155+
# Install a onnxruntime-gpu version compatible with CUDA 12. Specifying
156+
# --extra-index-url in requirements.txt seems to cause problems with the
157+
# RTD build.
158+
RUN if [ "${TARGETARCH}" != "arm64" ]; then \
159+
uv pip install onnxruntime-gpu==1.19; fi
160+
161+
#------------------------------------------------------------------------
162+
163+
ENTRYPOINT ["mamba", "run", "--no-capture-output", "-n", "rv", "bash", "-l", "-c"]
198164
CMD ["bash"]

‎README.md

+7
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,13 @@ You can ask questions and talk to developers (let us know what you're working on
5959
* [Discussion Forum](https://github.com/azavea/raster-vision/discussions)
6060
* [Mailing List](https://groups.google.com/forum/#!forum/raster-vision)
6161

62+
## Developing
63+
64+
To set up the development environment:
65+
- For and clone the repo and navigate to it.
66+
- Create and activate a new Python virtual environment via your environment manager of choice (`mamba`, `uv`, `pyenv`, etc.).
67+
- Run `scripts/setup_dev_env.sh` to install all Raster Vision plugins in editable mode along with all the dependencies.
68+
6269
## Contributing
6370

6471
*For more information, see [Contributing](https://docs.rastervision.io/en/stable/CONTRIBUTING.html).*

‎docs/release.rst

+1-8
Original file line numberDiff line numberDiff line change
@@ -69,13 +69,6 @@ Minor or Major Version Release
6969
docker push quay.io/azavea/raster-vision:pytorch-<version>
7070
7171
#. Make a GitHub `tag <https://github.com/azavea/raster-vision/tags>`_ and `release <https://github.com/azavea/raster-vision/releases>`_ using the previous release as a template.
72-
#. Remove artifacts from previous builds. From the repo root:
73-
74-
.. code-block:: console
75-
76-
rm -rf build/ dist/ *.egg-info
77-
rm -rf rastervision_*/build rastervision_*/dist rastervision_*/*.egg-info
78-
7972
#. Publish all packages to PyPI. This step requires `twine <https://twine.readthedocs.io/en/stable/>`__ which you can install with
8073

8174
.. code-block:: console
@@ -100,7 +93,7 @@ Minor or Major Version Release
10093

10194
.. code-block:: console
10295
103-
scripts/pypi_build
96+
scripts/build_packages
10497
10598
Publish to TestPyPI:
10699

‎docs/requirements.txt

-13
This file was deleted.

‎pyproject.toml

+82
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
[build-system]
2+
requires = ["setuptools", "setuptools_scm", "wheel"]
3+
build-backend = "setuptools.build_meta"
4+
5+
[project]
6+
name = "rastervision"
7+
version = "0.31.2-dev"
8+
description = "An open source framework for deep learning on satellite and aerial imagery."
9+
readme = "README.md"
10+
authors = [{ name = "Azavea", email = "[email protected]" }]
11+
license = { text = "Apache License 2.0" }
12+
classifiers = [
13+
"Intended Audience :: Developers",
14+
"License :: OSI Approved :: Apache Software License",
15+
"Programming Language :: Python",
16+
"Programming Language :: Python :: 3",
17+
]
18+
keywords = [
19+
"computer-vision",
20+
"deep-learning",
21+
"earth-observation",
22+
"machine-learning",
23+
"ml",
24+
"geospatial",
25+
"geospatial-machine-learning",
26+
"geospatial-processing",
27+
"raster",
28+
]
29+
dependencies = [
30+
"rastervision_pipeline @ file://${PROJECT_ROOT}/rastervision_pipeline/dist/rastervision_pipeline-0.31.2.dev0-py3-none-any.whl",
31+
"rastervision_aws_s3 @ file://${PROJECT_ROOT}/rastervision_aws_s3/dist/rastervision_aws_s3-0.31.2.dev0-py3-none-any.whl",
32+
"rastervision_aws_batch @ file://${PROJECT_ROOT}/rastervision_aws_batch/dist/rastervision_aws_batch-0.31.2.dev0-py3-none-any.whl",
33+
"rastervision_core @ file://${PROJECT_ROOT}/rastervision_core/dist/rastervision_core-0.31.2.dev0-py3-none-any.whl",
34+
"rastervision_pytorch_learner @ file://${PROJECT_ROOT}/rastervision_pytorch_learner/dist/rastervision_pytorch_learner-0.31.2.dev0-py3-none-any.whl",
35+
"rastervision_pytorch_backend @ file://${PROJECT_ROOT}/rastervision_pytorch_backend/dist/rastervision_pytorch_backend-0.31.2.dev0-py3-none-any.whl",
36+
]
37+
38+
[tool.setuptools.dynamic]
39+
dependencies = { file = ["requirements.in"] }
40+
41+
[project.optional-dependencies]
42+
debugging = []
43+
full = [
44+
"rastervision_aws_sagemaker @ file://${PROJECT_ROOT}/rastervision_aws_sagemaker/dist/rastervision_aws_sagemaker-0.31.2.dev0-py3-none-any.whl",
45+
"rastervision_gdal_vsi @ file://${PROJECT_ROOT}/rastervision_gdal_vsi/dist/rastervision_gdal_vsi-0.31.2.dev0-py3-none-any.whl",
46+
]
47+
dev = [
48+
"awscli==1.33.40",
49+
"build",
50+
"coverage==7.2.0",
51+
"flake8==5.0.4",
52+
"jupyter_contrib_nbextensions==0.7.0",
53+
"jupyter==1.0.0",
54+
"jupyterlab==4.2.5",
55+
"moto[s3]==5.0.5",
56+
"pystac_client==0.8.3",
57+
"seaborn==0.13.2",
58+
"sphinx-autobuild==2021.3.14",
59+
"unify==0.5",
60+
"yapf==0.23.0",
61+
]
62+
docs = [
63+
"autodoc-pydantic==2.2.0",
64+
"furo==2022.9.29",
65+
# Needed to make syntax highlighting work correctly on readthedocs.
66+
# See https://github.com/spatialaudio/nbsphinx/issues/24.
67+
"ipython>=8.26.0",
68+
"myst-parser==0.18.1",
69+
"nbsphinx==0.8.9",
70+
"sphinx-copybutton==0.5.*",
71+
# update when this is resolved: https://github.com/spatialaudio/nbsphinx/issues/655
72+
"sphinx-gallery>=0.10,<0.11",
73+
"sphinx==5.3.0",
74+
]
75+
76+
[project.urls]
77+
"Homepage" = "https://github.com/azavea/raster-vision"
78+
"GitHub" = "https://github.com/azavea/raster-vision"
79+
80+
[tool.setuptools]
81+
include-package-data = true
82+
packages = []

0 commit comments

Comments
 (0)