Skip to content

Commit d94b133

Browse files
ewdurbinewdurbin
authored
Deploy to Cabotage (#2396)
* add health check endpoint * add Dockerfile for cabotage * bind to cabotage socket * rename config * SOURCE_VERSION -> SOURCE_COMMIT * migrate from nginx buildpack * collectstatic * fixup docker files * add ca cert verification to haystack/elasticsearch * run worker and worker-beat
1 parent 2a2ef06 commit d94b133

11 files changed

+265
-13
lines changed

‎Dockerfile

+2-1
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
1616
set -x \
1717
&& apt-get update \
1818
&& apt-get install --no-install-recommends -y \
19-
pandoc \
2019
texlive-latex-base \
2120
texlive-latex-recommended \
2221
texlive-fonts-recommended \
@@ -35,6 +34,8 @@ WORKDIR /code
3534

3635
COPY dev-requirements.txt /code/
3736
COPY base-requirements.txt /code/
37+
COPY prod-requirements.txt /code/
38+
COPY requirements.txt /code/
3839

3940
RUN pip --no-cache-dir --disable-pip-version-check install --upgrade pip setuptools wheel
4041

‎Dockerfile.cabotage

+49
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
FROM python:3.9-bullseye
2+
COPY --from=ewdurbin/nginx-static:1.25.x /usr/bin/nginx /usr/bin/nginx
3+
ENV PYTHONUNBUFFERED=1
4+
ENV PYTHONDONTWRITEBYTECODE=1
5+
6+
# By default, Docker has special steps to avoid keeping APT caches in the layers, which
7+
# is good, but in our case, we're going to mount a special cache volume (kept between
8+
# builds), so we WANT the cache to persist.
9+
RUN set -eux; \
10+
rm -f /etc/apt/apt.conf.d/docker-clean; \
11+
echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache;
12+
13+
# Install System level build requirements, this is done before
14+
# everything else because these are rarely ever going to change.
15+
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
16+
--mount=type=cache,target=/var/lib/apt,sharing=locked \
17+
set -x \
18+
&& apt-get update \
19+
&& apt-get install --no-install-recommends -y \
20+
texlive-latex-base \
21+
texlive-latex-recommended \
22+
texlive-fonts-recommended \
23+
texlive-plain-generic \
24+
lmodern
25+
26+
RUN case $(uname -m) in \
27+
"x86_64") ARCH=amd64 ;; \
28+
"aarch64") ARCH=arm64 ;; \
29+
esac \
30+
&& wget --quiet https://github.com/jgm/pandoc/releases/download/2.17.1.1/pandoc-2.17.1.1-1-${ARCH}.deb \
31+
&& dpkg -i pandoc-2.17.1.1-1-${ARCH}.deb
32+
33+
RUN mkdir /code
34+
WORKDIR /code
35+
36+
COPY dev-requirements.txt /code/
37+
COPY base-requirements.txt /code/
38+
COPY prod-requirements.txt /code/
39+
COPY requirements.txt /code/
40+
41+
RUN pip --no-cache-dir --disable-pip-version-check install --upgrade pip setuptools wheel
42+
43+
RUN --mount=type=cache,target=/root/.cache/pip \
44+
set -x \
45+
&& pip --disable-pip-version-check \
46+
install \
47+
-r requirements.txt -r prod-requirements.txt
48+
COPY . /code/
49+
RUN DJANGO_SETTINGS_MODULE=pydotorg.settings.static python manage.py collectstatic --noinput

‎Procfile

+2
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,4 @@
11
release: python manage.py migrate --noinput
22
web: bin/start-nginx gunicorn -c gunicorn.conf pydotorg.wsgi
3+
worker: celery -A pydotorg worker -l INFO
4+
worker-beat: celery -A pydotorg beat -l INFO --scheduler django_celery_beat.schedulers:DatabaseScheduler

‎bin/start-nginx

+70
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
#!/usr/bin/env bash
2+
3+
psmgr=/tmp/nginx-buildpack-wait
4+
rm -f $psmgr
5+
mkfifo $psmgr
6+
7+
n=1
8+
while getopts :f option ${@:1:2}
9+
do
10+
case "${option}"
11+
in
12+
f) FORCE=$OPTIND; n=$((n+1));;
13+
esac
14+
done
15+
16+
# Initialize log directory.
17+
mkdir -p /tmp/logs/nginx
18+
touch /tmp/logs/nginx/access.log /tmp/logs/nginx/error.log
19+
echo 'buildpack=nginx at=logs-initialized'
20+
21+
# Start log redirection.
22+
(
23+
# Redirect nginx logs to stdout.
24+
tail -qF -n 0 /tmp/logs/nginx/*.log
25+
echo 'logs' >$psmgr
26+
) &
27+
28+
# Start App Server
29+
(
30+
# Take the command passed to this bin and start it.
31+
# E.g. bin/start-nginx bundle exec unicorn -c config/unicorn.rb
32+
COMMAND=${@:$n}
33+
echo "buildpack=nginx at=start-app cmd=$COMMAND"
34+
$COMMAND
35+
echo 'app' >$psmgr
36+
) &
37+
38+
if [[ -z "$FORCE" ]]
39+
then
40+
FILE="/tmp/app-initialized"
41+
42+
# We block on app-initialized so that when nginx binds to $PORT
43+
# are app is ready for traffic.
44+
while [[ ! -f "$FILE" ]]
45+
do
46+
echo 'buildpack=nginx at=app-initialization'
47+
sleep 1
48+
done
49+
echo 'buildpack=nginx at=app-initialized'
50+
fi
51+
52+
# Start nginx
53+
(
54+
# We expect nginx to run in foreground.
55+
# We also expect a socket to be at /tmp/nginx.socket.
56+
echo 'buildpack=nginx at=nginx-start'
57+
cd /tmp
58+
/usr/bin/nginx -p . -c /code/config/nginx.conf
59+
echo 'nginx' >$psmgr
60+
) &
61+
62+
# This read will block the process waiting on a msg to be put into the fifo.
63+
# If any of the processes defined above should exit,
64+
# a msg will be put into the fifo causing the read operation
65+
# to un-block. The process putting the msg into the fifo
66+
# will use it's process name as a msg so that we can print the offending
67+
# process to stdout.
68+
read exit_process <$psmgr
69+
echo "buildpack=nginx at=exit process=$exit_process"
70+
exit 1

‎config/mime.types

+98
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
types {
2+
text/html html htm shtml;
3+
text/css css;
4+
text/xml xml;
5+
image/gif gif;
6+
image/jpeg jpeg jpg;
7+
application/javascript js;
8+
application/atom+xml atom;
9+
application/rss+xml rss;
10+
11+
text/mathml mml;
12+
text/plain txt;
13+
text/vnd.sun.j2me.app-descriptor jad;
14+
text/vnd.wap.wml wml;
15+
text/x-component htc;
16+
17+
image/avif avif;
18+
image/png png;
19+
image/svg+xml svg svgz;
20+
image/tiff tif tiff;
21+
image/vnd.wap.wbmp wbmp;
22+
image/webp webp;
23+
image/x-icon ico;
24+
image/x-jng jng;
25+
image/x-ms-bmp bmp;
26+
27+
font/woff woff;
28+
font/woff2 woff2;
29+
30+
application/java-archive jar war ear;
31+
application/json json;
32+
application/mac-binhex40 hqx;
33+
application/msword doc;
34+
application/pdf pdf;
35+
application/postscript ps eps ai;
36+
application/rtf rtf;
37+
application/vnd.apple.mpegurl m3u8;
38+
application/vnd.google-earth.kml+xml kml;
39+
application/vnd.google-earth.kmz kmz;
40+
application/vnd.ms-excel xls;
41+
application/vnd.ms-fontobject eot;
42+
application/vnd.ms-powerpoint ppt;
43+
application/vnd.oasis.opendocument.graphics odg;
44+
application/vnd.oasis.opendocument.presentation odp;
45+
application/vnd.oasis.opendocument.spreadsheet ods;
46+
application/vnd.oasis.opendocument.text odt;
47+
application/vnd.openxmlformats-officedocument.presentationml.presentation
48+
pptx;
49+
application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
50+
xlsx;
51+
application/vnd.openxmlformats-officedocument.wordprocessingml.document
52+
docx;
53+
application/vnd.wap.wmlc wmlc;
54+
application/wasm wasm;
55+
application/x-7z-compressed 7z;
56+
application/x-cocoa cco;
57+
application/x-java-archive-diff jardiff;
58+
application/x-java-jnlp-file jnlp;
59+
application/x-makeself run;
60+
application/x-perl pl pm;
61+
application/x-pilot prc pdb;
62+
application/x-rar-compressed rar;
63+
application/x-redhat-package-manager rpm;
64+
application/x-sea sea;
65+
application/x-shockwave-flash swf;
66+
application/x-stuffit sit;
67+
application/x-tcl tcl tk;
68+
application/x-x509-ca-cert der pem crt;
69+
application/x-xpinstall xpi;
70+
application/xhtml+xml xhtml;
71+
application/xspf+xml xspf;
72+
application/zip zip;
73+
74+
application/octet-stream bin exe dll;
75+
application/octet-stream deb;
76+
application/octet-stream dmg;
77+
application/octet-stream iso img;
78+
application/octet-stream msi msp msm;
79+
80+
audio/midi mid midi kar;
81+
audio/mpeg mp3;
82+
audio/ogg ogg;
83+
audio/x-m4a m4a;
84+
audio/x-realaudio ra;
85+
86+
video/3gpp 3gpp 3gp;
87+
video/mp2t ts;
88+
video/mp4 mp4;
89+
video/mpeg mpeg mpg;
90+
video/quicktime mov;
91+
video/webm webm;
92+
video/x-flv flv;
93+
video/x-m4v m4v;
94+
video/x-mng mng;
95+
video/x-ms-asf asx asf;
96+
video/x-ms-wmv wmv;
97+
video/x-msvideo avi;
98+
}

‎config/nginx.conf.erb ‎config/nginx.conf

+8-10
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
daemon off;
2-
#Heroku dynos have at least 4 cores.
3-
worker_processes <%= ENV['NGINX_WORKERS'] || 4 %>;
2+
worker_processes 2;
43

54
events {
65
use epoll;
@@ -15,9 +14,8 @@ http {
1514

1615
server_tokens off;
1716

18-
log_format l2met 'measure#nginx.service=$request_time request_id=$http_x_request_id';
19-
access_log logs/nginx/access.log l2met;
20-
error_log logs/nginx/error.log;
17+
access_log /tmp/logs/nginx/access.log;
18+
error_log /tmp/logs/nginx/error.log;
2119

2220
include mime.types;
2321
default_type application/octet-stream;
@@ -29,11 +27,11 @@ http {
2927
client_max_body_size 32m;
3028

3129
upstream app_server {
32-
server unix:/tmp/nginx.socket fail_timeout=0;
30+
server unix:/var/run/cabotage/nginx.sock fail_timeout=0;
3331
}
3432

3533
server {
36-
listen <%= ENV["PORT"] %>;
34+
listen unix:/var/run/cabotage/cabotage.sock;
3735
server_name _;
3836
keepalive_timeout 5;
3937

@@ -317,17 +315,17 @@ http {
317315
}
318316

319317
location /static/ {
320-
alias /app/static-root/;
318+
alias /code/static-root/;
321319
add_header Cache-Control "max-age=604800, public"; # 604800 is 7 days
322320
}
323321

324322
location /images/ {
325-
alias /app/static-root/images/;
323+
alias /code/static-root/images/;
326324
add_header Cache-Control "max-age=604800, public"; # 604800 is 7 days
327325
}
328326

329327
location /favicon.ico {
330-
alias /app/static-root/favicon.ico;
328+
alias /code/static-root/favicon.ico;
331329
add_header Cache-Control "max-age=604800, public"; # 604800 is 7 days
332330
}
333331

‎gunicorn.conf

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
bind = 'unix:/tmp/nginx.socket'
1+
bind = 'unix:/var/run/cabotage/nginx.sock'
22
backlog = 1024
33
preload_app = True
44
max_requests = 2048

‎pydotorg/settings/heroku.py ‎pydotorg/settings/cabotage.py

+4-1
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,9 @@
2929
'ENGINE': 'haystack.backends.elasticsearch7_backend.Elasticsearch7SearchEngine',
3030
'URL': HAYSTACK_SEARCHBOX_SSL_URL,
3131
'INDEX_NAME': config('HAYSTACK_INDEX', default='haystack-prod'),
32+
'KWARGS': {
33+
'ca_certs': '/var/run/secrets/cabotage.io/ca.crt',
34+
}
3235
},
3336
}
3437

@@ -68,7 +71,7 @@
6871

6972
RAVEN_CONFIG = {
7073
"dsn": config('SENTRY_DSN'),
71-
"release": config('SOURCE_VERSION'),
74+
"release": config('SOURCE_COMMIT'),
7275
}
7376

7477
AWS_ACCESS_KEY_ID = config('AWS_ACCESS_KEY_ID')

‎pydotorg/settings/static.py

+25
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
import os
2+
3+
import dj_database_url
4+
import raven
5+
from decouple import Csv
6+
7+
from .base import *
8+
9+
DEBUG = TEMPLATE_DEBUG = False
10+
11+
HAYSTACK_CONNECTIONS = {
12+
'default': {
13+
'ENGINE': 'haystack.backends.elasticsearch5_backend.Elasticsearch5SearchEngine',
14+
'URL': 'http://127.0.0.1:9200',
15+
'INDEX_NAME': 'haystack-null',
16+
},
17+
}
18+
19+
MIDDLEWARE = [
20+
'whitenoise.middleware.WhiteNoiseMiddleware',
21+
] + MIDDLEWARE
22+
23+
MEDIAFILES_LOCATION = 'media'
24+
DEFAULT_FILE_STORAGE = 'custom_storages.storages.MediaStorage'
25+
STATICFILES_STORAGE = 'custom_storages.storages.PipelineManifestStorage'

‎pydotorg/urls.py

+1
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
urlpatterns = [
1717
# homepage
1818
path('', views.IndexView.as_view(), name='home'),
19+
re_path(r'^_health/?', views.health, name='health'),
1920
path('authenticated', views.AuthenticatedView.as_view(), name='authenticated'),
2021
re_path(r'^humans.txt$', TemplateView.as_view(template_name='humans.txt', content_type='text/plain')),
2122
re_path(r'^robots.txt$', TemplateView.as_view(template_name='robots.txt', content_type='text/plain')),

‎pydotorg/views.py

+5
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,15 @@
11
from django.conf import settings
2+
from django.http import HttpResponse
23
from django.views.generic.base import RedirectView, TemplateView
34

45
from codesamples.models import CodeSample
56
from downloads.models import Release
67

78

9+
def health(request):
10+
return HttpResponse('OK')
11+
12+
813
class IndexView(TemplateView):
914
template_name = "python/index.html"
1015

0 commit comments

Comments
 (0)