Updates for Python 3

.buildkite/docker-compose.yml

@@ -1,10 +1,6 @@
-version: "2"
 services:
   mltshp:
-    image: mltshp/mltshp-web:latest
-    build:
-      context: ..
-      dockerfile: Dockerfile
+    build: ..
     volumes:
       - ./settings.py:/srv/mltshp.com/mltshp/settings.py
       - ../.git:/srv/mltshp.com/mltshp/.git

.buildkite/settings.py

@@ -28,6 +28,7 @@
     "use_workers": False,
     "debug_workers": True,
     "superuser_list": "admin",
+    "tornado_logging": False,
     # these must be set for testing test/unit/externalservice_tests.py
     # "twitter_consumer_key" : "twitter_consumer_key_here",
     # "twitter_consumer_secret" : "twitter_consumer_secret_key_here",

.buildkite/steps/build-web.sh

@@ -1,9 +1,15 @@
 #!/bin/bash
 
+# exit if any command fails (e); strict variable substitution (u);
+# set exit code to non-zero for any failed piped commands (o pipefail)
+# See also: http://redsymbol.net/articles/unofficial-bash-strict-mode/
 set -euo pipefail
 
-# pull prior image to populate layer cache
-docker pull mltshp/mltshp-web:latest
+echo "--- Pulling base Docker image"
+    docker pull mltshp/mltshp-web:latest
 
-docker build -t mltshp/mltshp-web:build-${BUILDKITE_BUILD_NUMBER} .
-docker push mltshp/mltshp-web:build-${BUILDKITE_BUILD_NUMBER}
+echo "+++ Building Docker image for web node"
+    docker build -t mltshp/mltshp-web:build-${BUILDKITE_BUILD_NUMBER} .
+
+echo "--- Pushing build Docker image to Docker Hub"
+    docker push mltshp/mltshp-web:build-${BUILDKITE_BUILD_NUMBER}

.buildkite/steps/build-worker.sh

@@ -1,9 +1,15 @@
 #!/bin/bash
 
+# exit if any command fails (e); strict variable substitution (u);
+# set exit code to non-zero for any failed piped commands (o pipefail)
+# See also: http://redsymbol.net/articles/unofficial-bash-strict-mode/
 set -euo pipefail
 
-# pull prior image to populate layer cache
-docker pull mltshp/mltshp-worker:latest
+echo "--- Pulling base Docker image"
+    docker pull mltshp/mltshp-worker:latest
 
-docker build -t mltshp/mltshp-worker:build-${BUILDKITE_BUILD_NUMBER} -f Dockerfile.worker .
-docker push mltshp/mltshp-worker:build-${BUILDKITE_BUILD_NUMBER}
+echo "+++ Building Docker image for worker node"
+    docker build -t mltshp/mltshp-worker:build-${BUILDKITE_BUILD_NUMBER} -f Dockerfile.worker .
+
+echo "--- Pushing build Docker image to Docker Hub"
+    docker push mltshp/mltshp-worker:build-${BUILDKITE_BUILD_NUMBER}

.buildkite/steps/deploy.sh

@@ -1,6 +1,9 @@
 #!/bin/bash
 
-set -eo pipefail
+# exit if any command fails (e); strict variable substitution (u);
+# set exit code to non-zero for any failed piped commands (o pipefail)
+# See also: http://redsymbol.net/articles/unofficial-bash-strict-mode/
+set -euo pipefail
 
 function slackpost {
     # Usage: slackpost <channel> <message>
@@ -18,13 +21,19 @@ function slackpost {
     fi
 }
 
-# Grab CI images
-docker pull mltshp/mltshp-web:build-${BUILDKITE_BUILD_NUMBER}
-docker tag mltshp/mltshp-web:build-${BUILDKITE_BUILD_NUMBER} mltshp/mltshp-web:latest
-docker push mltshp/mltshp-web:latest
+echo "--- Pulling Docker image for web node build ${BUILDKITE_BUILD_NUMBER}"
+    docker pull mltshp/mltshp-web:build-${BUILDKITE_BUILD_NUMBER}
 
-docker pull mltshp/mltshp-worker:build-${BUILDKITE_BUILD_NUMBER}
-docker tag mltshp/mltshp-worker:build-${BUILDKITE_BUILD_NUMBER} mltshp/mltshp-worker:latest
-docker push mltshp/mltshp-worker:latest
+echo "--- Tagging Docker image as latest and pushing to Docker Hub"
+    docker tag mltshp/mltshp-web:build-${BUILDKITE_BUILD_NUMBER} mltshp/mltshp-web:latest
+    docker push mltshp/mltshp-web:latest
 
-slackpost "#operations" "Build ${BUILDKITE_BUILD_NUMBER} has been pushed to Docker cloud by ${BUILDKITE_UNBLOCKER}: ${BUILDKITE_BUILD_URL}"
+echo "--- Pulling Docker image for worker node build ${BUILDKITE_BUILD_NUMBER}"
+    docker pull mltshp/mltshp-worker:build-${BUILDKITE_BUILD_NUMBER}
+
+echo "--- Tagging Docker image as latest and pushing to Docker Hub"
+    docker tag mltshp/mltshp-worker:build-${BUILDKITE_BUILD_NUMBER} mltshp/mltshp-worker:latest
+    docker push mltshp/mltshp-worker:latest
+
+echo "--- Posting Slack alert!"
+    slackpost "#operations" "Build ${BUILDKITE_BUILD_NUMBER} has been pushed to Docker cloud by ${BUILDKITE_UNBLOCKER}: ${BUILDKITE_BUILD_URL}"

.buildkite/steps/test.sh

@@ -1,30 +1,33 @@
 #!/bin/bash
 
+# exit if any command fails (e); strict variable substitution (u);
+# set exit code to non-zero for any failed piped commands (o pipefail)
+# See also: http://redsymbol.net/articles/unofficial-bash-strict-mode/
 set -euo pipefail
 
 wait_for() {
     echo Waiting for $1 to listen on $2...
     while ! nc -z $1 $2; do echo sleeping; sleep 2; done
 }
 
-docker pull mltshp/mltshp-web:build-${BUILDKITE_BUILD_NUMBER}
-docker tag mltshp/mltshp-web:build-${BUILDKITE_BUILD_NUMBER} mltshp/mltshp-web:latest
+echo "--- Pulling base Docker image"
+    docker pull mltshp/mltshp-web:build-${BUILDKITE_BUILD_NUMBER}
+    docker tag mltshp/mltshp-web:build-${BUILDKITE_BUILD_NUMBER} mltshp/mltshp-web:latest
 
-# launch fakes3/mysql/web app
-docker-compose -f .buildkite/docker-compose.yml up -d
+echo "--- Launching Docker containers"
+    docker compose -f .buildkite/docker-compose.yml up -d --build
 
-# let's wait and allow mysql/fakes3 to spin up
-#wait_for localhost 3306
-#wait_for localhost 8000
-sleep 10
+echo "~~~ Waiting for containers to start"
+    #wait_for localhost 3306
+    #wait_for localhost 8000
+    sleep 10
 
-# run our tests against it
-docker exec -t buildkite_mltshp_1 ./run-tests.sh
+echo "+++ Running unit tests"
+    docker exec -t buildkite_mltshp_1 ./run-tests.sh
 
-# submit coverage data
-docker exec -t -e BUILDKITE -e BUILDKITE_JOB_ID -e BUILDKITE_BRANCH -e COVERALLS_REPO_TOKEN buildkite_mltshp_1 ./coveralls-report.sh
+echo "--- Submitting coverage data"
+    docker exec -t -e BUILDKITE -e BUILDKITE_JOB_ID -e BUILDKITE_BRANCH -e COVERALLS_REPO_TOKEN buildkite_mltshp_1 ./coveralls-report.sh
 
-# tear down containers
-docker-compose -f .buildkite/docker-compose.yml down
-
-docker container prune -f
+echo "~~~ Stopping containers; cleanup"
+    docker compose -f .buildkite/docker-compose.yml down
+    docker container prune -f

.gitignore

@@ -9,6 +9,7 @@ pip-log.txt
 .DS_Store
 *.swp
 env
+env3
 mounts
 .env
 .deploy.env

Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:16.04
+FROM ubuntu:22.04
 LABEL maintainer "[email protected]"
 ENV PYTHONUNBUFFERED 1
 
@@ -7,9 +7,9 @@ ENV PYTHONUNBUFFERED 1
 # whole layer and steps to build it should be cached.
 RUN apt-get -y update && apt-get install -y \
         supervisor \
+        python3-dev \
         libmysqlclient-dev \
         mysql-client \
-        python-dev \
         libjpeg-dev \
         libcurl4-openssl-dev \
         curl \
@@ -20,20 +20,14 @@ RUN apt-get -y update && apt-get install -y \
         libpcre3-dev \
         libssl-dev \
         libffi-dev \
-        python-pip && \
+        python3-pip && \
     rm -rf /var/lib/apt/lists/* && \
-    \
-    pip install -U 'pip==20.3.4' 'setuptools==44.0.0' distribute && \
-    # fixes a weird issue where distribute complains about setuptools "0.7"
-    # (incorrectly matching version "20.7.0" which ubuntu 16.04 has preinstalled)
-    rm -rf /usr/lib/python2.7/dist-packages/setuptools-20.7.0.egg-info && \
-    \
     # install nginx + upload module
     mkdir -p /tmp/install && \
     cd /tmp/install && \
-    wget http://nginx.org/download/nginx-0.8.55.tar.gz && tar zxf nginx-0.8.55.tar.gz && \
-    wget https://github.com/fdintino/nginx-upload-module/archive/2.2.0.tar.gz && tar zxf 2.2.0.tar.gz && \
-    cd /tmp/install/nginx-0.8.55 && \
+    wget http://nginx.org/download/nginx-1.25.3.tar.gz && tar zxf nginx-1.25.3.tar.gz && \
+    wget https://github.com/fdintino/nginx-upload-module/archive/2.3.0.tar.gz && tar zxf 2.3.0.tar.gz && \
+    cd /tmp/install/nginx-1.25.3 && \
     ./configure \
         --with-http_ssl_module \
         --with-http_stub_status_module \
@@ -43,7 +37,7 @@ RUN apt-get -y update && apt-get install -y \
         --conf-path=/etc/nginx/nginx.conf \
         --error-log-path=/srv/mltshp.com/nginx-error.log \
         --http-log-path=/srv/mltshp.com/nginx-access.log \
-        --add-module=/tmp/install/nginx-upload-module-2.2.0 && \
+        --add-module=/tmp/install/nginx-upload-module-2.3.0 && \
     make && make install && \
     mkdir -p /etc/nginx && \
     rm -rf /tmp/install && \

Dockerfile.worker

@@ -1,4 +1,4 @@
-FROM ubuntu:16.04
+FROM ubuntu:22.04
 LABEL maintainer "[email protected]"
 ENV PYTHONUNBUFFERED 1
 
@@ -10,7 +10,7 @@ RUN apt-get -y update && \
     cron \
     libmysqlclient-dev \
     mysql-client \
-    python-dev \
+    python3-dev \
     libjpeg-dev \
     libcurl4-openssl-dev \
     curl \
@@ -22,14 +22,8 @@ RUN apt-get -y update && \
     libpcre3-dev \
     libssl-dev \
     libffi-dev \
-    python-pip && \
+    python3-pip && \
     rm -rf /var/lib/apt/lists/* && \
-    pip install -U 'pip==20.3.4' 'setuptools==44.0.0' distribute && \
-    # Fix for a really weird issue when installing postmark library
-    # distribute fails to run since it sees a setuptools with "0.7"
-    # in the name, even though ubuntu:16.04 has pre-installed "20.7.0"
-    # https://github.com/pypa/setuptools/issues/543
-    rm -rf /usr/lib/python2.7/dist-packages/setuptools-20.7.0.egg-info && \
     groupadd ubuntu --gid=1010 && \
     useradd ubuntu --create-home --home-dir=/home/ubuntu \
         --uid=1010 --gid=1010 && \

Makefile

@@ -1,31 +1,34 @@
-.PHONY: init-dev run shell test destroy migrate mysql
+.PHONY: init-dev run stop custom-build build shell test destroy migrate mysql
 
 init-dev:
 	cp settings.example.py settings.py
 	cp celeryconfig.example.py celeryconfig.py
 	mkdir -p mounts/mysql mounts/logs mounts/fakes3 mounts/uploaded
 
 run:
-	docker-compose --env-file .env up -d
+	docker compose up -d
 
 stop:
-	docker-compose --env-file .env down
+	docker compose down
+
+custom-build:
+	@read -p "build tag (default is 'latest'): " build_tag; \
+	docker build -t mltshp/mltshp-web:$${build_tag:-latest}
 
 build:
 	docker build -t mltshp/mltshp-web:latest .
 
 shell:
-	docker-compose --env-file .env exec mltshp bash
+	docker compose exec mltshp bash
 
 test:
-	docker-compose --env-file .env exec mltshp su ubuntu -c "cd /srv/mltshp.com/mltshp; python test.py $(TEST)"
+	docker compose exec mltshp su ubuntu -c "cd /srv/mltshp.com/mltshp; python3 test.py $(TEST)"
 
 destroy:
-	docker-compose down
-	rm -rf mounts
+	docker compose down && rm -rf mounts
 
 migrate:
-	docker-compose --env-file .env exec mltshp su ubuntu -c "cd /srv/mltshp.com/mltshp; python migrate.py"
+	docker compose exec mltshp su ubuntu -c "cd /srv/mltshp.com/mltshp; python3 migrate.py"
 
 mysql:
-	docker-compose --env-file .env exec mltshp su ubuntu -c "cd /srv/mltshp.com/mltshp; mysql -u root --host mysql mltshp"
+	docker compose exec mltshp su ubuntu -c "cd /srv/mltshp.com/mltshp; mysql -u root --host mysql mltshp"

README.md

@@ -4,6 +4,12 @@
 
 [![Build status](https://badge.buildkite.com/a86854c6272f21c9b46b8b6aafd3a4fb99bcfabe6e611bc370.svg)](https://buildkite.com/mltshp-inc/mltshp-web-service) [![Coverage Status](https://coveralls.io/repos/github/MLTSHP/mltshp/badge.svg?branch=master)](https://coveralls.io/github/MLTSHP/mltshp?branch=master)
 
+## Project Description
+
+This project is the codebase for running [mltshp.com](https://mltshp.com).
+It's a Python 3 application, utilizing a MySQL database, Amazon S3 for
+asset storage, and RabbitMQ for background jobs.
+
 ## Development Environment
 
 MLTSHP is a Dockerized application. This greatly simplifies running the
@@ -20,6 +26,11 @@ should be okay):
 
     $ make init-dev
 
+Check the process limits for your computer using the `ulimit -a` command. Increase
+the file descriptor limit if it's not at least 1000, using `ulimit -n 1000` (or
+some suitably higher value). You can add this to your shell startup script to
+make it permanent.
+
 You should be able to start the app itself using:
 
     $ make run
@@ -59,7 +70,7 @@ When you run the application, it launches it into a background process.
 But if you want to watch the realtime logs emitted by each service,
 just use this command:
 
-    $ docker-compose logs -f
+    $ docker compose logs -f
 
 In addition to that, the web app produces some log files that are
 captured under the "mounts/logs" folder of your git repository.
@@ -114,8 +125,7 @@ if you have a MySQL database you use locally for development and
 testing and keep it versus using the `destroy` and `init-dev`
 commands to make a new one. To update your database, just do this:
 
-    $ make shell
-    docker-shell$ cd /srv/mltshp.com/mltshp; python migrate.py
+    $ make migrate
 
 That should do it.
 
@@ -138,6 +148,11 @@ Then, just run:
 
 Which will invoke a Docker process to run the unit test suite.
 
+You can also run a specific unit test by setting a TEST environment
+variable (you can find the unit test names in `test.py`):
+
+    $ TEST=test.unit.shake_tests make test
+
 ## Connecting to the MLTSHP shell
 
 If you ever need to access the Docker image running the application,
@@ -150,6 +165,14 @@ This should place you in the /srv/mltshp.com/mltshp directory as the
 root user. You can use `apt-get` commands to install utilities you
 may need.
 
+## Connecting to the MLTSHP MySQL database
+
+You can also access the MySQL shell using:
+
+    $ make mysql
+
+Useful for inspecting the database for your local directly.
+
 ## Cleanup
 
 If you ever want to _wipe your local data_ and rebuild your Docker
@@ -160,7 +183,7 @@ containers, just use this command:
 If you just wish to rebuild the Docker container, use the Docker
 compose command:
 
-    $ docker-compose down
+    $ make stop
 
 Then, run another `make run`.
 
@@ -174,6 +197,6 @@ update from the pattern library.
 
 ## About
 
-MLTSHP is open-source software, ©2017 the MLTSHP team and released to the public under the terms of the Mozilla Public License. A copy of the MPL can be found in the LICENSE file.
+MLTSHP is open-source software, ©2023 the MLTSHP team and released to the public under the terms of the Mozilla Public License. A copy of the MPL can be found in the LICENSE file.
 
 [![Fastly logo](/static/images/fastly-logo.png)](https://www.fastly.com) MLTSHP is proudly powered by Fastly.