V3.1.0 chg open source prep (#88)

* CHG move k8s rpc logic into seperate utils directory

* ADD incluster and remote backups to digital ocean setup

* CHG expand the documentation

* ADD pydoc target

* ADD hpa

* ADD provision makefile

* ADD horizontal pod autoscaling

* CHG rename minikube debug target

* CHG move all regading rpc jobs to rpc queue

* RM react-vis public usage

* RM vpn

* CHG backup sftp

* RM unless_vpn

* FIX issue with backup yaml

* WIP add basics needed for cli authentication through NYU SSO

* CHG pip-compile requirements

* RM unused things

* CHG clean up imports

* ADD settable join_code

* CHG clean up web imports

* CHG replace logstash with filbeat

* CHG rework landing page, and apply StandardLayout to public pages

* ADD sqlalchemy deferred to large columns

* FIX issue with filebeat config

* CHG improve landing page

* CHG update caniuse-lite

* CHG optimize dockerfiles

* FIX landing page scramble text spacing

* ADD license

* CHG regenerate api/requirements.txt

* CHG rename all to provision in k8s/prod provision makefile

* CHG update language in design doc

* RM logstash from mindebug

* RM elastic.py

* CHG add req_assert function to api

* LINT

* CHG refactor to markdown posts

* CHG fix styling markdown, fetch -> axios

* CHG convert posts to md

* CHG fix small things

* RM old post components

* FIX issues causing tests to fail

* CHG replace useGet with useSWR

* FIX useSWR field access

* FIX not redirecting on 401

* CHG ignore .DS_Store

* RM digital ocean assumptions from prod provisioning

Co-authored-by: nysteo <[email protected]>

Author: wabscale

.gitignore

@@ -10,3 +10,5 @@ os-dump-*.sql
 .ipynb_checkpoints
 
 api/seed.ipynb
+
+.DS_Store

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2021 John McCann Cunniff Junior
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

Makefile

@@ -1,31 +1,21 @@
-PERSISTENT_SERVICES := db traefik kibana elasticsearch-coordinating redis-master logstash
+PERSISTENT_SERVICES := db traefik kibana elasticsearch-coordinating redis-master
 RESTART_ALWAYS_SERVICES := api web-dev rpc-default rpc-theia rpc-regrade
 PUSH_SERVICES := api web theia-init theia-proxy theia-admin theia-xv6
 
-
-
-CURRENT_DIR := $(shell basename $$(pwd) | tr '[:upper:]' '[:lower:]')
-IMAGES := $(shell \
-	ruby -ryaml -rjson -e 'puts JSON.pretty_generate(YAML.load(ARGF))' \
-	docker-compose.yml | jq '.services | .[].image | select(.!=null)' -r \
-	2> /dev/null \
-)
-BUILT_IMAGES := $(shell \
-	docker image ls | \
-	awk '{print $$1}' | \
-	grep -P '^($(CURRENT_DIR)_|os3224-)' \
-	2> /dev/null \
-)
-RUNNING_CONTAINERS := $(shell docker-compose ps -q)
-VOLUMES := $(shell docker volume ls | awk '{if (match($$2, /^anubis_.*$$/)) {print $$2}}')
-
-
 help:
 	@echo 'For convenience'
 	@echo
 	@echo 'Available make targets:'
 	@grep PHONY: Makefile | cut -d: -f2 | sed '1d;s/^/make/'
 
+startup-links:
+	@echo ''
+	@echo 'seed: http://localhost/api/admin/seed/'
+	@echo 'auth: http://localhost/api/admin/auth/token/superuser'
+	@echo 'auth: http://localhost/api/admin/auth/token/ta'
+	@echo 'auth: http://localhost/api/admin/auth/token/professor'
+	@echo 'site: http://localhost/'
+
 .PHONY: deploy       # Deploy Anubis to cluster
 deploy:
 	./k8s/deploy.sh
@@ -40,14 +30,6 @@ push:
 	docker-compose build --parallel --pull $(PUSH_SERVICES)
 	docker-compose push $(PUSH_SERVICES)
 
-startup-links:
-	@echo ''
-	@echo 'seed: http://localhost/api/admin/seed/'
-	@echo 'auth: http://localhost/api/admin/auth/token/superuser'
-	@echo 'auth: http://localhost/api/admin/auth/token/ta'
-	@echo 'auth: http://localhost/api/admin/auth/token/professor'
-	@echo 'site: http://localhost/'
-
 .PHONY: debug        # Start the cluster in debug mode
 debug:
 	docker-compose up -d $(PERSISTENT_SERVICES)
@@ -62,23 +44,23 @@ debug:
 
 .PHONY: mindebug     # Start the minimal cluster in debug mode
 mindebug:
-	docker-compose up -d traefik db redis-master logstash
+	docker-compose up -d traefik db redis-master
 	docker-compose up \
 		-d --force-recreate \
-		api web-dev rpc-default rpc-theia
+		api web-dev rpc-default
 	@echo 'Waiting a moment before running migrations'
 	sleep 3
 	@echo 'running migrations'
 	make -C api migrations
 	make startup-links
 
-.PHONY: mkdebug      # Start minikube debug
-mkdebug:
+.PHONY: debug-mk     # Start minikube debug
+debug-mk:
 	./k8s/debug/provision.sh
 	make startup-links
 
-.PHONY: mkrestart    # Restart minikube debug
-mkrestart:
+.PHONY: restart-mk   # Restart minikube debug
+restart-mk:
 	./k8s/debug/restart.sh
 	make startup-links
 
@@ -87,20 +69,3 @@ yeetdb:
 	docker-compose rm -f
 	docker volume rm anubis_db_data
 	docker-compose up -d --force-recreate db
-
-.PHONY: clean        # Clean up volumes, images and data
-clean:
-	docker-compose kill
-	if [ -n "$(RUNNING_CONTAINERS)" ]; then \
-		docker rm -f $(RUNNING_CONTAINERS); \
-	fi
-	if [ -n "$(IMAGES)" ]; then \
-		docker rmi -f $(IMAGES); \
-	fi
-	if [ -n "$(BUILT_IMAGES)" ]; then \
-		docker rmi -f $(BUILT_IMAGES); \
-	fi
-	if [ -n "${VOLUMES}" ]; then \
-		docker volume rm $(VOLUMES); \
-	fi
-	docker system prune -f