diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md deleted file mode 100644 index 00691bb0..00000000 --- a/docs/ARCHITECTURE.md +++ /dev/null @@ -1,33 +0,0 @@ -# Architecture - -Note: This is written as if it is completed and working, but currently a lot of it is not. As a result, this is all subject to change. - -## Manager - -The manager runs a Django web application. It uses Channels to handle websockets opened by the client, and Celery to perform potentially long-running tasks. Examples of long-running tasks include creating sites or editing parts of sites that require backend changes. - -## Orchestrator - -The orchestrator runs Nginx to serve static files and route incoming requests to the appropriate ports for each site. It uses Docker to actually serve the sites. - -It also runs a Flask application that handles updating Nginx's configuration, creating/updating Docker containers, etc. upon requests from the manager, as well as a Websocket server to handle interactive/long-running tasks. - -## Router - -The router runs Nginx to forward requests to the orchestrator(s), as well as a small Flask application to update Nginx's configuration and generate Let's Encrypt certificates upon requests from the manager. - -## Shell - -The shell server runs an SSH server using AsyncSSH. It communicates with the mamager - -## Communication between servers - -Frequently, one component needs to be able to make HTTP/Websocket requests to another component for various reasons. What follows is a list of all the types of access that may be required. - -- The manager should be able to make requests to all URLs on the orchestrator's Flask server. -- The manager should be able to make requests to all URLs on the orchestrator's Websocket server. -- The manager should be able to make requests to all URLs on the router's Flask server. -- The shell server should be able to make requests to all URLs with the prefix `/shell-server/` on the manager. No other client, especially regular web browsers, should be allowed to make requests to these URLs. -- The shell server should be able to make requests to all URLs with the prefix `/ws/shell-server/` on the orchestrator's Websocket server. It should not be allowed to make any other requests to the orchestrator or the router. - -Note: These restrictions are not enforced by Director 4.0 itself. Instead, each component with an HTTP server should be behind a load balancer that enforces access restrictions. diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 00000000..d0c3cbf1 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = source +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/SETUP.md b/docs/SETUP.md deleted file mode 100644 index 93ac1b7e..00000000 --- a/docs/SETUP.md +++ /dev/null @@ -1,11 +0,0 @@ -# Vagrant setup - -1. Install [VirtualBox](https://www.virtualbox.org/wiki/Downloads) and [Vagrant](http://docs.vagrantup.com/v2/installation/index.html). If you are running Windows, install [Git](https://gitforwindows.org/) and run `git config core.autocrlf input` to prevent line ending issues. -2. Clone the Director 4.0 repository onto your computer and `cd` into the new directory. If you have an SSH key, run `git clone git@github.com:tjresearch/research-theo_john.git director && cd director`. Otherwise, run `git clone https://github.com/tjresearch/research-theo_john.git director && cd director`. -3. Run `vagrant plugin install vagrant-vbguest`. -4. Run `vagrant up && vagrant reload`. This will download a Vagrant image and provision the resulting VM. -5. Run `vagrant ssh` to login to the VM. Once inside, run `cd director` to change into the repo and `./scripts/install_dependencies.sh` to install Director's Python dependencies using Pipenv. -6. You can now work on Director. `scripts/start-servers.sh` will open a `tmux` session with the four servers each running in a separate pane. - - Note: If you are not familiar with `tmux`, we recommend https://www.hamvocke.com/blog/a-quick-and-easy-guide-to-tmux/ and https://tmuxcheatsheet.com/ as starting resources. - - See [this](docs/tmux.md) for an explaination of the components of the development `tmux` -7. When you are finished, type `exit` to exit the VM and `vagrant halt` to stop it. When you want to work on Director 4.0 again, `cd` into this directory, run `vagrant up` and `vagrant ssh` to launch the VM and connect to it, and then run `exit` and `vagrant halt` to exit and shut it down. diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md deleted file mode 100644 index 90e69dce..00000000 --- a/docs/TROUBLESHOOTING.md +++ /dev/null @@ -1,15 +0,0 @@ -# Director 4.0: Troubleshooting - -**When I start Docker, I get a "Your kernel does not support swap memory limit" warning messsage.** - -By default, the kernel shipped with Ubuntu does not have memory accounting enabled. - -This is because: -> Memory and swap accounting incur an overhead of about 1% of the total available memory and a 10% overall performance degradation, even if Docker is not running. -- https://docs.docker.com/install/linux/linux-postinstall/#your-kernel-does-not-support-cgroup-swap-limit-capabilities - -1. Open `/etc/default/grub` as `root` on the Director appserver. -2. Add `cgroup_enable=memory swapaccount=1` to the `GRUB_CMDLINE_LINUX` line. -3. Run `update-grub` as `root` -4. Reboot. -5. It's fixed! diff --git a/docs/docker-swarm.md b/docs/docker-swarm.md deleted file mode 100644 index 0fad0f9c..00000000 --- a/docs/docker-swarm.md +++ /dev/null @@ -1,16 +0,0 @@ -# Director 4.0: Docker Swarm - -In particular, we are using the orchestration capabilities of a Swarm in order to help orchestrate the sites for Director 4.0. - -``` -Service - | - | - v -Task - | - | - v -Container - -``` diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 00000000..747ffb7b --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=source +set BUILDDIR=build + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +if "%1" == "" goto help + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/docs/rtd-requirements.txt b/docs/rtd-requirements.txt new file mode 100644 index 00000000..83602239 --- /dev/null +++ b/docs/rtd-requirements.txt @@ -0,0 +1,4 @@ +sphinx +sphinx-copybutton +myst_parser +furo diff --git a/docs/source/_templates/autosummary/class.rst b/docs/source/_templates/autosummary/class.rst new file mode 100644 index 00000000..8efc1a15 --- /dev/null +++ b/docs/source/_templates/autosummary/class.rst @@ -0,0 +1,34 @@ +{{ name | escape | underline}} + +Qualified name: ``{{ fullname }}`` + +.. currentmodule:: {{ module }} + +.. autoclass:: {{ objname }} + :show-inheritance: + :members: + :private-members: + + + {% block methods %} + {%- if methods %} + .. rubric:: {{ _('Methods') }} + + .. autosummary:: + :nosignatures: + {% for item in methods if item != '__init__' and item[0] != '_' and item not in inherited_members %} + ~{{ name }}.{{ item }} + {%- endfor %} + {%- endif %} + {%- endblock %} + + {% block attributes %} + {%- if attributes %} + .. rubric:: {{ _('Attributes') }} + + .. autosummary:: + {% for item in attributes if item[0] != '_' and item not in inherited_members %} + ~{{ name }}.{{ item }} + {%- endfor %} + {%- endif %} + {% endblock %} diff --git a/docs/source/_templates/autosummary/module.rst b/docs/source/_templates/autosummary/module.rst new file mode 100644 index 00000000..8aba97b3 --- /dev/null +++ b/docs/source/_templates/autosummary/module.rst @@ -0,0 +1,52 @@ +{{ name | escape | underline }} + +.. currentmodule:: {{ fullname }} + +.. automodule:: {{ fullname }} + + {% block classes %} + {% if classes %} + .. rubric:: Classes + + .. autosummary:: + :toctree: . + :nosignatures: + {% for class in classes %} + {{ class }} + {% endfor %} + {% endif %} + {% endblock %} + + {% block functions %} + {% if functions %} + .. rubric:: {{ _('Functions') }} + + {% for item in functions %} + .. autofunction:: {{ item }} + {%- endfor %} + {% endif %} + {% endblock %} + + {% block exceptions %} + {% if exceptions %} + .. rubric:: {{ _('Exceptions') }} + + .. autosummary:: + {% for item in exceptions %} + {{ item }} + {%- endfor %} + {% endif %} + {% endblock %} + +{% block modules %} +{% if modules %} +.. rubric:: Modules + +.. autosummary:: + :toctree: + :recursive: +{% for item in modules %} + {{ item }} +{%- endfor %} +{% endif %} +{% endblock %} diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 00000000..b8261b7c --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,148 @@ +# Configuration file for the Sphinx documentation builder. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information +import os +import sys +from datetime import datetime +from pathlib import Path + + +# -- Path setup -------------------------------------------------------------- +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. + + +BASE_ROOT = Path(__file__).parent.parent.resolve() +major_paths = ( + os.fspath(BASE_ROOT / proj) + for proj in ( + "manager", + "orchestrator", + "router", + "shell", + ) +) +sys.path = [*major_paths, *sys.path] + +# -- Project information ----------------------------------------------------- + +project = "Director4" +copyright = f"2020-{datetime.now().year}, Sysadmins at TJ CSL" +author = "Sysadmins at TJ CSL" + +# TODO: enable sphinxcontrib_django after all dependencies are merged into one Pipfile +# django_settings = "director.settings" + + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.autosummary", + "sphinx.ext.intersphinx", + "sphinx.ext.extlinks", + "sphinx.ext.napoleon", + "sphinx.ext.viewcode", + "sphinx_copybutton", + "myst_parser", + # "sphinxcontrib_django" +] + +add_function_parentheses = False + +# Automatically generate stub pages when using the .. autosummary directive +autosummary_generate = True + +# controls whether functions documented by the autofunction directive +# appear with their full module names +add_module_names = False + +# napoleon settings +napoleon_numpy_docstring = False # force google +napoleon_include_special_with_doc = False + +napoleon_custom_sections = [ + ("Special", "params_style"), +] + +templates_path = ["_templates"] +exclude_patterns = [] + +intersphinx_mapping = { + "python": ("https://docs.python.org/3/", None), + "sphinx": ("https://www.sphinx-doc.org/en/master/", None), + "django": ( + "https://docs.djangoproject.com/en/stable/", + "https://docs.djangoproject.com/en/stable/_objects/", + ), +} + +extlinks = { + "issue": ("https://github.com/tjcsl/director4/issues/%s", "issue %s"), + "pr": ("https://github.com/tjcsl/director4/pull/%s", "pr #%s"), +} +# warn hardcoded links +extlinks_detect_hardcoded_links = True + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# + +html_theme = "furo" + +# use |br| for linebreak +rst_prolog = """ +.. |br| raw:: html + +
+""" + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ["_static"] + +html_theme_options = { + "source_repository": "https://github.com/tjcsl/director4/", + "source_branch": "main", + "source_directory": "docs/source/", + "light_logo": "logo-full-black.svg", + "dark_logo": "logo-full-white.svg", + "sidebar_hide_name": True, + "light_css_variables": { + "color-content-foreground": "#000000", + "color-background-primary": "#ffffff", + "color-background-border": "#ffffff", + "color-sidebar-background": "#f8f9fb", + "color-brand-content": "#1c00e3", + "color-brand-primary": "#192bd0", + "color-link": "#c93434", + "color-link--hover": "#5b0000", + "color-inline-code-background": "#f6f6f6;", + "color-foreground-secondary": "#000", + }, + "dark_css_variables": { + "color-content-foreground": "#ffffffd9", + "color-background-primary": "#131416", + "color-background-border": "#303335", + "color-sidebar-background": "#1a1c1e", + "color-brand-content": "#2196f3", + "color-brand-primary": "#007fff", + "color-link": "#51ba86", + "color-link--hover": "#9cefc6", + "color-inline-code-background": "#262626", + "color-foreground-secondary": "#ffffffd9", + }, +} +html_title = "Director4" + +# This specifies any additional css files that will override the theme's +html_css_files = [] diff --git a/docs/source/dev.rst b/docs/source/dev.rst new file mode 100644 index 00000000..059a8bb4 --- /dev/null +++ b/docs/source/dev.rst @@ -0,0 +1,13 @@ +########## +Developing +########## + +A collection of reference documents for developers. + +.. toctree:: + :maxdepth: 1 + + dev/umask + dev/deploying + dev/docker-image-registry + dev/troubleshooting diff --git a/docs/DEPLOYING.md b/docs/source/dev/deploying.md similarity index 60% rename from docs/DEPLOYING.md rename to docs/source/dev/deploying.md index 92952218..784a1f1b 100644 --- a/docs/DEPLOYING.md +++ b/docs/source/dev/deploying.md @@ -1,6 +1,10 @@ # Deployment -**IMPORTANT: Running Director 4.0 is only supported on Linux-based systems. You may be able to get it working on other operating systems, but this is not supported and we may make changes breaking compatibility without warning.** +```{important} +Running Director 4.0 is only supported on Linux-based systems. +You may be able to get it working on other operating systems, but this is +not supported and we may make changes breaking compatibility without warning. +``` Each section here corresponds to a component of Director 4.0. @@ -8,7 +12,8 @@ Each section here corresponds to a component of Director 4.0. ### Dependencies -The manager uses Redis as the channel layer for Channels, RabbitMQ as the broker for Celery, and Nginx to serve static files. +The manager uses Redis as the channel layer for Channels, RabbitMQ as the broker for Celery, +and Nginx to serve static files. #### Redis @@ -16,13 +21,19 @@ Install [Redis](https://redis.io/). The default configuration should be sufficie #### RabbitMQ -[RabbitMQ](https://www.rabbitmq.com/) will also work out of the box. However, for security you should configure ot to only listen on `localhost`. This can be done by editing RabbitMQ's configuration file (usually located at `/etc/rabbitmq/rabbitmq.config`) to the following: `[{rabbit, [{tcp_listeners, [{"127.0.0.1", 5672}]}]}].` +[RabbitMQ](https://www.rabbitmq.com/) will also work out of the box. However, for security you should +configure it to only listen on `localhost`. This can be done by editing RabbitMQ's +configuration file (usually located at `/etc/rabbitmq/rabbitmq.config`) to the following: +`[{rabbit, [{tcp_listeners, [{"127.0.0.1", 5672}]}]}].` #### Nginx -For [Nginx](https://nginx.org). this configuration *should* work, though it has not been tested (the current production environment uses a more complex configuration). It should be placed in `/etc/nginx/sites-available/director.conf`, and a symbolic link should be created at `/etc/nginx/sites-enabled/director.conf` pointing to this file. +For [Nginx](https://nginx.org). this configuration *should* work, though it has not been tested +(the current production environment uses a more complex configuration). It should be placed in +`/etc/nginx/sites-available/director.conf`, and a symbolic link should be created at +`/etc/nginx/sites-enabled/director.conf` pointing to this file. -``` +```nginx server { listen 80 default_server; listen [::]:80 default_server; @@ -56,15 +67,18 @@ server { } ``` -Note that this will listen on port 80 (HTTP), so traffic will be sent unencrypted. If you wish to encrypt traffic using HTTPS, that will require additional setup beyond the scope of this document. +Note that this will listen on port 80 (HTTP), so traffic will be sent unencrypted. If you wish to encrypt traffic using HTTPS, +that will require additional setup beyond the scope of this document. #### Celery and Daphne -First, you will need to create a new user, clone this GitHub repository somewhere as that user, and run `pipenv install`. You will also need to give the `www-data` user access to the repository so it can serve static files. +First, you will need to create a new user, clone this GitHub repository somewhere as that user, and run `pipenv install`. +You will also need to give the `www-data` user access to the repository so it can serve static files. -If you have root access to the server with `git`, `pipenv`, and `sudo` installed, all of this can be done with the following (currently untested): +If you have root access to the server with `git`, `pipenv`, and `sudo` installed, all of this can be done with the +following (currently untested): -``` +```bash groupadd director useradd -g director director usermod -a -G director www-data @@ -72,12 +86,15 @@ mkdir /var/www/director cd /var/www/director chown director:director /var/www/director chmod 750 /var/www/director -sudo -u director git clone 'https://github.com/tjresearch/research-theo_john.git' +sudo -u director git clone 'https://github.com/tjcsl/director4' sudo -u director pipenv install ``` After this, you will need to restart Nginx. -Now you can start Celery with `pipenv run celery worker -A director --pool solo`, and Daphne with `pipenv run daphne -b 127.0.0.1 -p 9000 director.asgi:application`. Both should be run as the user you created in the previous step (if you ran the commands immediately above, this is the `director` user). You may wish to launch them using [supervisor](http://supervisord.org/) or your distribution's init system. +Now you can start Celery with `pipenv run celery worker -A director --pool solo`, and Daphne with +`pipenv run daphne -b 127.0.0.1 -p 9000 director.asgi:application`. Both should be run as the user you created in the previous step +(if you ran the commands immediately above, this is the `director` user). You may wish to launch them using +[supervisor](http://supervisord.org/) or your distribution's init system. Note that you can run multiple Daphne workers. See [Nginx HTTP Load Balancing](https://docs.nginx.com/nginx/admin-guide/load-balancer/http-load-balancer/) for more information on how to set up Nginx to handle this. diff --git a/docs/docker-image-registry.md b/docs/source/dev/docker-image-registry.md similarity index 100% rename from docs/docker-image-registry.md rename to docs/source/dev/docker-image-registry.md diff --git a/docs/source/dev/troubleshooting.rst b/docs/source/dev/troubleshooting.rst new file mode 100644 index 00000000..da2074ea --- /dev/null +++ b/docs/source/dev/troubleshooting.rst @@ -0,0 +1,21 @@ +############### +Troubleshooting +############### + +"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" +When I start Docker, I get a "Your kernel does not support swap memory limit" warning messsage +"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +By default, the kernel shipped with Ubuntu does not have memory accounting enabled. + +This is because: + + Memory and swap accounting incur an overhead of about 1% of the total available memory and a 10% overall performance degradation, even if Docker is not running. + + -- https://docs.docker.com/install/linux/linux-postinstall/#your-kernel-does-not-support-cgroup-swap-limit-capabilities + +1. Open ``/etc/default/grub`` as ``root`` on the Director appserver. +2. Add ``cgroup_enable=memory swapaccount=1`` to the ``GRUB_CMDLINE_LINUX`` line. +3. Run ``update-grub`` as ``root`` +4. Reboot. +5. It's fixed! diff --git a/docs/UMASK.md b/docs/source/dev/umask.md similarity index 96% rename from docs/UMASK.md rename to docs/source/dev/umask.md index 80073e5f..f28aa0ec 100644 --- a/docs/UMASK.md +++ b/docs/source/dev/umask.md @@ -1,12 +1,15 @@ # Umask settings -# WARNING WARNING WARNING: ALL OF THE INFORMATION ON THIS PAGE IS OUT OF DATE +```{warning} + +ALL OF THE INFORMATION ON THIS PAGE IS OUT OF DATE Director 4.0 now does something completely different for PHP sites. Changing the umask to something like `022`, `027`, or `077` would not be a big deal anymore. This page is nevertheless preserved for legacy reasons, in case you want to understand the reasoning behind Director 4.0's special handling of umasks. ---- + +``` **TL;DR**: Director sets umasks to `007` by default to fix weird permission issues. diff --git a/docs/source/guides.rst b/docs/source/guides.rst new file mode 100644 index 00000000..a5ea3fda --- /dev/null +++ b/docs/source/guides.rst @@ -0,0 +1,12 @@ +###### +Guides +###### + +A collection of guides explaining everything from +how to set up director, to how it works. + +.. toctree:: + :maxdepth: 1 + + guides/setup + guides/architecture diff --git a/docs/source/guides/architecture.rst b/docs/source/guides/architecture.rst new file mode 100644 index 00000000..28aaabb0 --- /dev/null +++ b/docs/source/guides/architecture.rst @@ -0,0 +1,21 @@ +############ +Architecture +############ + +This is a collection of guides of how Director works. + +If you're just getting started, check out the :doc:`overview `. + +.. toctree:: + :hidden: + + architecture/overview + +If you want to dive into how the different parts interact, check out +the individual guides. + +.. toctree:: + + architecture/manager + architecture/orchestrator + architecture/router_shell diff --git a/docs/source/guides/architecture/manager.rst b/docs/source/guides/architecture/manager.rst new file mode 100644 index 00000000..dfd25232 --- /dev/null +++ b/docs/source/guides/architecture/manager.rst @@ -0,0 +1,24 @@ +####### +Manager +####### + +The Manager is the driving force behind Director. It is a Django application +that runs the actual `director site `_. +For the most part, we'll only talk about the ``sites`` app in this section. + +The most useful models we use are the :class:`.Site` model and the :class:`.Operation` model. +The :class:`.Operation` model is just a representation of a task that needs to be done (ex: updating +an Nginx configuration). + +.. tip:: + + Given a :class:`.Operation`, you'll often see :func:`auto_run_operation_wrapper` being used to execute + the operation. This context manager returns a :class:`.OperationWrapper`. + The most important method you'll see used is :meth:`~.OperationWrapper.add_action`, which effectively schedules + a callback (see the overloads for how to use it as a method vs as a decorator). + Then, after running all of the callbacks, if all callbacks succeed, it will delete the original operation. + + +Communication with other parts +------------------------------ +Let's talk a little bit about how the Manager communicates with other parts. diff --git a/docs/source/guides/architecture/orchestrator.rst b/docs/source/guides/architecture/orchestrator.rst new file mode 100644 index 00000000..43bf69a0 --- /dev/null +++ b/docs/source/guides/architecture/orchestrator.rst @@ -0,0 +1,47 @@ +############ +Orchestrator +############ + +The orchestrator is responsible for the +high-level Nginx commands, as well as managing the docker containers. + +If you have not yet read :doc:`manager`, you should probably read that first. + +Nginx +----- +The Orchestrator manages it's own Nginx config to deal +with incoming requests at ``site-{site-id}.conf``. This config is +separate from the router's Nginx config, so we'll spend a moment talking +about some features and how the Orchestrator works with it. + +If the website type is ``static``, the Nginx config is where the files in ``/public`` +are served from: + +.. code-block:: nginx + + server { + # ... + + location / { + root site_dir/public; + } + } + +Note that Jinja is used for turning a template into a functional Nginx configuration. +The data to populate the template is NOT from the orchestrator, but rather from +a request from the Manager to the Orchestrator. Importantly, the Manager can send a +post request with the field ``custom_nginx_config`` to inject any other nginx configuration +needed. + +When updating the Nginx config, the orchestrator first moves the old configuration into +``site-{site-id}.conf.bak``. Only if that succeeds will it write the new configuration. + +Hosting a website +----------------- +When creating a site, the manager sends a request to the orchestrator to generate a ``nginx.conf`` +configuration, which is then moved to ``/data/nginx/director.d/``. The ``nginx.conf`` running the balancer +has a line that looks like:: + + include /data/nginx/director.d/*.conf + +This is the magic line that actually hosts the sites on the internet. diff --git a/docs/source/guides/architecture/overview.rst b/docs/source/guides/architecture/overview.rst new file mode 100644 index 00000000..8b8ab6e2 --- /dev/null +++ b/docs/source/guides/architecture/overview.rst @@ -0,0 +1,157 @@ +######## +Overview +######## + +Director is built out of several parts that play different roles +in the grand scheme of things. + +However, Director also requires a bit more prerequisite knowledge than basic Django and Flask skills (although those +are also neccessary). We'll talk a little bit about those first. + +------------- +Prerequisites +------------- + +.. note:: + + The following is not meant to be a comprehensive guide, but rather a brief overview of the concepts. + If you're interested in learning more, there are plenty of resources available online. + + +Web Requests: An Overview +~~~~~~~~~~~~~~~~~~~~~~~~~ +You want to access a website. You type in the URL, hit enter, and the website appears. +How? When you hit enter, your browser sends a request to the server hosting the website. +The request goes through several layers of software before a response with the html/css/js +is received. + +#. The request is first received by a server running a load balancer. + The load balancer forwards the request to a server that can handle it. + This way no one server is overloaded with requests while another is doing nothing. + + .. hint:: + + `Nginx `_ is a commonly used load balancer, and the one used for Director sites. + + +#. A WSGI (or ASGI for async programs) server receives the request. WSGI is a standard specification for Python web servers + to communicate with web applications. It basically translates the request into a format that the web server can understand. + Common WSGI server implementations are `gunicorn `_ and `uwsgi `_, + and these are often referred to as "application servers". +#. The web server (which is something like Django or Flask) processes the request, and communicates with the database. + It then sends a response back to the server. +#. The server sends the response back to the load balancer. +#. The load balancer sends the response back to your browser. +#. The browser renders the response and displays the website. + +We haven't mentioned caching above, and that's because there are several different caches! + +.. note:: + + The way that requests are cached isn't relevant, but we mention them for completeness + (and because it's cool). + +* Browser cache: The browser stores a copy of the (typically static) website so it doesn't have to send a request again. + This is more common with static websites that don't change often, and is why if you reload too much and then change some CSS + the site may not reflect the CSS change (Hard refresh with Ctrl+Shift+R fixes it). +* CDNs: Content Delivery Networks (CDNs) are services that cache static files (most commonly javascript and css) and serve them from a server + closer to you. This reduces latency and speeds up the website. +* Server cache: This happens at a load balancer level, and can involve things like fragment caching (caching parts of a page), + and object caching for the database. + + .. tip:: + + Nginx can actually use caching to deliver stale content if fresh content isn't + avaliable from the origin servers (``proxy_cache_use_stale``). This provides some extra fault tolerance. + +Director puts each site behind a load balancer, using Nginx. As you explore the Orchestrator, you'll see the exact +Nginx configuration that is used. + + +Docker Swarm +~~~~~~~~~~~~ +Director is able to handle different dependencies per site by using Docker containers +or each site. In order to manage these containers, Director uses Docker Swarm. Before we +start, lets talk a little bit about Docker. + +- A *Docker Image* is a package with all the dependencies in an application. +- A *Docker Container* is an instance of a Docker Image that has been given a CPU and RAM to run. + These containers are portable across all systems with Docker installed. + +A ``Dockerfile`` is a set of instructions that tells Docker how to build an image. You start by pulling ``FROM`` +a base image, and then you can ``RUN`` commands to install dependencies, ``COPY`` files into the container, and more. + +Docker Swarm allows you to orchestrate multiple Docker containers. It does this by having two types of nodes: + +- *Manager Nodes* are responsible for managing the swarm, and scheduling tasks. +- *Worker Nodes* are responsible for running the tasks. It is these nodes that run each container for each site. + +Docker swarm makes it easier to do stuff like load balancing, managing multiple nodes (called clusters), and much more. +If you're interested in learning more, you can check out the `docs `_. + + + +--------------------- +Director Architecture +--------------------- +Director is made out of four main parts. + +- The Manager is responsible for the director website. +- The Orchestrator is responsible for the staticfiles and docker builds + for user websites. +- The Router is responsible for routing traffic to the orchestrator. +- The Shell runs an ssh server using ``AsyncSSH`` to communicate with the Manager. + + +If you want to learn about each part in more detail, check out +the following articles (in the correct order): + +- :doc:`manager` +- :doc:`orchestrator` +- :doc:`router_shell` + + +Manager +~~~~~~~ + +The Manager is a "typical" Django application - it's responsible for the UI +a TJ student would see when on director. + +It uses Django Channels to handle websocket connections, and Celery to handle +long running tasks (e.g. creating sites). Effectively, it's the frontend of director. +It's where the database models for :class:`.Site` and :class:`.Operation` are located. + +Orchestrator +~~~~~~~~~~~~ + +The Orchestrator is a Flask application that handles a lot of Nginx's configuration +for individual user sites. + +Most often, these requests are things like changing the Nginx configuration, +updating Docker containers, or something similar. In fact, the Orchestrator +is where the setting of Director-specific environment variables happens! + +It also uses a Websocket server for long running tasks. + +It uses Nginx to serve static files and route incoming requests to the correct port +per site. To do this, it uses Docker to actually serve the site. + +Communication +~~~~~~~~~~~~~ + +Frequently, one component needs to be able to make HTTP/Websocket requests to another component for various reasons. +What follows is a list of all the types of access that may be required. + +- The manager should be able to make requests to all URLs on the orchestrator's Flask server. +- The manager should be able to make requests to all URLs on the orchestrator's Websocket server. +- The manager should be able to make requests to all URLs on the router's Flask server. +- The shell server should be able to make requests to all URLs with the prefix ``/shell-server/`` on the manager. + No other client, especially regular web browsers, should be allowed to make requests to these URLs. +- The shell server should be able to make requests to all URLs with the prefix ``/ws/shell-server/`` on the orchestrator's Websocket server. + It should not be allowed to make any other requests to the orchestrator or the router. + + +.. caution:: + + These restrictions are not enforced by Director 4.0 itself. Instead, each component with an HTTP + server should be behind a load balancer that enforces access restrictions. diff --git a/docs/source/guides/architecture/router_shell.rst b/docs/source/guides/architecture/router_shell.rst new file mode 100644 index 00000000..80796995 --- /dev/null +++ b/docs/source/guides/architecture/router_shell.rst @@ -0,0 +1,25 @@ +################ +Router and Shell +################ + +If you haven't read :doc:`orchestrator` and :doc:`manager`, +it's recommended to read those first to get an idea of how +the router especially fits into the big picture. + +Router +------ +Each part of Director serves a purpose: the manager for the frontend, +the orchestrator for editing configs, and so on. The job of the +router is much like an internet router: it is supposed to forward +Nginx requests to the orchestrator. Additionally, it also +is responsible for generating *Let's Encrypt* certificates if +requested by the manager. + +To be more specific, each :class:`.Site` has a custom nginx config stored in a separate +file (``site-{side-id}.conf`` at the time of writing). The router is responsible for managing +this file, making sure it is valid, and reloading nginx as needed. +This may be a little bit confusing because the Orchestrator also has a per-site Nginx config. +However, this nginx config is NOT the same: it's used to reroute requests to the orchestrator +app servers, and are stored in a different location then the orchestrators ``site-{site-id}.conf``. + + diff --git a/docs/source/guides/setup.md b/docs/source/guides/setup.md new file mode 100644 index 00000000..3542e5fc --- /dev/null +++ b/docs/source/guides/setup.md @@ -0,0 +1,18 @@ +# Vagrant setup + +1. Install [VirtualBox](https://www.virtualbox.org/wiki/Downloads) and [Vagrant](http://docs.vagrantup.com/v2/installation/index.html). + If you are running Windows, install [Git](https://gitforwindows.org/) and run `git config core.autocrlf input` to prevent line ending issues. +2. Clone the Director 4.0 repository onto your computer and `cd` into the new directory. If you have an SSH key, + run `git clone git@github.com:tjcsl/director4.git director && cd director`. Otherwise, run + `git clone https://github.com/tjcsl/director4.git director && cd director`. +3. Run `vagrant plugin install vagrant-vbguest`. +4. Run `vagrant up && vagrant reload`. This will download a Vagrant image and provision the resulting VM. +5. Run `vagrant ssh` to login to the VM. Once inside, run `cd director` to change into the repo and + `./scripts/install_dependencies.sh` to install Director's Python dependencies using Pipenv. +6. You can now work on Director. `scripts/start-servers.sh` will open a `tmux` session with the four servers each running in a separate pane. + - Note: If you are not familiar with `tmux`, we recommend + [https://www.hamvocke.com/blog/a-quick-and-easy-guide-to-tmux/](https://www.hamvocke.com/blog/a-quick-and-easy-guide-to-tmux/) + and [tmuxcheatsheet](https://tmuxcheatsheet.com/) as starting resources. +7. When you are finished, type `exit` to exit the VM and `vagrant halt` to stop it. When you want to work on Director 4.0 again, + `cd` into this directory, run `vagrant up` and `vagrant ssh` to launch the VM and connect to it, + and then run `exit` and `vagrant halt` to exit and shut it down. diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 00000000..d30238ea --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,19 @@ +Director4 Documentation +======================= + +Director 4.0 is a website management and hosting platform (based on the former +Director platform) that is designed to scale. It replaces the former Director +platform which had problems with performance, scalability, and ease of use. + +.. note:: + + From now onwards, we may refer to the Director 4.0 platform as "Director", + with the understanding that we are referring to the 4.0 version. + +.. toctree:: + :maxdepth: 1 + :caption: Contents: + + usage + guides + dev diff --git a/docs/source/usage.rst b/docs/source/usage.rst new file mode 100644 index 00000000..15cd2121 --- /dev/null +++ b/docs/source/usage.rst @@ -0,0 +1,14 @@ +##### +Usage +##### + +Welcome to Director! Check out the following articles +to learn how to use it: + +.. + TODO: Add toctree for Director documentation + This can be done by adding files to usage/ and + using + .. toctree:: + + usage/name_of_file diff --git a/manager/director/apps/sites/models.py b/manager/director/apps/sites/models.py index b3e4d4f1..56560ea6 100644 --- a/manager/director/apps/sites/models.py +++ b/manager/director/apps/sites/models.py @@ -172,6 +172,7 @@ def serialize_for_appserver(self) -> Dict[str, Any]: "name": self.name, "type": self.type, "is_being_served": self.is_being_served, + # remove duplicates before converting into list "no_redirect_domains": list({split_domain(url) for url in self.list_urls()}), "primary_url_base": main_url, "database_info": ( diff --git a/orchestrator/orchestrator/database.py b/orchestrator/orchestrator/database.py index afc2b496..1246fdfd 100644 --- a/orchestrator/orchestrator/database.py +++ b/orchestrator/orchestrator/database.py @@ -107,6 +107,7 @@ def open_site_cursor(database_info: Dict[str, Any]) -> ContextManager[Any]: def create_database(database_info: Dict[str, Any]) -> None: if database_info["db_type"] == "postgres": with open_admin_cursor(database_info["host"], dbname="postgres") as cursor: + # check if the username exists cursor.execute( "SELECT 1 FROM pg_catalog.pg_user WHERE usename = %s", (database_info["username"],) ) diff --git a/vagrant-config/nginx.conf b/vagrant-config/nginx.conf index eaf5fdbd..6dbf59e6 100644 --- a/vagrant-config/nginx.conf +++ b/vagrant-config/nginx.conf @@ -61,4 +61,4 @@ http { include /etc/nginx/director.d/*.conf; } -# vim: ft=conf ts=4 expandtab autoindent sw=4 +# vim: ft=nginx ts=4 expandtab autoindent sw=4