Implement parallelism on post server validations

Avoid failing on alias record validation when others pass

Author: dormant-user

.pre-commit-config.yaml

@@ -1,25 +1,38 @@
 fail_fast: true
+exclude: ^docs/
 repos:
-  -
-    repo: https://github.com/PyCQA/flake8
-    rev: '6.1.0'
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
     hooks:
-      -
-        id: flake8
-        additional_dependencies:
-          - flake8-docstrings
-          - flake8-sfs
-        args: [--max-line-length=120, --extend-ignore=SFS3 D107 SFS301 D100 D104 D401 SFS101 SFS201]
+      - id: check-added-large-files
+      - id: check-ast
+      - id: check-byte-order-marker
+      - id: check-builtin-literals
+      - id: check-case-conflict
+      - id: check-docstring-first
+      - id: check-executables-have-shebangs
+      - id: check-shebang-scripts-are-executable
+      - id: check-merge-conflict
+      - id: check-toml
+      - id: check-vcs-permalinks
+      - id: check-xml
+      - id: debug-statements
+      - id: destroyed-symlinks
+      - id: detect-aws-credentials
+      - id: detect-private-key
+      - id: end-of-file-fixer
+      - id: fix-byte-order-marker
+      - id: mixed-line-ending
+      - id: name-tests-test
+      - id: requirements-txt-fixer
+      - id: trailing-whitespace
 
-  -
-    repo: https://github.com/PyCQA/isort
-    rev: '5.12.0'
+  - repo: https://github.com/PyCQA/isort
+    rev: 5.12.0
     hooks:
-      -
-        id: isort
+      - id: isort
 
-  -
-    repo: local
+  - repo: local
     hooks:
       -
         id: docs

README.md

@@ -47,7 +47,7 @@ vpn_server.delete_vpn_server()
 ## Coding Standards
 Docstring format: [`Google`](https://google.github.io/styleguide/pyguide.html#38-comments-and-docstrings) <br>
 Styling conventions: [`PEP 8`](https://www.python.org/dev/peps/pep-0008/) <br>
-Clean code with pre-commit hooks: [`flake8`](https://flake8.pycqa.org/en/latest/) and 
+Clean code with pre-commit hooks: [`flake8`](https://flake8.pycqa.org/en/latest/) and
 [`isort`](https://pycqa.github.io/isort/)
 
 ### [Release Notes](https://github.com/thevickypedia/vpn-server/blob/main/release_notes.rst)

docs/README.md

@@ -47,7 +47,7 @@ vpn_server.delete_vpn_server()
 ## Coding Standards
 Docstring format: [`Google`](https://google.github.io/styleguide/pyguide.html#38-comments-and-docstrings) <br>
 Styling conventions: [`PEP 8`](https://www.python.org/dev/peps/pep-0008/) <br>
-Clean code with pre-commit hooks: [`flake8`](https://flake8.pycqa.org/en/latest/) and 
+Clean code with pre-commit hooks: [`flake8`](https://flake8.pycqa.org/en/latest/) and
 [`isort`](https://pycqa.github.io/isort/)
 
 ### [Release Notes](https://github.com/thevickypedia/vpn-server/blob/main/release_notes.rst)

docs/_sources/README.md.txt

@@ -47,7 +47,7 @@ vpn_server.delete_vpn_server()
 ## Coding Standards
 Docstring format: [`Google`](https://google.github.io/styleguide/pyguide.html#38-comments-and-docstrings) <br>
 Styling conventions: [`PEP 8`](https://www.python.org/dev/peps/pep-0008/) <br>
-Clean code with pre-commit hooks: [`flake8`](https://flake8.pycqa.org/en/latest/) and 
+Clean code with pre-commit hooks: [`flake8`](https://flake8.pycqa.org/en/latest/) and
 [`isort`](https://pycqa.github.io/isort/)
 
 ### [Release Notes](https://github.com/thevickypedia/vpn-server/blob/main/release_notes.rst)

docs/index.html

@@ -310,20 +310,27 @@ <h1>Welcome to VPN Server’s documentation!<a class="headerlink" href="#welcome
 
 <dl class="py method">
 <dt class="sig sig-object py" id="vpn.main.VPNServer._test_get">
-<span class="sig-name descname"><span class="pre">_test_get</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">server_hostname</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">timeout</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Tuple</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">(3,</span> <span class="pre">3)</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">&#x2192;</span> <span class="sig-return-typehint"><span class="pre">Response</span></span></span><a class="headerlink" href="#vpn.main.VPNServer._test_get" title="Permalink to this definition">¶</a></dt>
+<span class="sig-name descname"><span class="pre">_test_get</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">host</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">timeout</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Tuple</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">(3,</span> <span class="pre">3)</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">retries</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">int</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">5</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">&#x2192;</span> <span class="sig-return-typehint"><span class="pre">Response</span></span></span><a class="headerlink" href="#vpn.main.VPNServer._test_get" title="Permalink to this definition">¶</a></dt>
 <dd><p>Test GET connection with multiple hostnames.</p>
 <dl class="field-list simple">
 <dt class="field-odd">Parameters<span class="colon">:</span></dt>
 <dd class="field-odd"><ul class="simple">
-<li><p><strong>server_hostname</strong> – Public IP address or DNS name or alias record (entrypoint)</p></li>
+<li><p><strong>host</strong> – Public IP address or DNS name or alias record (entrypoint)</p></li>
 <li><p><strong>timeout</strong> – Tuple of connection timeout and read timeout.</p></li>
+<li><p><strong>retries</strong> – Number of times to retry in case of connection errors.</p></li>
 </ul>
 </dd>
-<dt class="field-even">Returns<span class="colon">:</span></dt>
-<dd class="field-even"><p>Response object.</p>
+</dl>
+<div class="admonition seealso">
+<p class="admonition-title">See also</p>
+<p>Retries with exponential intervals between each attempt in case of a failure.</p>
+</div>
+<dl class="field-list simple">
+<dt class="field-odd">Returns<span class="colon">:</span></dt>
+<dd class="field-odd"><p>Response object.</p>
 </dd>
-<dt class="field-odd">Return type<span class="colon">:</span></dt>
-<dd class="field-odd"><p>Response</p>
+<dt class="field-even">Return type<span class="colon">:</span></dt>
+<dd class="field-even"><p>Response</p>
 </dd>
 </dl>
 </dd></dl>
@@ -339,11 +346,13 @@ <h1>Welcome to VPN Server’s documentation!<a class="headerlink" href="#welcome
 </dl>
 <div class="admonition seealso">
 <p class="admonition-title">See also</p>
+<p>All the tests run in parallel to improve runtime.</p>
 <ul class="simple">
 <li><p>GET request against the public IP of the ec2 instance.</p></li>
 <li><p>GET request against the public DNS of the ec2 instance.</p></li>
 <li><p>SSH connection with the OpenVPN Access Server.</p></li>
 <li><p>Test <code class="docutils literal notranslate"><span class="pre">openvpnas</span></code> service availability on the server.</p></li>
+<li><p>Test alias record if values for <code class="docutils literal notranslate"><span class="pre">hosted_zone</span></code> and <code class="docutils literal notranslate"><span class="pre">subdomain</span></code> were provided</p></li>
 </ul>
 </div>
 <dl class="field-list simple">

docs/searchindex.js

@@ -1,6 +1,10 @@
 Release Notes
 =============
 
+1.6 (01/29/2024)
+----------------
+- Includes speed and stability improvements for server validations
+
 1.5.2 (09/28/2023)
 ------------------
 - Includes instance information in return statements during creation and deletion

pre_commit.sh

@@ -4,4 +4,4 @@
 from vpn.models import (config, exceptions, image_factory,  # noqa: F401
                         logger, route53, server, util)
 
-version = "1.5.2"
+version = "1.6"

release_notes.rst

@@ -3,6 +3,7 @@
 import time
 import warnings
 from logging import Logger
+from multiprocessing.pool import ThreadPool
 from typing import Dict, Tuple, Union
 
 import boto3
@@ -90,6 +91,8 @@ def __init__(self, **kwargs: Unpack[Union[EnvConfig, Logger]]):
         self.image_id = None
         self.zone_id = None
 
+        self.engine = inflect.engine()
+
     def _init(self,
               start: Union[bool, int]) -> None:
         """Initializer function.
@@ -403,24 +406,35 @@ def _terminate_ec2_instance(self,
             self.logger.warning('API call to terminate the instance has failed.')
             self.logger.error(error)
 
-    def _test_get(self, server_hostname: str, timeout: Tuple = (3, 3)) -> Response:
+    def _test_get(self, host: str, timeout: Tuple = (3, 3), retries: int = 5) -> Response:
         """Test GET connection with multiple hostnames.
 
         Args:
-            server_hostname: Public IP address or DNS name or alias record (entrypoint)
+            host: Public IP address or DNS name or alias record (entrypoint)
             timeout: Tuple of connection timeout and read timeout.
+            retries: Number of times to retry in case of connection errors.
+
+        See Also:
+            Retries with exponential intervals between each attempt in case of a failure.
 
         Returns:
             Response:
             Response object.
         """
-        try:
-            response = requests.get(url=f"https://{server_hostname}:{self.env.vpn_port}",
-                                    verify=False, timeout=timeout)
-            self.logger.debug(response)
-            return response
-        except requests.RequestException as error:
-            self.logger.error(error)
+        for i in range(1, retries + 1):
+            try:
+                response = requests.get(url=f"https://{host}:{self.env.vpn_port}",
+                                        verify=False, timeout=timeout)
+                self.logger.debug(response)
+                return response
+            except requests.RequestException as error:
+                if i < retries:
+                    exponent = 2 ** i
+                    self.logger.info("Failed to validate %s in %s attempt, next attempt in %d seconds",
+                                     host, self.engine.ordinal(i), exponent)
+                    time.sleep(exponent)
+                else:
+                    self.logger.error(error)
 
     def _tester(self, data: Dict[str, Union[str, int]]) -> None:
         """Tests ``GET`` and ``SSH`` connections on the existing server.
@@ -429,31 +443,44 @@ def _tester(self, data: Dict[str, Union[str, int]]) -> None:
             data: Takes the instance information in a dictionary format as an argument.
 
         See Also:
+            All the tests run in parallel to improve runtime.
+
             - GET request against the public IP of the ec2 instance.
             - GET request against the public DNS of the ec2 instance.
             - SSH connection with the OpenVPN Access Server.
             - Test ``openvpnas`` service availability on the server.
+            - Test alias record if values for ``hosted_zone`` and ``subdomain`` were provided
 
         Raises:
             AssertionError:
             When any of the tests fail.
         """
         urllib3.disable_warnings(InsecureRequestWarning)  # Disable warnings for self-signed certificates
-        self.logger.info(f"Testing GET connection to https://{data.get('public_ip')}:{self.env.vpn_port}")
-        ip_check = self._test_get(data.get('public_ip'))
-        host_check = self._test_get(data.get('public_dns'))
-        alias_check = Response()
-        alias_check.status_code = 200
+        alias_thread = None
         if self.settings.entrypoint:
-            alias_check = self._test_get(self.settings.entrypoint)
-        assert all((ip_check, host_check, alias_check)) and all((ip_check.ok, host_check.ok, alias_check.ok)), \
+            alias_thread = ThreadPool(processes=1).apply_async(self._test_get,
+                                                               args=(self.settings.entrypoint,))
+        self.logger.info("Testing GET connections to VPN server, via hostname and IP address.")
+        ip_thread = ThreadPool(processes=1).apply_async(self._test_get,
+                                                        kwds=dict(host=data.get('public_ip'), retries=2))
+        host_thread = ThreadPool(processes=1).apply_async(self._test_get,
+                                                          kwds=dict(host=data.get('public_dns'), retries=2))
+        ip_check = ip_thread.get()
+        host_check = host_thread.get()
+        assert all((ip_check, host_check)) and all((ip_check.ok, host_check.ok)), \
             "One or more tests for GET connection has failed. Please check the logs for more information."
-        self.logger.info(f"Testing SSH connection to {data.get('public_dns')}")
+        self.logger.info("Connections to VPN server, via hostname and IP address were successful.")
+        self.logger.info("Testing SSH connection to %s", data.get('public_dns'))
         test_ssh = Server(username=self.env.vpn_username, hostname=data.get('public_dns'), logger=self.logger,
                           env=self.env, settings=self.settings)
         test_ssh.test_service(display=False, timeout=5)
-        self.logger.info(f"Connection to https://{data.get('public_ip')}:{self.env.vpn_port} and "
-                         f"SSH to {data.get('public_dns')} was successful.")
+        self.logger.info(f"SSH to {data.get('public_dns')} was successful.")
+        if alias_thread:
+            if (alias_response := alias_thread.get()) and alias_response.ok:
+                self.logger.info("Connection to VPN server, via alias record %s was successful.",
+                                 self.settings.entrypoint)
+            else:
+                self.logger.error("Failed to test A record, it may be DNS propagation delay. ")
 
     def test_vpn(self) -> None:
         """Tests the ``GET`` and ``SSH`` connections to an existing VPN server."""
@@ -548,15 +575,18 @@ def create_vpn_server(self) -> Union[Dict[str, Union[str, int]], None]:
 
         self._configure_vpn(instance.public_dns_name)
         if self.settings.entrypoint:
-            change_record_set(source=self.settings.entrypoint,
-                              destination=instance.public_ip_address,
-                              logger=self.logger,
-                              client=self.route53_client,
-                              zone_id=self.zone_id, action='UPSERT')
-            instance_info['entrypoint'] = self.settings.entrypoint
-            with open(self.env.vpn_info, 'w') as file:
-                json.dump(instance_info, file, indent=2)
-                file.flush()
+            if change_record_set(source=self.settings.entrypoint,
+                                 destination=instance.public_ip_address,
+                                 logger=self.logger,
+                                 client=self.route53_client,
+                                 zone_id=self.zone_id, action='UPSERT'):
+                instance_info['entrypoint'] = self.settings.entrypoint
+                with open(self.env.vpn_info, 'w') as file:
+                    json.dump(instance_info, file, indent=2)
+                    file.flush()
+            else:
+                self.logger.error("Failed to add entrypoint as alias")
+                self.settings.entrypoint = None
 
         try:
             self._tester(data=instance_info)
@@ -580,7 +610,7 @@ def _configure_vpn(self, public_dns: str) -> None:
             try:
                 server = Server(hostname=public_dns, username='openvpnas', logger=self.logger,
                                 env=self.env, settings=self.settings)
-                self.logger.info("Connection established on %s attempt", inflect.engine().ordinal(i + 1))
+                self.logger.info("Connection established on %s attempt", self.engine.ordinal(i + 1))
                 break
             except Exception as error:
                 self.logger.error(error)