testing: fix up various aspects that are breaking tests (bug 1887042)

WIP DO NOT MERGE
Skip following modules:
- test_auth
- test_dockerflow
- test_notifications
- test_diff_warnings
- test_landing_job (partial)

- fix ups to get tests to run
- remove flask celery functionality
- add boilerplate django celery impl
- add db to github workflow

Author: zzzeid

.github/workflows/build.yml

@@ -8,6 +8,19 @@ on:
 jobs:
   run-tests:
     runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres:latest
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+        ports:
+          - 5432:5432
+        options:
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python
@@ -22,7 +35,10 @@ jobs:
           python -m pip install -r requirements.txt
           python -m pip install -e .
       - name: Test
+        env:
+          DEFAULT_DB_HOST: 127.0.0.1
         run: |
           source env/bin/activate
+          lando migrate
           lando test
-          pytest
+          pytest src/lando/api

Dockerfile

@@ -4,9 +4,20 @@ EXPOSE 80
 ENV PYTHONUNBUFFERED=1
 ENV PYTHONDONTWRITEBYTECODE 1
 RUN adduser --system --no-create-home app
+
 RUN mkdir /code
 COPY ./ /code
 RUN pip install --upgrade pip
+
+
+# Install the Rust toolchain. Some packages do not have pre-built wheels (e.g.
+# rs-parsepatch) and require this in order to compile.
+RUN curl https://sh.rustup.rs -sSf | sh -s -- -y
+
+# Include ~/.cargo/bin in PATH.
+# See: rust-lang.org/tools/install (Configuring the PATH environment variable).
+ENV PATH="/root/.cargo/bin:${PATH}"
+
 RUN pip install -r /code/requirements.txt
 RUN pip install -e /code
 USER app

pyproject.toml

@@ -32,3 +32,6 @@ build-backend = "setuptools.build_meta"
 
 [tool.setuptools.packages.find]
 where = ["src"]
+
+[tool.pytest.ini_options]
+DJANGO_SETTINGS_MODULE = "lando.settings"

requirements.txt

@@ -1,6 +1,15 @@
 black
 jinja2
+kombu
 pytest
 pytest-django
+python-hglib==2.6.2
+python-jose
+redis
+requests-mock
 ruff
 uwsgi
+celery
+datadog
+rs-parsepatch
+networkx

src/lando/api/legacy/api/transplants.py

@@ -10,7 +10,7 @@
 from lando.main.support import ProblemException, problem, g
 from lando import settings
 
-from lando.api import auth
+from lando.api.legacy import auth
 from lando.api.legacy.commit_message import format_commit_message
 from lando.api.legacy.decorators import require_phabricator_api_key
 from lando.main.models.landing_job import (

src/lando/api/legacy/app.py

@@ -9,7 +9,6 @@
 import connexion
 from connexion.resolver import RestyResolver
 
-import landoapi.models  # noqa, makes sure alembic knows about the models.
 from lando.api.legacy.auth import auth0_subsystem
 from lando.api.legacy.cache import cache_subsystem
 from lando.api.legacy.celery import celery_subsystem

src/lando/api/legacy/auth.py

@@ -32,7 +32,7 @@
 
 
 def get_auth_token() -> str:
-    auth = request.headers.get("Authorization")
+    auth = request["headers"].get("Authorization")
     if auth is None:
         raise ProblemException(
             401,

src/lando/api/legacy/celery.py

@@ -1,9 +1,9 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
+import os
 import logging
 
-import flask
 from celery import Celery
 from celery.signals import (
     after_task_publish,
@@ -15,87 +15,24 @@
 )
 from datadog import statsd
 
-from lando.api.legacy.systems import Subsystem
-
 logger = logging.getLogger(__name__)
 
 
-class FlaskCelery(Celery):
-    """Celery which executes task in a flask app context."""
-
-    def __init__(self, *args, **kwargs):
-        # Avoid passing the flask app to base Celery.
-        flask_app = kwargs.pop("app", None)
-
-        super().__init__(*args, **kwargs)
-
-        # Important to run this after __init__ since task_cls
-        # argument to base Celery can change what we're basing on.
-        self._flask_override_task_class()
-
-        if flask_app is not None:
-            self.init_app(flask_app)
-
-    @property
-    def dispatch_disabled(self):
-        """Will the Celery job system dispatch tasks to the workers?"""
-        return bool(self.app.config.get("DISABLE_CELERY"))
-
-    def init_app(self, app, config=None):
-        """Initialize with a flask app."""
-        self.app = app
-
-        config = config or {}
-        self.conf.update(main=app.import_name, **config)
-
-        if self.dispatch_disabled:
-            logger.warning(
-                "DISABLE_CELERY application configuration variable set, the Celery job "
-                "system has been disabled! Any features that depend on the job system "
-                "will not function."
-            )
-
-    def _flask_override_task_class(self):
-        """Change Task class to one which executes in a flask context."""
-        # Define a Task subclass that saves a reference to self in the Task object so
-        # the task object can find self.app (the Flask application object) even if
-        # self.app hasn't been set yet.
-        #
-        # We need to delay all of the task's calls to self.app using a custom Task class
-        # because the reference to self.app may not be valid at the time the Celery
-        # application object creates it set of Task objects.  The programmer may
-        # set self.app via the self.init_app() method at any time in the future.
-        #
-        # self.app is expected to be valid and usable by Task objects after the web
-        # application is fully initialized and ready to serve requests.
-        BaseTask = self.Task
-        celery_self = self
-
-        class FlaskTask(BaseTask):
-            """A Celery Task subclass that has a reference to a Flask app."""
+# Set the default Django settings module for the 'celery' program.
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'lando.settings')
 
-            def __call__(self, *args, **kwargs):
-                # Override immediate calling of tasks, such as mytask().  This call
-                # method is used by the Celery worker process.
-                if flask.has_app_context():
-                    return super().__call__(*args, **kwargs)
+app = Celery('lando')
 
-                with celery_self.app.app_context():
-                    return super().__call__(*args, **kwargs)
+# Using a string here means the worker doesn't have to serialize
+# the configuration object to child processes.
+# - namespace='CELERY' means all celery-related configuration keys
+#   should have a `CELERY_` prefix.
+app.config_from_object('django.conf:settings', namespace='CELERY')
 
-            def apply_async(self, *args, **kwargs):
-                # Override delayed calling of tasks, such as mytask.apply_async().
-                # This call method is used by the Celery app when it wants to
-                # schedule a job for eventual execution on a worker.
-                if celery_self.dispatch_disabled:
-                    return None
-                else:
-                    return super().apply_async(*args, **kwargs)
+# Load task modules from all registered Django apps.
+app.autodiscover_tasks()
 
-        self.Task = FlaskTask
-
-
-celery = FlaskCelery()
+celery = app
 
 
 @after_task_publish.connect
@@ -129,29 +66,3 @@ def count_task_retried(**kwargs):
 def setup_celery_logging(**kwargs):
     # Prevent celery from overriding our logging configuration.
     pass
-
-
-class CelerySubsystem(Subsystem):
-    name = "celery"
-
-    def init_app(self, app):
-        super().init_app(app)
-
-        # Import tasks to discover celery tasks.
-        import landoapi.tasks  # noqa
-
-        celery.init_app(
-            self.flask_app,
-            config={"broker_url": settings.CELERY_BROKER_URL},
-        )
-        celery.log.setup()
-
-    def ready(self):
-        if settings.DISABLE_CELERY:
-            return True
-
-        # TODO: Check connection to CELERY_BROKER_URL
-        return True
-
-
-celery_subsystem = CelerySubsystem()

src/lando/api/legacy/cli.py

@@ -8,13 +8,11 @@
 from typing import Optional
 
 import click
-import connexion
-from flask.cli import FlaskGroup
 
 from lando.main.models.configuration import (
     ConfigurationKey,
     ConfigurationVariable,
-    VariableType,
+    VariableTypeChoices,
 )
 from lando.api.legacy.systems import Subsystem
 
@@ -35,7 +33,7 @@ def get_subsystems(exclude: Optional[list[Subsystem]] = None) -> list[Subsystem]
     return [s for s in SUBSYSTEMS if s not in exclusions]
 
 
-def create_lando_api_app() -> connexion.App:
+def create_lando_api_app():
     from lando.api.legacy.app import construct_app, load_config
 
     config = load_config()
@@ -46,7 +44,6 @@ def create_lando_api_app() -> connexion.App:
     return app.app
 
 
-@click.group(cls=FlaskGroup, create_app=create_lando_api_app)
 def cli():
     """Lando API cli."""
 
@@ -83,21 +80,21 @@ def landing_worker():
 def run_pre_deploy_sequence():
     """Runs the sequence of commands required before a deployment."""
     ConfigurationVariable.set(
-        ConfigurationKey.API_IN_MAINTENANCE, VariableType.BOOL, "1"
+        ConfigurationKey.API_IN_MAINTENANCE, VariableTypeChoices.BOOL, "1"
     )
     ConfigurationVariable.set(
-        ConfigurationKey.LANDING_WORKER_PAUSED, VariableType.BOOL, "1"
+        ConfigurationKey.LANDING_WORKER_PAUSED, VariableTypeChoices.BOOL, "1"
     )
 
 
 @cli.command(name="run-post-deploy-sequence")
 def run_post_deploy_sequence():
     """Runs the sequence of commands required after a deployment."""
     ConfigurationVariable.set(
-        ConfigurationKey.API_IN_MAINTENANCE, VariableType.BOOL, "0"
+        ConfigurationKey.API_IN_MAINTENANCE, VariableTypeChoices.BOOL, "0"
     )
     ConfigurationVariable.set(
-        ConfigurationKey.LANDING_WORKER_PAUSED, VariableType.BOOL, "0"
+        ConfigurationKey.LANDING_WORKER_PAUSED, VariableTypeChoices.BOOL, "0"
     )
 
 

src/lando/api/legacy/decorators.py

@@ -37,7 +37,7 @@ def __init__(self, optional: bool = False, provide_client: bool = True):
     def __call__(self, f: Callable) -> Callable:
         @functools.wraps(f)
         def wrapped(*args, **kwargs):
-            api_key = request.headers.get("X-Phabricator-API-Key")
+            api_key = request["headers"].get("X-Phabricator-API-Key")
 
             if api_key is None and not self.optional:
                 return problem(

src/lando/api/legacy/hg.py

@@ -165,7 +165,7 @@ class HgRepo:
         "extensions.purge": "",
         "extensions.strip": "",
         "extensions.rebase": "",
-        "extensions.set_landing_system": "/app/hgext/set_landing_system.py",
+        "extensions.set_landing_system": "/code/src/lando/api/legacy/hgext/set_landing_system.py",
     }
 
     def __init__(self, path, config=None):

src/lando/api/legacy/mocks/auth.py

@@ -90,4 +90,4 @@ def access_token(self):
 
     @property
     def mock_headers(self):
-        return [("Authorization", "Bearer {}".format(self.access_token))]
+        return dict([("Authorization", "Bearer {}".format(self.access_token))])

src/lando/api/legacy/projects.py

@@ -4,7 +4,8 @@
 import logging
 from typing import Optional
 
-from lando.api.legacy.cache import DEFAULT_CACHE_KEY_TIMEOUT_SECONDS, cache
+from django.core.cache import cache
+
 from lando.api.legacy.phabricator import PhabricatorClient, result_list_to_phid_dict
 
 logger = logging.getLogger(__name__)
@@ -50,14 +51,14 @@ def project_search(
     project_phids.sort()
     cache_key = ",".join(project_phids)
 
-    if cache.has(cache_key):
+    if cache.has_key(cache_key):
         return cache.get(cache_key)
 
     projects = phabricator.call_conduit(
         "project.search", constraints={"phids": project_phids}
     )
     result = result_list_to_phid_dict(phabricator.expect(projects, "data"))
-    cache.set(cache_key, result, timeout=DEFAULT_CACHE_KEY_TIMEOUT_SECONDS)
+    cache.set(cache_key, result)
     return result
 
 
@@ -80,7 +81,7 @@ def get_project_phid(
         A string with the project's PHID or None if the project isn't found.
     """
     key = f"PROJECT_{project_slug}"
-    if cache.has(key):
+    if cache.has_key(key):
         return cache.get(key)
 
     project = phabricator.single(
@@ -92,7 +93,7 @@ def get_project_phid(
     )
 
     value = phabricator.expect(project, "phid") if project else None
-    cache.set(key, value, timeout=DEFAULT_CACHE_KEY_TIMEOUT_SECONDS)
+    cache.set(key, value)
     return value
 
 

src/lando/api/legacy/transplants.py

@@ -216,8 +216,8 @@ def warning_previously_landed(*, revision, diff, **kwargs):
 
     job = (
         LandingJob.revisions_query([revision_id])
-        .filter_by(status=LandingJobStatus.LANDED)
-        .order_by(LandingJob.updated_at.desc())
+        .filter(status=LandingJobStatus.LANDED)
+        .order_by("-updated_at")
         .first()
     )
 
@@ -233,7 +233,7 @@ def warning_previously_landed(*, revision, diff, **kwargs):
         revision_to_diff_id.update(legacy_data)
     landed_diff_id = revision_to_diff_id[revision_id]
     same = diff_id == landed_diff_id
-    only_revision = len(job.revisions) == 1
+    only_revision = job.revisions.count() == 1
 
     return (
         "Already landed with {is_same_string} diff ({landed_diff_id}), "