testing: fix up various aspects that are breaking tests (bug 1887042)

Author: zzzeid

pyproject.toml

@@ -32,3 +32,6 @@ build-backend = "setuptools.build_meta"
 
 [tool.setuptools.packages.find]
 where = ["src"]
+
+[tool.pytest.ini_options]
+DJANGO_SETTINGS_MODULE = "lando.settings"

requirements.txt

@@ -1,6 +1,11 @@
 black
 jinja2
+kombu
 pytest
 pytest-django
+python-hglib
+python-jose
+redis
+requests-mock
 ruff
 uwsgi

src/lando/api/legacy/projects.py

@@ -4,7 +4,8 @@
 import logging
 from typing import Optional
 
-from lando.api.legacy.cache import DEFAULT_CACHE_KEY_TIMEOUT_SECONDS, cache
+from django.core.cache import cache
+
 from lando.api.legacy.phabricator import PhabricatorClient, result_list_to_phid_dict
 
 logger = logging.getLogger(__name__)
@@ -57,7 +58,7 @@ def project_search(
         "project.search", constraints={"phids": project_phids}
     )
     result = result_list_to_phid_dict(phabricator.expect(projects, "data"))
-    cache.set(cache_key, result, timeout=DEFAULT_CACHE_KEY_TIMEOUT_SECONDS)
+    cache.set(cache_key, result)
     return result
 
 
@@ -92,7 +93,7 @@ def get_project_phid(
     )
 
     value = phabricator.expect(project, "phid") if project else None
-    cache.set(key, value, timeout=DEFAULT_CACHE_KEY_TIMEOUT_SECONDS)
+    cache.set(key, value)
     return value
 
 

src/lando/api/legacy/uplift.py

@@ -18,8 +18,7 @@
     Version,
 )
 
-from lando.api import bmo
-from lando.api.legacy.cache import DEFAULT_CACHE_KEY_TIMEOUT_SECONDS, cache
+from lando.api.legacy import bmo
 from lando.api.legacy.phabricator import PhabricatorClient
 from lando.api.legacy.phabricator_patch import patch_to_changes
 from lando.api.legacy.repos import (
@@ -68,9 +67,9 @@ def get_uplift_request_form(revision: dict) -> Optional[str]:
     return bug
 
 
-@cache.cached(
-    key_prefix="uplift-repositories", timeout=DEFAULT_CACHE_KEY_TIMEOUT_SECONDS
-)
+#@cache.cached(
+#    key_prefix="uplift-repositories"
+#)
 def get_uplift_repositories(phab: PhabricatorClient) -> list:
     repos = phab.call_conduit(
         "diffusion.repository.search",

src/lando/api/legacy/workers/landing_worker.py

@@ -26,8 +26,8 @@
     TreeApprovalRequired,
     TreeClosed,
 )
-from lando.api.legacy.models.configuration import ConfigurationKey
-from lando.api.legacy.models.landing_job import LandingJob, LandingJobAction, LandingJobStatus
+from lando.main.models.configuration import ConfigurationKey
+from lando.main.models.landing_job import LandingJob, LandingJobAction, LandingJobStatus
 from lando.api.legacy.notifications import (
     notify_user_of_bug_update_failure,
     notify_user_of_landing_failure,

src/lando/api/tests/conftest.py

@@ -9,17 +9,15 @@
 from pathlib import Path
 from types import SimpleNamespace
 
-import flask.testing
+from django.test import Client
 import pytest
 import redis
 import requests
 import requests_mock
-import sqlalchemy
-from flask import current_app
-from pytest_flask.plugin import JSONResponse
+from django.http import JsonResponse as JSONResponse
+from django.core.cache import cache
 
-from lando.api.legacy.app import SUBSYSTEMS, construct_app, load_config
-from lando.api.legacy.cache import cache
+from lando import settings
 from lando.api.legacy.mocks.auth import TEST_JWKS, MockAuth0
 from lando.api.legacy.phabricator import PhabricatorClient
 from lando.api.legacy.projects import (
@@ -29,10 +27,8 @@
     SEC_PROJ_SLUG,
 )
 from lando.api.legacy.repos import SCM_LEVEL_1, SCM_LEVEL_3, Repo
-from lando.api.legacy.storage import db as _db
-from lando.api.legacy.tasks import celery
 from lando.api.legacy.transplants import CODE_FREEZE_OFFSET, tokens_are_equal
-from tests.mocks import PhabricatorDouble, TreeStatusDouble
+from lando.api.tests.mocks import PhabricatorDouble, TreeStatusDouble
 
 PATCH_NORMAL_1 = r"""
 # HG changeset patch
@@ -102,7 +98,7 @@ def _patch(number=0):
     return _patch
 
 
-class JSONClient(flask.testing.FlaskClient):
+class JSONClient(Client):
     """Custom Flask test client that sends JSON by default.
 
     HTTP methods have a 'json=...' keyword that will JSON-encode the
@@ -242,39 +238,6 @@ def init_app(self, app, db):
     monkeypatch.setattr("landoapi.storage.migrate", StubAlembic())
 
 
-@pytest.fixture
-def app(versionfile, docker_env_vars, disable_migrations, mocked_repo_config):
-    """Needed for pytest-flask."""
-    config = load_config()
-    # We need the TESTING setting turned on to get tracebacks when testing API
-    # endpoints with the TestClient.
-    config["TESTING"] = True
-    config["CACHE_DISABLED"] = True
-    app = construct_app(config)
-    flask_app = app.app
-    flask_app.test_client_class = JSONClient
-    for system in SUBSYSTEMS:
-        system.init_app(flask_app)
-
-    return flask_app
-
-
-@pytest.fixture
-def db(app):
-    """Reset database for each test."""
-    try:
-        _db.engine.connect()
-    except sqlalchemy.exc.OperationalError:
-        if EXTERNAL_SERVICES_SHOULD_BE_PRESENT:
-            raise
-        else:
-            pytest.skip("Could not connect to PostgreSQL")
-    _db.create_all()
-    yield _db
-    _db.session.remove()
-    _db.drop_all()
-
-
 @pytest.fixture
 def jwks(monkeypatch):
     monkeypatch.setattr("landoapi.auth.get_jwks", lambda *args, **kwargs: TEST_JWKS)
@@ -356,8 +319,8 @@ def set_value(val):
 @pytest.fixture
 def get_phab_client(app):
     def get_client(api_key=None):
-        api_key = api_key or current_app.config["PHABRICATOR_UNPRIVILEGED_API_KEY"]
-        return PhabricatorClient(current_app.config["PHABRICATOR_URL"], api_key)
+        api_key = api_key or settings.PHABRICATOR_UNPRIVILEGED_API_KEY
+        return PhabricatorClient(settings.PHABRICATOR_URL, api_key)
 
     return get_client
 
@@ -379,21 +342,6 @@ def redis_cache(app):
     cache.init_app(app, config={"CACHE_TYPE": "null", "CACHE_NO_NULL_WARNING": True})
 
 
-@pytest.fixture
-def celery_app(app):
-    """Configure our app's Celery instance for use with the celery_worker fixture."""
-    # The test suite will fail if we don't override the default worker and
-    # default task set.
-    # Note: the test worker will fail if we don't specify a result_backend.  The test
-    # harness uses the backend for a custom ping() task that it uses as a health check.
-    celery.conf.update(broker_url="memory://", result_backend="rpc")
-    # Workaround for https://github.com/celery/celery/issues/4032.  If 'tasks.ping' is
-    # missing from the loaded task list then the test worker will fail with an
-    # AssertionError.
-    celery.loader.import_module("celery.contrib.testing.tasks")
-    return celery
-
-
 @pytest.fixture
 def treestatus_url():
     """A string holding the Tree Status base URL."""

src/lando/api/tests/mocks.py

@@ -13,7 +13,7 @@
     ReviewerStatus,
 )
 from lando.api.legacy.treestatus import TreeStatus, TreeStatusError
-from tests.canned_responses.phabricator.diffs import (
+from lando.api.tests.canned_responses.phabricator.diffs import (
     CANNED_DEFAULT_DIFF_CHANGES,
     CANNED_RAW_DEFAULT_DIFF,
 )

src/lando/api/tests/test_try.py

@@ -11,7 +11,7 @@
     get_timestamp_from_git_date_header,
     parse_git_author_information,
 )
-from lando.api.legacy.models.landing_job import LandingJob, LandingJobStatus
+from lando.main.models.landing_job import LandingJob, LandingJobStatus
 from lando.api.legacy.repos import SCM_LEVEL_1, Repo
 from lando.api.legacy.workers.landing_worker import LandingWorker
 

src/lando/jinja2.py src/lando/jinja.py

@@ -1,6 +1,8 @@
 from django.contrib import admin
 
-from lando.main.models import LandingJob, Repo, Revision, Worker
+from lando.main.models.landing_job import LandingJob
+from lando.main.models.revision import Revision
+from lando.main.models.base import Repo, Worker
 
 admin.site.register(LandingJob, admin.ModelAdmin)
 admin.site.register(Revision, admin.ModelAdmin)

src/lando/main/admin.py

@@ -1,3 +0,0 @@
-from lando.main.models.base import *
-from lando.main.models.landing_job import *
-from lando.main.models.revision import *

src/lando/main/models/__init__.py

@@ -8,7 +8,7 @@
 from django.db import models
 from django.utils.translation import gettext_lazy
 
-from lando.main.models import BaseModel
+from lando.main.models.base import BaseModel
 
 logger = logging.getLogger(__name__)
 

src/lando/main/models/configuration.py

@@ -17,7 +17,7 @@
 from mots.config import FileConfig
 from mots.directory import Directory
 
-from lando.main.models import BaseModel
+from lando.main.models.base import BaseModel
 from lando.main.models.revision import Revision, RevisionLandingJob
 
 logger = logging.getLogger(__name__)

src/lando/main/models/landing_job.py

@@ -13,7 +13,7 @@
 from django.db import models
 from django.utils.translation import gettext_lazy
 
-from lando.main.models import BaseModel
+from lando.main.models.base import BaseModel
 from lando.utils import build_patch_for_revision
 
 logger = logging.getLogger(__name__)

src/lando/main/models/revision.py

@@ -59,7 +59,7 @@
         "DIRS": [],
         "APP_DIRS": True,
         "OPTIONS": {
-            "environment": "lando.jinja2.environment"
+            "environment": "lando.jinja.environment"
         },
     },
     {