[backcompat-tests] Remove tests against older webserver than code server

Author: smackesey

.buildkite/dagster-buildkite/dagster_buildkite/steps/integration.py

@@ -51,8 +51,6 @@ def build_integration_steps() -> List[BuildkiteStep]:
 
 def build_backcompat_suite_steps() -> List[BuildkiteTopLevelStep]:
     tox_factors = [
-        "webserver-latest-release",
-        "webserver-earliest-release",
         "user-code-latest-release",
         "user-code-earliest-release",
     ]
@@ -66,43 +64,25 @@ def build_backcompat_suite_steps() -> List[BuildkiteTopLevelStep]:
 
 def backcompat_extra_cmds(_, factor: str) -> List[str]:
     tox_factor_map = {
-        "webserver-latest-release": {
-            "webserver": LATEST_DAGSTER_RELEASE,
-            "user_code": DAGSTER_CURRENT_BRANCH,
-        },
-        "webserver-earliest-release": {
-            "webserver": EARLIEST_TESTED_RELEASE,
-            "user_code": DAGSTER_CURRENT_BRANCH,
-        },
-        "user-code-latest-release": {
-            "webserver": DAGSTER_CURRENT_BRANCH,
-            "user_code": LATEST_DAGSTER_RELEASE,
-        },
-        "user-code-earliest-release": {
-            "webserver": DAGSTER_CURRENT_BRANCH,
-            "user_code": EARLIEST_TESTED_RELEASE,
-        },
+        "user-code-latest-release": LATEST_DAGSTER_RELEASE,
+        "user-code-earliest-release": EARLIEST_TESTED_RELEASE,
     }
 
-    release_mapping = tox_factor_map[factor]
-    webserver_version = release_mapping["webserver"]
+    webserver_version = DAGSTER_CURRENT_BRANCH
     webserver_library_version = _get_library_version(webserver_version)
-    webserver_package = _infer_webserver_package(webserver_version)
-    user_code_version = release_mapping["user_code"]
+    user_code_version = tox_factor_map[factor]
     user_code_library_version = _get_library_version(user_code_version)
     user_code_definitions_file = _infer_user_code_definitions_files(user_code_version)
 
     return [
         f"export EARLIEST_TESTED_RELEASE={EARLIEST_TESTED_RELEASE}",
-        f"export WEBSERVER_PACKAGE={webserver_package}",
         f"export USER_CODE_DEFINITIONS_FILE={user_code_definitions_file}",
         "pushd integration_tests/test_suites/backcompat-test-suite/webserver_service",
         " ".join(
             [
                 "./build.sh",
                 webserver_version,
                 webserver_library_version,
-                webserver_package,
                 user_code_version,
                 user_code_library_version,
                 user_code_definitions_file,
@@ -119,17 +99,6 @@ def backcompat_extra_cmds(_, factor: str) -> List[str]:
     ]
 
 
-def _infer_webserver_package(release: str) -> str:
-    """Returns `dagster-webserver` if on source or version >=1.3.14 (first dagster-webserver
-    release), `dagit` otherwise.
-    """
-    if release == "current_branch":
-        return "dagster-webserver"
-    else:
-        version = packaging.version.parse(release)
-        return "dagit" if version < packaging.version.Version("1.3.14") else "dagster-webserver"
-
-
 def _infer_user_code_definitions_files(release: str) -> str:
     """Returns `repo.py` if on source or version >=1.0, `legacy_repo.py` otherwise."""
     if release == "current_branch":

integration_tests/test_suites/backcompat-test-suite/README.md

@@ -1,6 +1,6 @@
 # Backcompatability Integration Tests
 
-This test suite ensures that the branch Dagster code can successfully communicate cross-process with older Dagster code.
+This test suite ensures that the branch Dagster code can successfully communicate cross-process with older Dagster code. It only tests older user code (i.e. code server), because our backcompat policy is that the host process (e.g. webserver) version must be >= user code version.
 
 ## Looking at test artifacts in BuildKite
 
@@ -38,19 +38,13 @@ If you are on MacOS, ensure you have docker running
 
 From `integration_tests/test_suites/backcompat-test-suite` run any of the following commands
 
-- `pytest -m webserver-latest-release -xvv -ff tests/test_backcompat.py`
-- `pytest -m webserver-earliest-release -xvv -ff tests/test_backcompat.py`
 - `pytest -m user-code-latest-release -xvv -ff tests/test_backcompat.py`
 - `pytest -m user-code-earliest-release -xvv -ff tests/test_backcompat.py`
-- `tox webserver-latest-release`
-- `tox webserver-earliest-release`
 - `tox user-code-latest-release`
 - `tox user-code-earliest-release`
 
 where:
 
-- webserver-latest-release: webserver on most recent release (`dagster-webserver`) and user code on current branch
-- webserver-earliest-release: webserver on earliest release (this will run the deprecated `dagit` package if `EARLIEST_TESTED_RELEASE` is less than 1.3.14) to maintain backcompat for, and user code on current branch
 - user-code-latest-release: webserver on current branch and user code on latest minor release
 - user-code-earliest-release: webserver on current branch and user code on earliest release to maintain backcompat for
 

integration_tests/test_suites/backcompat-test-suite/tests/test_backcompat.py

@@ -30,13 +30,12 @@
 pytest_plugins = ["dagster_test.fixtures"]
 
 
-# Maps pytest marks to (webserver-version, user-code-version) 2-tuples. These versions are CORE
-# versions-- library versions are derived from these later with `get_library_version`.
+# Maps pytest marks to user code version numbers. The webserver version is always the current
+# branch. Note that these versions are CORE versions-- library versions are derived from these later
+# with `get_library_version`.
 MARK_TO_VERSIONS_MAP = {
-    "webserver-earliest-release": (EARLIEST_TESTED_RELEASE, DAGSTER_CURRENT_BRANCH),
-    "user-code-earliest-release": (DAGSTER_CURRENT_BRANCH, EARLIEST_TESTED_RELEASE),
-    "webserver-latest-release": (MOST_RECENT_RELEASE_PLACEHOLDER, DAGSTER_CURRENT_BRANCH),
-    "user-code-latest-release": (DAGSTER_CURRENT_BRANCH, MOST_RECENT_RELEASE_PLACEHOLDER),
+    "user-code-earliest-release": EARLIEST_TESTED_RELEASE,
+    "user-code-latest-release": MOST_RECENT_RELEASE_PLACEHOLDER,
 }
 
 
@@ -64,19 +63,6 @@ def infer_user_code_definitions_files(release: str) -> str:
         return "legacy_repo.py" if version < packaging.version.Version("1.0") else "repo.py"
 
 
-def infer_webserver_package(release: str) -> str:
-    """Returns `dagster-webserver` if on source or version >=1.3.14 (first dagster-webserver
-    release), `dagit` otherwise.
-    """
-    if release == "current_branch":
-        return "dagster-webserver"
-    else:
-        if not EARLIEST_TESTED_RELEASE:
-            check.failed("Environment variable `$EARLIEST_TESTED_RELEASE` must be set.")
-        version = packaging.version.parse(release)
-        return "dagit" if version < packaging.version.Version("1.3.14") else "dagster-webserver"
-
-
 def assert_run_success(client: DagsterGraphQLClient, run_id: str) -> None:
     start_time = time.time()
     while True:
@@ -117,25 +103,17 @@ def dagster_most_recent_release() -> str:
     scope="session",
 )
 def release_test_map(request, dagster_most_recent_release: str) -> Mapping[str, str]:
-    webserver_version = request.param[0]
-    if webserver_version == MOST_RECENT_RELEASE_PLACEHOLDER:
-        webserver_version = dagster_most_recent_release
-    user_code_version = request.param[1]
-    if user_code_version == MOST_RECENT_RELEASE_PLACEHOLDER:
-        user_code_version = dagster_most_recent_release
-
-    return {"webserver": webserver_version, "user_code": user_code_version}
+    user_code_version = (
+        dagster_most_recent_release
+        if request.param == MOST_RECENT_RELEASE_PLACEHOLDER
+        else request.param
+    )
+    return {"webserver": DAGSTER_CURRENT_BRANCH, "user_code": user_code_version}
 
 
-def check_webserver_connection(host: str, webserver_package: str, retrying_requests) -> None:
-    if webserver_package == "dagit":
-        url_path = "dagit_info"
-        json_key = "dagit_version"
-    else:  # dagster-webserver
-        url_path = "server_info"
-        json_key = "dagster_webserver_version"
-    result = retrying_requests.get(f"http://{host}:3000/{url_path}")
-    assert result.json().get(json_key)
+def check_webserver_connection(host: str, retrying_requests) -> None:
+    result = retrying_requests.get(f"http://{host}:3000/server_info")
+    assert result.json().get("dagster_webserver_version")
 
 
 def upload_docker_logs_to_buildkite():
@@ -204,7 +182,6 @@ def docker_service(
 
     # Infer additional parameters used in our docker setup from webserver/usercode versions.
     webserver_library_version = get_library_version(webserver_version)
-    webserver_package = infer_webserver_package(webserver_version)
     user_code_library_version = get_library_version(user_code_version)
     user_code_definitions_files = infer_user_code_definitions_files(user_code_version)
 
@@ -214,7 +191,6 @@ def docker_service(
             file_relative_path(docker_compose_file, "./build.sh"),
             webserver_version,
             webserver_library_version,
-            webserver_package,
             user_code_version,
             user_code_library_version,
             user_code_definitions_files,
@@ -223,11 +199,10 @@ def docker_service(
     build_process.wait()
     assert build_process.returncode == 0
 
-    # Create the docker service. $WEBSERVER_PACKAGE and $USER_CODE_DEFINITIONS_FILE are referenced
-    # in the entrypoint of a container so we need to make them available as environment variables
-    # while creating the service.
+    # Create the docker service. $USER_CODE_DEFINITIONS_FILE is referenced in the entrypoint of a
+    # container so we need to make it available as an environment variable while creating the
+    # service.
     env = {
-        "WEBSERVER_PACKAGE": webserver_package,
         "USER_CODE_DEFINITIONS_FILE": user_code_definitions_files,
         **os.environ,
     }
@@ -259,15 +234,14 @@ def graphql_client(
     release_test_map: Mapping[str, str], retrying_requests
 ) -> Iterator[DagsterGraphQLClient]:
     webserver_version = release_test_map["webserver"]
-    webserver_package = infer_webserver_package(webserver_version)
 
     # On Buildkite, the docker service is set up and torn down outside of pytest. The webserver is
     # exposed through the BACKCOMPAT_TESTS_WEBSERVER_HOST environment variable. We can just connect
     # to it and yield the client.
     if IS_BUILDKITE:
         webserver_host = os.environ["BACKCOMPAT_TESTS_WEBSERVER_HOST"]
         try:
-            check_webserver_connection(webserver_host, webserver_package, retrying_requests)
+            check_webserver_connection(webserver_host, retrying_requests)
             yield DagsterGraphQLClient(webserver_host, port_number=3000)
         finally:
             upload_docker_logs_to_buildkite()
@@ -281,8 +255,7 @@ def graphql_client(
             webserver_version=webserver_version,
             user_code_version=release_test_map["user_code"],
         ):
-            print("INSIDE DOCKER SERVICE")
-            check_webserver_connection(webserver_host, webserver_package, retrying_requests)
+            check_webserver_connection(webserver_host, retrying_requests)
             yield DagsterGraphQLClient(webserver_host, port_number=3000)
 
 

integration_tests/test_suites/backcompat-test-suite/tox.ini

@@ -24,7 +24,5 @@ allowlist_externals =
   uv
 commands =
   !windows: /bin/bash -c '! pip list --exclude-editable | grep -e dagster'
-  webserver-latest-release: pytest -m "webserver-latest-release" -vv -s {posargs}
-  webserver-earliest-release: pytest -m "webserver-earliest-release" -vv -s {posargs}
   user-code-latest-release: pytest -m "user-code-latest-release" -vv -s {posargs}
   user-code-earliest-release: pytest -m "user-code-earliest-release" -vv -s {posargs}

integration_tests/test_suites/backcompat-test-suite/webserver_service/Dockerfile_webserver_release

@@ -2,7 +2,6 @@ FROM python:3.8-slim
 
 ARG WEBSERVER_VERSION
 ARG WEBSERVER_LIBRARY_VERSION
-ARG WEBSERVER_PACKAGE
 
 COPY pins.txt pins.txt
 
@@ -12,7 +11,7 @@ RUN pip install \
     dagster-postgres=="${WEBSERVER_LIBRARY_VERSION}" \
     dagster-docker=="${WEBSERVER_LIBRARY_VERSION}" \
     dagster-graphql=="${WEBSERVER_VERSION}" \
-    ${WEBSERVER_PACKAGE}=="${WEBSERVER_VERSION}"
+    dagster-webserver=="${WEBSERVER_VERSION}"
 
 ENV DAGSTER_HOME=/opt/dagster/dagster_home/
 RUN mkdir -p $DAGSTER_HOME

integration_tests/test_suites/backcompat-test-suite/webserver_service/build.sh

@@ -1,10 +1,9 @@
 #!/bin/bash
 WEBSERVER_VERSION=$1
 WEBSERVER_LIBRARY_VERSION=$2
-WEBSERVER_PACKAGE=$3
-USER_CODE_VERSION=$4
-USER_CODE_LIBRARY_VERSION=$5
-USER_CODE_DEFINITIONS_FILE=$6
+USER_CODE_VERSION=$3
+USER_CODE_LIBRARY_VERSION=$4
+USER_CODE_DEFINITIONS_FILE=$5
 
 if [ "$WEBSERVER_VERSION" = "current_branch" ]; then
     export WEBSERVER_DOCKERFILE="./Dockerfile_webserver_source"
@@ -65,7 +64,6 @@ echo -e "--- \033[32m:docker: Building Docker images\033[0m"
 docker-compose build \
     --build-arg WEBSERVER_VERSION="${WEBSERVER_VERSION}" \
     --build-arg WEBSERVER_LIBRARY_VERSION="${WEBSERVER_LIBRARY_VERSION}" \
-    --build-arg WEBSERVER_PACKAGE="${WEBSERVER_PACKAGE}" \
     --build-arg USER_CODE_VERSION="${USER_CODE_VERSION}" \
     --build-arg USER_CODE_LIBRARY_VERSION="${USER_CODE_LIBRARY_VERSION}" \
     --build-arg USER_CODE_DEFINITIONS_FILE="${USER_CODE_DEFINITIONS_FILE}"

integration_tests/test_suites/backcompat-test-suite/webserver_service/docker-compose.yml

@@ -48,7 +48,7 @@ services:
     expose:
       - "3000"
     entrypoint:
-      - ${WEBSERVER_PACKAGE}
+      - dagster-webserver
       - -h
       - "0.0.0.0"
       - -p