Merge pull request #88 from theodo/support-wpt-private-instances

Add support for WebPageTest Private Instances

Author: phacks

CHANGELOG.md

@@ -5,6 +5,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+- Add support for WebPageTest Private Instances 🎉 (@phacks)
 
 ## [1.0.3] - 2019-12-10
 

README.md

@@ -29,6 +29,7 @@
 - 👥 Invite the whole team so that everyone (devs, ops, product, marketing…) is involved in performance
 - 🗺 Audit the performance of individual URLs or entire user journeys ([even on Single Page Apps!](https://css-tricks.com/recipes-for-performance-testing-single-page-applications-in-webpagetest/))
 - 📸 Easily access and compare WebPageTest results between audits
+- 🙈 Can be used with your own Private Instance of WebPageTest
 
 You can try a demo version by logging in to https://falco.theo.do with the credentials `demo / demodemo`.
 
@@ -40,6 +41,8 @@ You can deploy Falco on Heroku by clicking on the following button:
 
 You will need to provide your credit card details to Heroku, but you will be under the free tier by default. You can find more details on why they are needed and Heroku’s pricing policy [in the docs](https://getfal.co).
 
+After deployment, you can connect to Falco (and the admin interface at `/admin/`) with the credentials `admin` and `admin`: make sure to change your password after connecting!
+
 <details>
 <summary>Heroku Teams user? Click here to deploy Falco.</summary>
 <br />

backend/audits/tasks.py

@@ -37,11 +37,13 @@ def request_audit(audit_uuid):
         payload["url"] = audit.page.url
         payload["lighthouse"] = 1
         payload["k"] = audit.page.project.wpt_api_key
+        wpt_instance_url = audit.page.project.wpt_instance_url
     elif audit.script is not None:
         payload["script"] = audit.script.script
         payload["k"] = audit.script.project.wpt_api_key
+        wpt_instance_url = audit.script.project.wpt_instance_url
 
-    r = requests.post("https://www.webpagetest.org/runtest.php", params=payload)
+    r = requests.post(f"{wpt_instance_url}/runtest.php", params=payload)
     response = r.json()
     if response["statusCode"] == 200:
         audit_status_queueing = AuditStatusHistory(
@@ -85,9 +87,14 @@ def poll_audit_results(audit_uuid, json_url):
         audit_status_requested.save()
         poll_audit_results.apply_async((audit_uuid, json_url), countdown=15)
     elif status_code == 200:
+        if audit.page is not None:
+            wpt_instance_url = audit.page.project.wpt_instance_url
+        elif audit.script is not None:
+            wpt_instance_url = audit.script.project.wpt_instance_url
+
         parsed_url = urlparse(json_url)
         test_id = parse_qs(parsed_url.query)["test"][0]
-        wpt_results_user_url = f"https://www.webpagetest.org/result/{test_id}"
+        wpt_results_user_url = f"{wpt_instance_url}/result/{test_id}"
         try:
             if audit.page is not None:
                 project = audit.page.project
@@ -259,37 +266,53 @@ def clean_old_audit_statuses():
 
 
 @shared_task
-def get_wpt_audit_configurations():
+def get_wpt_audit_configurations(wpt_instance_url="https://webpagetest.org"):
     """gets all the available locations from WPT"""
-    response = requests.get("https://www.webpagetest.org/getLocations.php?f=json&k=A")
+
+    # For some reason, the key mask to get API-available locations is different between
+    # public and private WPT instances
+    wpt_key_mask = ""
+    if wpt_instance_url == "https://webpagetest.org":
+        wpt_key_mask = "A"
+
+    response = requests.get(
+        f"{wpt_instance_url}/getLocations.php?f=json&k={wpt_key_mask}"
+    )
 
     if response.status_code != 200:
         logging.error("Invalid response from WebPageTest API: non-200 response code")
-        return
+        raise Exception("Invalid response from WebPageTest API: non-200 response code")
 
     try:
         data = response.json()["data"]
     except KeyError:
         logging.error(
             "Invalid response from WebPageTest API: 'data' key is not present"
         )
-        return
+        raise Exception(
+            "Invalid response from WebPageTest API: 'data' key is not present"
+        )
 
-    for available_audit_parameter in AvailableAuditParameters.objects.all():
+    for available_audit_parameter in AvailableAuditParameters.objects.filter(
+        wpt_instance_url=wpt_instance_url
+    ):
         available_audit_parameter.is_active = False
         available_audit_parameter.save()
 
     for location, location_data in data.items():
         browsers = location_data["Browsers"].split(",")
-        group = location_data["group"]
+        group = location_data.get(
+            "group", ""
+        )  # Private instances locations may not be grouped
         label = location_data["labelShort"]
-        for brower in browsers:
+        for browser in browsers:
             configuration, created = AvailableAuditParameters.objects.update_or_create(
-                browser=brower,
+                browser=browser,
                 location=location,
                 defaults={
                     "location_label": label,
                     "location_group": group,
                     "is_active": True,
                 },
+                wpt_instance_url=wpt_instance_url,
             )

backend/audits/tests/json_mocks/wpt_GET_getLocations.json

@@ -0,0 +1,58 @@
+{
+  "statusCode": 200,
+  "statusText": "Ok",
+  "data": {
+    "Dulles_MotoG4": {
+      "Label": "Moto G (gen 4)",
+      "location": "Dulles_MotoG4",
+      "Browsers": "Moto G4 - Chrome,Moto G4 - Firefox",
+      "status": "OK",
+      "relayServer": null,
+      "relayLocation": null,
+      "labelShort": "Dulles, VA",
+      "group": "Android Devices - Dulles, VA",
+      "PendingTests": {
+        "p1": 0,
+        "p2": 1,
+        "p3": 0,
+        "p4": 0,
+        "p5": 84,
+        "p6": 0,
+        "p7": 0,
+        "p8": 0,
+        "p9": 0,
+        "Total": 101,
+        "HighPriority": 0,
+        "LowPriority": 85,
+        "Testing": 16,
+        "Idle": 0
+      }
+    },
+    "Dulles_MotoG": {
+      "Label": "Moto G (gen 1)",
+      "location": "Dulles_MotoG",
+      "Browsers": "Moto G - Chrome",
+      "status": "OK",
+      "relayServer": null,
+      "relayLocation": null,
+      "labelShort": "Dulles, VA",
+      "group": "Android Devices - Dulles, VA",
+      "PendingTests": {
+        "p1": 0,
+        "p2": 0,
+        "p3": 0,
+        "p4": 0,
+        "p5": 0,
+        "p6": 0,
+        "p7": 0,
+        "p8": 0,
+        "p9": 0,
+        "Total": 0,
+        "HighPriority": 0,
+        "LowPriority": 0,
+        "Testing": 0,
+        "Idle": 13
+      }
+    }
+  }
+}

backend/audits/tests/test_tasks.py

@@ -1,8 +1,7 @@
 import httpretty
 from unittest.mock import MagicMock
-import json
-from django.test import TestCase, override_settings
-from audits.tasks import request_audit
+from django.test import TestCase
+from audits.tasks import request_audit, get_wpt_audit_configurations
 from projects.models import (
     Page,
     Project,
@@ -15,22 +14,21 @@
 
 class TasksTestCase(TestCase):
     @httpretty.activate
-    @override_settings(CELERY_EAGER=True)
     @patch("audits.tasks.poll_audit_results")
     def test_request_audit(self, poll_audit_results_mock):
         poll_audit_results_mock.apply_async = MagicMock()
         POST_runtest_data = open("audits/tests/json_mocks/wpt_POST_runtest.json").read()
-        GET_jsonResults_data = json.load(
-            open("audits/tests/json_mocks/wpt_GET_jsonResult.json")
-        )
+        GET_jsonResults_data = open(
+            "audits/tests/json_mocks/wpt_GET_jsonResult.json"
+        ).read()
         httpretty.register_uri(
             httpretty.POST,
-            "https://www.webpagetest.org/runtest.php",
+            "https://webpagetest.org/runtest.php",
             body=POST_runtest_data,
         )
         httpretty.register_uri(
             httpretty.GET,
-            "https://www.webpagetest.org/jsonResult.php?test=191024_HA_976b046886025ec8693cbe4f1145929e",
+            "https://webpagetest.org/jsonResult.php?test=191024_HA_976b046886025ec8693cbe4f1145929e",
             body=GET_jsonResults_data,
         )
         project = Project.objects.create()
@@ -71,3 +69,36 @@ def test_request_audit(self, poll_audit_results_mock):
             ),
             countdown=15,
         )
+
+    @httpretty.activate
+    def test_get_wpt_audit_configurations__create_configurations_for_default_instance(
+        self
+    ):
+        GET_getLocations_data = open(
+            "audits/tests/json_mocks/wpt_GET_getLocations.json"
+        ).read()
+        httpretty.register_uri(
+            httpretty.GET,
+            "https://webpagetest.org/getLocations.php?f=json&k=A",
+            body=GET_getLocations_data,
+        )
+        get_wpt_audit_configurations()
+        available_audit_parameters = AvailableAuditParameters.objects.all()
+        self.assertEqual(len(available_audit_parameters), 3)
+
+    @httpretty.activate
+    def test_get_wpt_audit_configurations__create_configurations_for_private_instance(
+        self
+    ):
+        GET_getLocations_data = open(
+            "audits/tests/json_mocks/wpt_GET_getLocations.json"
+        ).read()
+        private_instance_name = "http://myprivateinstance.com"
+        httpretty.register_uri(
+            httpretty.GET,
+            f"{private_instance_name}/getLocations.php?f=json&k=",
+            body=GET_getLocations_data,
+        )
+        get_wpt_audit_configurations(private_instance_name)
+        available_audit_parameters = AvailableAuditParameters.objects.all()
+        self.assertEqual(len(available_audit_parameters), 3)

backend/projects/migrations/0031_auto_20191122_1154.py

@@ -0,0 +1,21 @@
+# Generated by Django 2.2.4 on 2019-11-22 10:54
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [("projects", "0030_auto_20190904_1152")]
+
+    operations = [
+        migrations.AddField(
+            model_name="availableauditparameters",
+            name="wpt_instance_url",
+            field=models.CharField(default="https://webpagetest.org", max_length=100),
+        ),
+        migrations.AddField(
+            model_name="project",
+            name="wpt_instance_url",
+            field=models.CharField(default="https://webpagetest.org", max_length=100),
+        ),
+    ]

backend/projects/models.py

@@ -14,6 +14,9 @@ class Project(BaseModel):
         User, blank=True, related_name="member_of", through="ProjectMemberRole"
     )
     is_active = models.BooleanField(default=True)
+    wpt_instance_url = models.CharField(
+        max_length=100, blank=False, null=False, default="https://webpagetest.org"
+    )
 
     @property
     def latest_audit_at(self):
@@ -108,6 +111,9 @@ class AvailableAuditParameters(BaseModel):
     location_label = models.CharField(max_length=100, blank=False, null=False)
     location_group = models.CharField(max_length=100, blank=False, null=False)
     is_active = models.BooleanField(default=True)
+    wpt_instance_url = models.CharField(
+        max_length=100, blank=False, null=False, default="https://webpagetest.org"
+    )
 
     class Meta:
         ordering = ("location", "browser")

backend/projects/serializers.py

@@ -86,7 +86,13 @@ class Meta:
 class AvailableAuditParameterSerializer(serializers.ModelSerializer):
     class Meta:
         model = AvailableAuditParameters
-        fields = ("uuid", "browser", "location_label", "location_group")
+        fields = (
+            "uuid",
+            "browser",
+            "location_label",
+            "location_group",
+            "wpt_instance_url",
+        )
 
 
 class ProjectAuditParametersSerializer(serializers.ModelSerializer):
@@ -156,5 +162,6 @@ class Meta:
             "screenshot_url",
             "latest_audit_at",
             "wpt_api_key",
+            "wpt_instance_url",
             "has_siblings",
         )

backend/projects/urls.py

@@ -17,6 +17,9 @@
         views.project_audit_parameters_detail,
     ),
     path("available_audit_parameters", views.available_audit_parameters),
+    path(
+        "available_audit_parameters/discover", views.discover_available_audit_parameters
+    ),
     path("<uuid:project_uuid>/scripts", views.project_scripts),
     path("<uuid:project_uuid>/scripts/<uuid:script_uuid>", views.project_script_detail),
 ]

backend/projects/views.py

@@ -6,6 +6,7 @@
 from rest_framework import permissions, status
 from rest_framework.decorators import api_view, permission_classes
 from rest_framework.parsers import JSONParser
+from requests.exceptions import ConnectionError
 from projects.models import (
     Page,
     Project,
@@ -28,6 +29,8 @@
     is_admin_of_project,
 )
 
+from audits.tasks import get_wpt_audit_configurations
+
 
 def get_user_projects(user_id):
     return Project.objects.filter(members__id=user_id, is_active=True)
@@ -404,6 +407,49 @@ def project_members(request, project_uuid):
     )
 
 
+@swagger_auto_schema(
+    methods=["post"],
+    request_body=openapi.Schema(
+        type="object", properties={"wpt_instance_url": openapi.Schema(type="string")}
+    ),
+    responses={
+        201: openapi.Response(
+            "Returns discovered available audit parameters for the WPT instance URL passed in parameter",
+            AvailableAuditParameterSerializer,
+        )
+    },
+    tags=["Project Audit Parameters"],
+)
+@api_view(["POST"])
+def discover_available_audit_parameters(request):
+    data = JSONParser().parse(request)
+    if "wpt_instance_url" in data:
+        try:
+            get_wpt_audit_configurations(data["wpt_instance_url"])
+        except ConnectionError:
+            return JsonResponse(
+                {
+                    "error": "UNREACHABLE",
+                    "details": "The WPT instance is not reachable, please check the URL",
+                },
+                status=status.HTTP_400_BAD_REQUEST,
+            )
+        available_audit_parameters = AvailableAuditParameters.objects.filter(
+            is_active=True
+        )
+        serializer = AvailableAuditParameterSerializer(
+            available_audit_parameters, many=True
+        )
+        return JsonResponse(serializer.data, safe=False)
+    return JsonResponse(
+        {
+            "error": "MISSING_PARAMETER",
+            "details": "You must provide a wpt_instance_url in the request body",
+        },
+        status=status.HTTP_400_BAD_REQUEST,
+    )
+
+
 @swagger_auto_schema(
     methods=["get"],
     responses={