feat(server): add mysql support (#175)

Author: HappyTobi

.travis.yml

@@ -39,13 +39,16 @@ jobs:
       before_script:
         - google-chrome-stable --version
         - export CHROME_PATH="$(which google-chrome-stable)"
-        - export POSTGRES_DB_URL="postgres://postgres@localhost/lighthouse_ci_test"
         - export RUN_E2E_TESTS="true"
+        - export POSTGRES_DB_URL="postgres://postgres@localhost/lighthouse_ci_test"
         - psql -c 'create database lighthouse_ci_test;' -U postgres
+        - export MYSQL_DB_URL="mysql://root@localhost/lighthouse_ci_test"
+        - mysql -e 'create database if not exists lighthouse_ci_test;' -u root
       script:
         - yarn build
         - yarn test
         - yarn ci:dogfood
       services:
         - postgresql
+        - mysql
         - xvfb

docs/cli.md

@@ -288,7 +288,7 @@ Options:
   --logLevel        [string] [choices: "silent", "verbose"] [default: "verbose"]
   --port, -p                                            [number] [default: 9001]
   --storage.sqlDialect
-                    [string] [choices: "sqlite", "postgres"] [default: "sqlite"]
+                    [string] [choices: "sqlite", "postgres", "mysql"] [default: "sqlite"]
   --storage.sqlDatabasePath              The path to a SQLite database on disk.
   --storage.sqlConnectionUrl             The connection url to a postgres
                                          database.

package.json

@@ -60,6 +60,7 @@
     "jest-fetch-mock": "^2.1.2",
     "jest-image-snapshot": "^2.9.0",
     "lerna": "^3.16.4",
+    "mysql2": "^1.7.0",
     "parcel-bundler": "^1.12.3",
     "pg": "^7.12.1",
     "pg-hstore": "^2.3.3",

packages/cli/src/server/server.js

@@ -27,14 +27,14 @@ function buildCommand(yargs) {
     },
     'storage.sqlDialect': {
       type: 'string',
-      choices: ['sqlite', 'postgres'],
+      choices: ['sqlite', 'postgres', 'mysql'],
       default: 'sqlite',
     },
     'storage.sqlDatabasePath': {
       description: 'The path to a SQLite database on disk.',
     },
     'storage.sqlConnectionUrl': {
-      description: 'The connection url to a postgres database.',
+      description: 'The connection url to a postgres or mysql database.',
     },
     'storage.sqlConnectionSsl': {
       type: 'boolean',

packages/server/package.json

@@ -22,7 +22,7 @@
     "compression": "^1.7.4",
     "express": "^4.16.4",
     "morgan": "^1.9.1",
-    "sequelize": "^4.42.0",
+    "sequelize": "^4.44.3",
     "umzug": "^2.2.0",
     "uuid": "^3.3.2"
   },

packages/server/src/api/routes/projects.js

@@ -69,6 +69,9 @@ function createRouter(context) {
   router.get(
     '/:projectId/builds',
     handleAsyncError(async (req, res) => {
+      if (Number.isInteger(parseInt(req.query.limit))) {
+        req.query.limit = parseInt(req.query.limit);
+      }
       const builds = await context.storageMethod.getBuilds(req.params.projectId, req.query);
       res.json(builds);
     })

packages/server/src/api/storage/sql/build-model.js

@@ -24,9 +24,11 @@ const attributes = {
   avatarUrl: {type: Sequelize.STRING(256)},
   ancestorHash: {type: Sequelize.STRING(40)},
   externalBuildUrl: {type: Sequelize.STRING(256)},
-  runAt: {type: Sequelize.DATE()}, // should mostly be equal to createdAt but modifiable by the consumer
-  committedAt: {type: Sequelize.DATE()},
-  ancestorCommittedAt: {type: Sequelize.DATE()},
+  runAt: {type: Sequelize.DATE(6)}, // should mostly be equal to createdAt but modifiable by the consumer
+  committedAt: {type: Sequelize.DATE(6)},
+  ancestorCommittedAt: {type: Sequelize.DATE(6)},
+  createdAt: {type: Sequelize.DATE(6)},
+  updatedAt: {type: Sequelize.DATE(6)},
 };
 
 module.exports = {

packages/server/src/api/storage/sql/migrations/20191008-initial.js

@@ -18,8 +18,8 @@ module.exports = {
       id: {type: Sequelize.UUID(), primaryKey: true},
       name: {type: Sequelize.STRING(40)},
       externalUrl: {type: Sequelize.STRING(256)},
-      createdAt: {type: Sequelize.DATE()},
-      updatedAt: {type: Sequelize.DATE()},
+      createdAt: {type: Sequelize.DATE(6)},
+      updatedAt: {type: Sequelize.DATE(6)},
     });
     await queryInterface.createTable('builds', {
       id: {type: Sequelize.UUID(), primaryKey: true},
@@ -32,9 +32,9 @@ module.exports = {
       avatarUrl: {type: Sequelize.STRING(256)},
       ancestorHash: {type: Sequelize.STRING(40)},
       externalBuildUrl: {type: Sequelize.STRING(256)},
-      runAt: {type: Sequelize.DATE()},
-      createdAt: {type: Sequelize.DATE()},
-      updatedAt: {type: Sequelize.DATE()},
+      runAt: {type: Sequelize.DATE(6)},
+      createdAt: {type: Sequelize.DATE(6)},
+      updatedAt: {type: Sequelize.DATE(6)},
     });
     await queryInterface.createTable('runs', {
       id: {type: Sequelize.UUID(), primaryKey: true},
@@ -43,8 +43,8 @@ module.exports = {
       representative: {type: Sequelize.BOOLEAN},
       url: {type: Sequelize.STRING({length: 256})},
       lhr: {type: Sequelize.TEXT},
-      createdAt: {type: Sequelize.DATE()},
-      updatedAt: {type: Sequelize.DATE()},
+      createdAt: {type: Sequelize.DATE(6)},
+      updatedAt: {type: Sequelize.DATE(6)},
     });
     await queryInterface.createTable('statistics', {
       id: {type: Sequelize.UUID(), primaryKey: true},
@@ -53,8 +53,8 @@ module.exports = {
       url: {type: Sequelize.STRING({length: 256})},
       name: {type: Sequelize.STRING({length: 100})},
       value: {type: Sequelize.NUMERIC(12, 4)},
-      createdAt: {type: Sequelize.DATE()},
-      updatedAt: {type: Sequelize.DATE()},
+      createdAt: {type: Sequelize.DATE(6)},
+      updatedAt: {type: Sequelize.DATE(6)},
     });
   },
   /**

packages/server/src/api/storage/sql/migrations/20191009-project-token.js

@@ -14,7 +14,12 @@ module.exports = {
    */
   up: async (queryInterface, Sequelize) => {
     await queryInterface.addColumn('projects', 'token', {type: Sequelize.UUID()});
-    await queryInterface.bulkUpdate('projects', {token: Sequelize.col('id')}, {token: null});
+    await queryInterface.bulkUpdate(
+      'projects',
+      {token: Sequelize.col('id')},
+      {token: null},
+      {type: Sequelize.QueryTypes.BULKUPDATE}
+    );
   },
   /**
    * @param {import('sequelize').QueryInterface} queryInterface

packages/server/src/api/storage/sql/migrations/20191029-committed-at.js

@@ -13,8 +13,8 @@ module.exports = {
    * @param {typeof import('sequelize')} Sequelize
    */
   up: async (queryInterface, Sequelize) => {
-    await queryInterface.addColumn('builds', 'committedAt', {type: Sequelize.DATE()});
-    await queryInterface.addColumn('builds', 'ancestorCommittedAt', {type: Sequelize.DATE()});
+    await queryInterface.addColumn('builds', 'committedAt', {type: Sequelize.DATE(6)});
+    await queryInterface.addColumn('builds', 'ancestorCommittedAt', {type: Sequelize.DATE(6)});
   },
   /**
    * @param {import('sequelize').QueryInterface} queryInterface

packages/server/src/api/storage/sql/migrations/2019110801-project-slug.js

@@ -14,7 +14,12 @@ module.exports = {
    */
   up: async (queryInterface, Sequelize) => {
     await queryInterface.addColumn('projects', 'slug', {type: Sequelize.STRING(40)});
-    await queryInterface.bulkUpdate('projects', {slug: Sequelize.col('id')}, {slug: null});
+    await queryInterface.bulkUpdate(
+      'projects',
+      {slug: Sequelize.col('id')},
+      {slug: null},
+      {type: Sequelize.QueryTypes.BULKUPDATE}
+    );
     await queryInterface.addIndex('projects', {
       // @ts-ignore - Sequelize types are out of date
       name: 'projects_unique_slug',

packages/server/src/api/storage/sql/migrations/2019111001-statistic-version.js

@@ -14,7 +14,12 @@ module.exports = {
    */
   up: async (queryInterface, Sequelize) => {
     await queryInterface.addColumn('statistics', 'version', {type: Sequelize.NUMERIC(8, 2)});
-    await queryInterface.bulkUpdate('statistics', {version: 1}, {version: null});
+    await queryInterface.bulkUpdate(
+      'statistics',
+      {version: 1},
+      {version: null},
+      {type: Sequelize.QueryTypes.BULKUPDATE}
+    );
   },
   /**
    * @param {import('sequelize').QueryInterface} queryInterface

packages/server/src/api/storage/sql/project-model.js

@@ -16,6 +16,8 @@ const attributes = {
   slug: {type: Sequelize.STRING(40)},
   externalUrl: {type: Sequelize.STRING(256)},
   token: {type: Sequelize.UUID()},
+  createdAt: {type: Sequelize.DATE(6)},
+  updatedAt: {type: Sequelize.DATE(6)},
 };
 
 module.exports = {

packages/server/src/api/storage/sql/run-model.js

@@ -20,6 +20,8 @@ const attributes = {
   representative: {type: Sequelize.BOOLEAN},
   url: {type: Sequelize.STRING({length: 256})},
   lhr: {type: Sequelize.TEXT('long')},
+  createdAt: {type: Sequelize.DATE(6)},
+  updatedAt: {type: Sequelize.DATE(6)},
 };
 
 module.exports = {

packages/server/src/api/storage/sql/statistic-model.js

@@ -21,6 +21,8 @@ const attributes = {
   url: {type: Sequelize.STRING({length: 256})},
   name: {type: Sequelize.STRING({length: 100})},
   value: {type: Sequelize.NUMERIC(12, 4)},
+  createdAt: {type: Sequelize.DATE(6)},
+  updatedAt: {type: Sequelize.DATE(6)},
 };
 
 module.exports = {

packages/server/test/mysql-server.test.js

@@ -0,0 +1,47 @@
+/**
+ * @license Copyright 2019 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+/* eslint-env jest */
+
+const runTests = require('./server-test-suite.js').runTests;
+const runServer = require('../src/server.js').createServer;
+
+describe('mysql server', () => {
+  if (!process.env.MYSQL_DB_URL) {
+    it.skip('should work on mysql', () => {});
+    return;
+  }
+
+  const state = {
+    port: undefined,
+    closeServer: undefined,
+  };
+
+  beforeAll(async () => {
+    const {port, close, storageMethod} = await runServer({
+      logLevel: 'silent',
+      port: 0,
+      storage: {
+        storageMethod: 'sql',
+        sqlDialect: 'mysql',
+        sqlConnectionUrl: process.env.MYSQL_DB_URL,
+        sqlConnectionSsl: !!process.env.MYSQL_DB_SSL,
+        sqlDangerouslyResetDatabase: true,
+      },
+    });
+
+    state.port = port;
+    state.closeServer = close;
+    state.storageMethod = storageMethod;
+  });
+
+  afterAll(() => {
+    state.closeServer();
+  });
+
+  runTests(state);
+});