Mysql test

.github/workflows/test.yml

@@ -25,6 +25,9 @@ jobs:
           - test-mariadb-11.4
           - test-mariadb-11.7.2
           - test-mariadb-11.8
+          - test-mysql-8.0.44
+          - test-mysql-8.4
+          - test-mysql-9.5.0
 
     steps:
       - name: Checkout repository

Makefile

@@ -23,7 +23,7 @@ resources/public/admin:
 # All other phony targets run lrsql instances that can be used and tested
 # during development. All start up with fixed DB properties and seed creds.
 
-.phony: clean-dev, ci, ephemeral, ephemeral-prod, sqlite, postgres, bench, bench-async, keycloak-demo, ephemeral-oidc, superset-demo, clamav-demo, test-sqlite, test-postgres, test-postgres-11, test-postgres-12, test-postgres-13, test-postgres-14, test-postgres-15, test-mariadb, test-mariadb-10.6, test-mariadb-10.11, test-mariadb-11.4, test-mariadb-11.7.2, test-mariadb-11.8, mariadb
+.phony: clean-dev, ci, ephemeral, ephemeral-prod, sqlite, postgres, mariadb, mysql, bench, bench-async, keycloak-demo, ephemeral-oidc, superset-demo, clamav-demo, test-sqlite, test-postgres, test-postgres-11, test-postgres-12, test-postgres-13, test-postgres-14, test-postgres-15, test-mariadb, test-mariadb-10.6, test-mariadb-10.11, test-mariadb-11.4, test-mariadb-11.7.2, test-mariadb-11.8, test-mysql, test-mysql-8.0.44, test-mysql-8.4, test-mysql-9.5.0
 
 clean-dev:
 	rm -rf *.db *.log resources/public tmp
@@ -33,10 +33,15 @@ clean-dev:
 test-sqlite:
 	clojure -M:test -m lrsql.test-runner --database sqlite $(if $(ns),--ns $(ns))
 
+# Without LRSQL_TEST_DB_VERSION, defaults to version 11
 TEST_PG_COMMAND ?= clojure -M:test -m lrsql.test-runner --database postgres $(if $(ns),--ns $(ns))
+
+# Without LRSQL_TEST_DB_VERSION, defaults to version 11.7.2
 TEST_MARIADB_COMMAND ?= clojure -M:test -m lrsql.test-runner --database mariadb $(if $(ns),--ns $(ns))
 
-# Without LRSQL_TEST_DB_VERSION, defaults to version 11
+# Without LRSQL_TEST_DB_VERSION, defaults to version 8.0.44
+TEST_MYSQL_COMMAND ?= clojure -M:test -m lrsql.test-runner --database mysql $(if $(ns),--ns $(ns))
+
 test-postgres:
 	$(TEST_PG_COMMAND)
 
@@ -73,10 +78,17 @@ test-mariadb-11.7.2:
 test-mariadb-11.8:
 	LRSQL_TEST_DB_VERSION=11.8 $(TEST_MARIADB_COMMAND)
 
+test-mysql:
+	 $(TEST_MYSQL_COMMAND)
 
-ci: test-sqlite test-postgres test-mariadb
-
+test-mysql-8.0.44:
+	LRSQL_TEST_DB_VERSION=8.0.44 $(TEST_MYSQL_COMMAND)
+test-mysql-8.4:
+	LRSQL_TEST_DB_VERSION=8.4 $(TEST_MYSQL_COMMAND)
+test-mysql-9.5.0:
+	LRSQL_TEST_DB_VERSION=9.5.0 $(TEST_MYSQL_COMMAND)
 
+ci: test-sqlite test-postgres test-mariadb test-mysql
 
 # Dev
 
@@ -220,6 +232,10 @@ target/bundle/lrsql_mariadb.exe: exe/lrsql_mariadb.exe
 	mkdir -p target/bundle
 	cp exe/lrsql_mariadb.exe target/bundle/lrsql_mariadb.exe
 
+target/bundle/lrsql_mysql.exe: exe/lrsql_mysql.exe
+	mkdir -p target/bundle
+	cp exe/lrsql_mysql.exe target/bundle/lrsql_mysql.exe
+
 # Copy Admin UI
 
 target/bundle/admin: resources/public/admin
@@ -231,9 +247,9 @@ target/bundle/admin: resources/public/admin
 BUNDLE_RUNTIMES ?= true
 
 ifeq ($(BUNDLE_RUNTIMES),true)
-target/bundle: target/bundle/config target/bundle/doc target/bundle/bin target/bundle/lrsql.jar target/bundle/admin target/bundle/lrsql.exe target/bundle/lrsql_pg.exe target/bundle/lrsql_mariadb.exe target/bundle/LICENSE target/bundle/NOTICE target/bundle/customization target/bundle/bench.jar target/bundle/bench target/bundle/runtimes
+target/bundle: target/bundle/config target/bundle/doc target/bundle/bin target/bundle/lrsql.jar target/bundle/admin target/bundle/lrsql.exe target/bundle/lrsql_pg.exe target/bundle/lrsql_mariadb.exe target/bundle/lrsql_mysql.exe target/bundle/LICENSE target/bundle/NOTICE target/bundle/customization target/bundle/bench.jar target/bundle/bench target/bundle/runtimes
 else
-target/bundle: target/bundle/config target/bundle/doc target/bundle/bin target/bundle/lrsql.jar target/bundle/admin target/bundle/lrsql.exe target/bundle/lrsql_pg.exe target/bundle/lrsql_mariadb.exe target/bundle/LICENSE target/bundle/NOTICE target/bundle/customization target/bundle/bench.jar target/bundle/bench
+target/bundle: target/bundle/config target/bundle/doc target/bundle/bin target/bundle/lrsql.jar target/bundle/admin target/bundle/lrsql.exe target/bundle/lrsql_pg.exe target/bundle/lrsql_mariadb.exe target/bundle/lrsql_mysql.exe target/bundle/LICENSE target/bundle/NOTICE target/bundle/customization target/bundle/bench.jar target/bundle/bench
 endif
 
 bundle: target/bundle
@@ -283,15 +299,21 @@ else
 	launch4j exe/config_mariadb.xml
 endif
 
+exe/lrsql_mysql.exe:
+ifeq (,$(shell which launch4j))
+	$(error "ERROR: launch4j is not installed!")
+else
+	launch4j exe/config_mysql.xml
+endif
 
-exe: exe/lrsql.exe exe/lrsql_pg.exe exe/lrsql_mariadb.exe
+exe: exe/lrsql.exe exe/lrsql_pg.exe exe/lrsql_mariadb.exe exe/lrsql_mysql.exe
 
 # *** Run build ***
 
 # These targets create a bundle containing a lrsql JAR and then runs
 # the JAR to create the specific lrsql instance.
 
-.phony: run-jar-sqlite, run-jar-sqlite-ephemeral, run-jar-postgres, run-jar-mariadb
+.phony: run-jar-sqlite, run-jar-sqlite-ephemeral, run-jar-postgres, run-jar-mariadb, run-jar-mysql
 
 run-jar-sqlite-ephemeral: target/bundle
 	cd target/bundle; \
@@ -326,6 +348,14 @@ run-jar-mariadb: target/bundle
 	LRSQL_API_SECRET_DEFAULT=password \
 	bin/run_mariadb.sh
 
+run-jar-mysql: target/bundle
+	cd target/bundle; \
+	LRSQL_ADMIN_USER_DEFAULT=username \
+	LRSQL_ADMIN_PASS_DEFAULT=password \
+	LRSQL_API_KEY_DEFAULT=username \
+	LRSQL_API_SECRET_DEFAULT=password \
+	bin/run_mysql.sh
+
 # *** Report Dependency Graph to GitHub ***
 
 ## This pom.xml file is generated solely in an action to populate the GitHub

bin/run_mysql.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+
+MACHINE=`bin/machine.sh`
+
+runtimes/$MACHINE/bin/java -Dfile.encoding=UTF-8 -server -cp lrsql.jar lrsql.mariadb.main $@
+#COMMENT: SQL-LRS uses the same code to interface with MySQL as it uses for MariaDB; the above is intentional.

deps.edn

@@ -125,7 +125,10 @@
                  {:mvn/version "2.6.1"}
                  org.mariadb.jdbc/mariadb-java-client
                  {:mvn/version "3.5.3"}
+                 com.mysql/mysql-connector-j
+                 {:mvn/version "9.4.0"}
                  org.testcontainers/mariadb {:mvn/version "1.21.2"}
+                 org.testcontainers/mysql   {:mvn/version "1.21.3"}
 
                  ;; Other test deps
                  org.clojure/test.check {:mvn/version "1.1.1"}

dev-resources/mariadb/docker-compose.yml

@@ -19,8 +19,8 @@ services:
   clamav:
     image: clamav/clamav:1.2.1
   lrs:
-    build: ../../. # switch to this for active dev
-    #image: yetanalytics/lrsql:latest
+    #build: ../../. # switch to this for active dev
+    image: yetanalytics/lrsql:latest
     command:
       - /lrsql/bin/run_mariadb.sh
     ports:

dev-resources/mysql/docker-compose.yml

@@ -1,11 +1,11 @@
-# Runs SQL LRS with MariaDB - Provided for demonstration purposes only!
+# Runs SQL LRS with MySQL - Provided for demonstration purposes only!
 # To run: docker compose up
 # See the Docker Compose docs for more info: https://docs.docker.com/compose/
 volumes:
   db_data:
 services:
   db:
-    image: mysql:8.0
+    image: mysql:9.5.0
     volumes:
       - db_data:/var/lib/mysql/
     environment:
@@ -19,16 +19,15 @@ services:
   clamav:
     image: clamav/clamav:1.2.1
   lrs:
-    build: ../../. # switch to this for active dev
-    #image: yetanalytics/lrsql:latest
+    #build: ../../. # switch to this for active dev
+    image: yetanalytics/lrsql:latest
     command:
-      - /lrsql/bin/run_mariadb.sh
+      - /lrsql/bin/run_mysql.sh
     ports:
       - "8080:8080"
     depends_on:
       - db
     environment:
-      LRSQL_DB_TYPE: mysql
       LRSQL_API_KEY_DEFAULT: my_key
       LRSQL_API_SECRET_DEFAULT: my_secret
       LRSQL_ADMIN_USER_DEFAULT: my_username
@@ -37,7 +36,7 @@ services:
       LRSQL_DB_NAME: lrsql_db
       LRSQL_DB_USER: lrsql_user
       LRSQL_DB_PASSWORD: lrsql_password
-      # If MariaDB is too slow to start, increase this
+      # If MySQL is too slow to start, increase this
       LRSQL_POOL_INITIALIZATION_FAIL_TIMEOUT: 10000
       # Set to true if using dev UI, domain name, proxy server, etc.
       LRSQL_ALLOW_ALL_ORIGINS: false

doc/dev.md

@@ -23,7 +23,16 @@ The SQL LRS can be built or run with the following Makefile targets. They can be
 | ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
 | `ci`               | Called when running continuous integration; runs all test cases in all SQL flavors.                                                                                                                          |
 | `test-sqlite`      | Run all tests with SQLite database.                                                                                                                                                                          |
-| `test-mariadb`       | Run all tests with Mariadb database version 11.7.2. Set the `LRSQL_TEST_DB_VERSION` env var to a valid Postgres docker tag to use another version.                                                              |
+| `test-mariadb`       | Run all tests with Mariadb database version 11.7.2. Set the `LRSQL_TEST_DB_VERSION` env var to a valid MariaDB docker tag to use another version.                                                              |
+| `test-mariadb-10.6`       | Run all tests with Mariadb database version 10.6.                                                              |
+| `test-mariadb-10.11`       | Run all tests with Mariadb database version 10.11.                                                              |
+| `test-mariadb-11.4`       | Run all tests with Mariadb database version 11.4.                                                              |
+| `test-mariadb-11.7.2`       | Run all tests with Mariadb database version 11.7.2.                                                              |
+| `test-mariadb-11.8`       | Run all tests with Mariadb database version 11.8.                                                              |
+| `test-mysql`       | Run all tests with MySQL database version 8.0.44. Set the `LRSQL_TEST_DB_VERSION` env var to a valid MySQL docker tag to use another version.                                                                                                                                                             |
+| `test-mysql-8.0.44` | Run all tests with MySQL database version 8.0.44                                                                                                                                                             |
+| `test-mysql-8.4` | Run all tests with MySQL database version 8.4                                                                                                                                                      |
+| `test-mysql-9.5.0` | Run all tests with MySQL database version 9.5.0                                                                                                                                                             |
 | `test-postgres`    | Run all tests with Postgres database version 11. Set the `LRSQL_TEST_DB_VERSION` env var to a valid Postgres docker tag to use another version.                                                              |
 | `test-postgres-11` | Run all tests with Postgres database version 11.                                                                                                                                                             |
 | `test-postgres-12` | Run all tests with Postgres database version 12.                                                                                                                                                             |
@@ -43,10 +52,13 @@ The SQL LRS can be built or run with the following Makefile targets. They can be
 | `sqlite`                   | Start a SQLite-based SQL LRS.                                                                                                              |
 | `postgres`                 | Start a Postgres SQL LRS. Requires a running Postgres instance.                                                                            |
 | `mariadb`                  | Start a Mariadb SQL LRS. Requires a running MariaDB instance.                                                                             |
+| `mysql`                    | Start a MySQL SQL LRS. Requires a running MySQL instance.                                                                             |
 | `run-jar-sqlite`           | Similar to `sqlite` but it runs the finished Jar instead of directly from Clojure. Runs with a predefined default set of env variables.    |
 | `run-jar-sqlite-ephemeral` | Similar to `ephemeral` but it runs the finished Jar instead of directly from Clojure. Runs with a predefined default set of env variables. |
 | `run-jar-postgres`         | Similar to `postgres` but it runs the finished Jar instead of directly from Clojure. Runs with a predefined default set of env variables.  |
 | `run-jar-mariadb`          | Similar to `mariadb` but it runs the finished Jar instead of directly from Clojure. Runs with a predefined default set of env variables.  |
+| `run-jar-mysql`          | Similar to `mysql` but it runs the finished Jar instead of directly from Clojure. Runs with a predefined default set of env variables.  |
+
 
 #### Cleanup Targets
 

doc/index.md

@@ -22,6 +22,7 @@
 
 - [Postgres](postgres.md)
 - [MariaDB](mariadb.md)
+- [MySQL](mysql.md)
 - [SQLite](sqlite.md)
 
 ### Reference

doc/mysql.md

@@ -0,0 +1,41 @@
+[<- Back to Index](index.md)
+
+# MySQL
+
+SQL-LRS supports MySQL.
+
+To run SQL-LRS with MySQL, you need a properly configured MySQL instance.  "Properly configured" really just means setting up a database, user, and password, and configuring SQL-LRS to connect to/with those.
+
+## Environment Variables
+
+You can configure MySQLDB by setting the relevant environment variables.  Here is the `environment` entry from our [MySQL docker-compose demo](https://github.com/yetanalytics/lrsql/blob/main/dev-resources/mysql/docker-compose.yml)
+
+```yml
+    environment:
+      MYSQL_ROOT_PASSWORD: lrsql_root_password 
+      MYSQL_DATABASE: lrsql_db
+      MYSQL_USER: lrsql_user
+      MYSQL_PASSWORD: lrsql_password
+```
+
+(`MYSQL_ROOT_PASSWORD` can be anything, as SQL-LRS doesn't use the root account.  It is only included here because the MySQL Docker container requires a root password setting.  See the [MySQL docs](https://dev.mysql.com/doc/refman/8.4/en/docker-mysql-more-topics.html] for details)
+
+The corresponding `lrsql.json` would look like
+
+```json
+{
+  ...
+  "database": {
+    "dbHost": "0.0.0.0",
+    "dbPort": 3306,
+    "dbName": "lrsql_db",
+    "dbUser": "lrsql_user",
+    "dbPassword": "lrsql_password",
+  }
+}
+
+```
+
+## Precision Limitation
+
+SQL-LRS stores statements in its implementation databases as JSON.  Due to the way MySQL interprets numbers in JSON, we **cannot guarantee more than 15 significant digits of precision** when using SQL-LRS alongside MySQL.  If you need that much precision, consider using SQL-LRS alongside [Postgres](postgres.md) or [MariaDB](mariadb.md) instead.

docker-compose.yml

@@ -7,7 +7,7 @@ services:
   db:
     image: postgres
     volumes:
-      - db_data:/var/lib/postgresql/data
+      - db_data:/var/lib/postgresql
     environment:
       POSTGRES_USER: lrsql_user
       POSTGRES_PASSWORD: lrsql_password

exe/config_mysql.xml

@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<launch4jConfig>
+  <dontWrapJar>true</dontWrapJar>
+  <headerType>console</headerType>
+  <classPath>
+    <mainClass>lrsql.mariadb.main</mainClass>
+    <cp>lrsql.jar</cp>
+  </classPath>
+  <outfile>lrsql_mysql.exe</outfile>
+  <errTitle>You must have Java 11+ installed to run SQL LRS</errTitle>
+  <chdir>.</chdir>
+  <priority>normal</priority>
+  <downloadUrl>http://java.com/download</downloadUrl>
+  <supportUrl>https://yet.zendesk.com/hc/en-us</supportUrl>
+  <stayAlive>false</stayAlive>
+  <restartOnCrash>false</restartOnCrash>
+  <manifest></manifest>
+  <icon>lrsql.ico</icon>
+  <jre>
+    <path>runtimes/windows</path>
+    <bundledJre64Bit>false</bundledJre64Bit>
+    <bundledJreAsFallback>false</bundledJreAsFallback>
+    <minVersion>11</minVersion>
+    <maxVersion></maxVersion>
+    <jdkPreference>preferJre</jdkPreference>
+    <runtimeBits>64/32</runtimeBits>
+    <opt>-Dfile.encoding=UTF-8</opt>
+  </jre>
+</launch4jConfig>

resources/lrsql/config/config.edn

@@ -29,6 +29,18 @@
 
                  ;; Testing Only! Specify the version used with testcontainers
                  :test-db-version #or [#env LRSQL_TEST_DB_VERSION "11.7.2"]}
+
+  :test-mysql {:db-type       "mysql"
+               :db-name       "lrsql_db"
+               :db-host       "0.0.0.0"
+               :db-port       3306
+               :db-user       "lrsql_user"
+               :db-password   "lrsql_password"
+               :db-properties "allowMultiQueries=true"
+
+               ;; Testing Only! Specify the version used with testcontainers
+               :test-db-version #or [#env LRSQL_TEST_DB_VERSION "8.0.44"]}
+
   :test-oidc {:db-type "sqlite"
               :db-name ":memory:"}
 
@@ -45,12 +57,14 @@
   :test-sqlite-mem #include "test/sqlite_mem/connection.edn"
   :test-postgres   #include "test/postgres/connection.edn"
   :test-mariadb    #include "test/mariadb/connection.edn"
+  :test-mysql      #include "test/mysql/connection.edn"
   :test-oidc       #include "test/default/connection.edn"
   ;; Production
   :prod-sqlite     #include "prod/default/connection.edn"
   :prod-sqlite-mem #include "prod/sqlite_mem/connection.edn"
   :prod-postgres   #include "prod/postgres/connection.edn"
   :prod-mariadb    #include "prod/mariadb/connection.edn"
+  :prod-mysql    #include   "prod/mysql/connection.edn"
   }
  :tuning
  #profile
@@ -59,12 +73,14 @@
   :test-sqlite-mem #include "test/default/tuning.edn"
   :test-postgres   #include "test/postgres/tuning.edn"
   :test-mariadb    #include "test/mariadb/tuning.edn"
+  :test-mysql      #include "test/mysql/tuning.edn"
   :test-oidc       #include "test/default/tuning.edn"
   ;; Production
   :prod-sqlite     #include "prod/default/tuning.edn"
   :prod-sqlite-mem #include "prod/default/tuning.edn"
   :prod-postgres   #include "prod/postgres/tuning.edn"
   :prod-mariadb    #include "prod/mariadb/tuning.edn"
+  :prod-mysql      #include "prod/mysql/tuning.edn"
   }
  :lrs
  #profile
@@ -73,26 +89,30 @@
   :test-sqlite-mem #include "test/default/lrs.edn"
   :test-postgres   #include "test/default/lrs.edn"
   :test-mariadb    #include "test/default/lrs.edn"
+  :test-mysql      #include "test/default/lrs.edn"
   :test-oidc       #include "test/oidc/lrs.edn"
   ;; Production
   :prod-sqlite     #include "prod/default/lrs.edn"
   :prod-sqlite-mem #include "prod/default/lrs.edn"
   :prod-postgres   #include "prod/default/lrs.edn"
   :prod-mariadb    #include "prod/default/lrs.edn"
+  :prod-mysql      #include "prod/default/lrs.edn"
   }
  :webserver
  #profile
  {;; Test/Dev
   :test-sqlite     #include "test/default/webserver.edn"
   :test-sqlite-mem #include "test/default/webserver.edn"
   :test-postgres   #include "test/default/webserver.edn"
-  :test-mariadb    #include  "test/default/webserver.edn"
+  :test-mariadb    #include "test/default/webserver.edn"
+  :test-mysql      #include "test/default/webserver.edn"
   :test-oidc       #include "test/oidc/webserver.edn"
   ;; Production
   :prod-sqlite     #include "prod/default/webserver.edn"
   :prod-sqlite-mem #include "prod/default/webserver.edn"
   :prod-postgres   #include "prod/default/webserver.edn"
-  :prod-mariadb    #include  "prod/default/webserver.edn"
+  :prod-mariadb    #include "prod/default/webserver.edn"
+  :prod-mysql      #include "prod/default/webserver.edn"
   }
  ;; A user-provided JSON file for merge-with merge into this map
  :user-config-json #or [#env LRSQL_USER_CONFIG_JSON "config/lrsql.json"]

resources/lrsql/config/prod/mysql/connection.edn

@@ -0,0 +1,4 @@
+#merge
+[#include "prod/default/connection.edn"
+ {:pool-minimum-idle #long #or [#env LRSQL_POOL_MINIMUM_IDLE 10]
+  :pool-maximum-size #long #or [#env LRSQL_POOL_MAXIMUM_SIZE 10]}]